Skip to content

Commit

Permalink
git subrepo pull (merge) microservices/services/query-metric
Browse files Browse the repository at this point in the history 
subrepo:
  subdir:   "microservices/services/query-metric"
  merged:   "3141847bcf"
upstream:
  origin:   "[email protected]:NationalSecurityAgency/datawave-query-metric-service.git"
  branch:   "main"
  commit:   "08c303c49d"
git-subrepo:
  version:  "0.4.9"
  origin:   "https://github.com/ingydotnet/git-subrepo"
  commit:   "cce3d93"
  • Loading branch information
@ivakegg
ivakegg committed Jan 6, 2025
1 parent 3396eec commit 931d801
Show file tree
Hide file tree
Showing 10 changed files with 92 additions and 195 deletions.
2 changes: 1 addition & 1 deletion microservices/services/query-metric/.gitrepo
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
[subrepo]
remote = [email protected]:NationalSecurityAgency/datawave-query-metric-service.git
branch = main
commit = 4bce0c89219d8bb7901f1e9c4993460a7bd50452
commit = 08c303c49d2c648b29242e0d7828294b6ff83bdd
parent = 5b0f7db7e95406ad92030eca551caa579d7debe9
method = merge
cmdver = 0.4.9
6 changes: 3 additions & 3 deletions microservices/services/query-metric/api/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
<relativePath>../../../microservice-parent/pom.xml</relativePath>
</parent>
<artifactId>query-metric-api</artifactId>
<version>4.0.10-SNAPSHOT</version>
<version>4.0.12-SNAPSHOT</version>
<url>https://code.nsa.gov/datawave-query-metric-service</url>
<scm>
<connection>scm:git:https://github.com/NationalSecurityAgency/datawave-query-metric-service.git</connection>
Expand All @@ -21,8 +21,8 @@
<spotbugs.excludes.file>${basedir}/spotbugs-exclude.xml</spotbugs.excludes.file>
<version.commons-lang3>3.12.0</version.commons-lang3>
<version.commons-text>1.9</version.commons-text>
<version.datawave>7.1.3</version.datawave>
<version.datawave.accumulo-utils>4.0.0</version.datawave.accumulo-utils>
<version.datawave>7.10.1</version.datawave>
<version.datawave.accumulo-utils>4.0.1</version.datawave.accumulo-utils>
<version.datawave.base-rest-responses>4.0.0</version.datawave.base-rest-responses>
<version.datawave.query-api>1.0.0</version.datawave.query-api>
<version.glassfish>2.3.6</version.glassfish>
Expand Down
7 changes: 5 additions & 2 deletions ...rvices/services/query-metric/api/src/main/resources/templates/querymetric-horizontal.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ <h1>Query Metrics</h1>
</script>

<div th:each="metric,metricIter : ${metricList}">
<h3 th:text="${metric.queryId}"/>
<table class="metric-horizontal">
<tr>
<th>Visibility</th>
Expand Down Expand Up @@ -93,6 +94,7 @@ <h1>Query Metrics</h1>
<td th:text="${metric.createCallTimeStr}"></td>
<td th:text="${metric.numPagesStr}"></td>
<td th:text="${metric.numResultsStr}"></td>
<td th:text="${metric.docSizeStr}"></td>
<td th:text="${metric.docRangesStr}"></td>
<td th:text="${metric.fiRangesStr}"></td>
<td th:text="${metric.sourceCountStr}"></td>
Expand Down Expand Up @@ -144,8 +146,9 @@ <h1>Query Metrics</h1>
<td th:text="${page.bytesWrittenStr}"></td>
</tr>
</table>
<br th:if="metricIter.last == false"/>
<br th:if="metricIter.last == false"/>
<br th:if="${!metricIter.last}"/>
<br th:if="${!metricIter.last}"/>
<br th:if="${!metricIter.last}"/>
</div>
<br/>
<div th:utext="${footer}"></div>
Expand Down
27 changes: 15 additions & 12 deletions microservices/services/query-metric/api/src/main/resources/templates/querymetric.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ <h1>Query Metrics</h1>


<div th:each="metric,metricIter : ${metricList}">
<h3 th:text="${metric.queryId}"/>
<table class="metric-vertical">
<tr class="highlight"><td>Visibility</td><td th:text="${metric.columnVisibility}"></td></tr>
<tr><td>Query Date</td><td th:text="${metric.createDateStr}"></td></tr>
Expand Down Expand Up @@ -125,16 +126,17 @@ <h1>Query Metrics</h1>
<tr class="highlight"><td>Login Time (ms)</td><td th:text="${metric.loginTimeStr}"></td></tr>
<tr><td>Query Setup Time (ms)</td><td th:text="${metric.setupTimeStr}"></td></tr>
<tr class="highlight"><td>Query Setup Call Time (ms)</td><td th:text="${metric.createCallTimeStr}"></td></tr>
<tr><td>Doc Ranges</td><td th:text="${metric.docRangesStr}"></td></tr>
<tr class="highlight"><td>FI Ranges</td><td th:text="${metric.fiRangesStr}"></td></tr>
<tr><td>Sources</td><td th:text="${metric.sourceCountStr}"></td></tr>
<tr class="highlight"><td>Next Calls</td><td th:text="${metric.nextCountStr}"></td>
<tr><td>Seek Calls</td><td th:text="${metric.seekCountStr}"></td></tr>
<tr class="highlight"><td>Yield Count</td><td th:text="${metric.yieldCountStr}"></td></tr>
<tr><td>Total Page Time (ms)</td><td th:text="${metric.totalPageTimeStr}"></td></tr>
<tr class="highlight"><td>Total Page Call Time (ms)</td><td th:text="${metric.totalPageCallTimeStr}"></td></tr>
<tr><td>Total Page Serialization Time (ms)</td><td th:text="${metric.totalSerializationTimeStr}"></td>
<tr class="highlight"><td>Total Page Bytes Sent (uncompressed)</td><td th:text="${metric.totalBytesSentStr}"></td></tr>
<tr><td>Doc Size</td><td th:text="${metric.docSizeStr}"></td></tr>
<tr class="highlight"><td>Doc Ranges</td><td th:text="${metric.docRangesStr}"></td></tr>
<tr><td>FI Ranges</td><td th:text="${metric.fiRangesStr}"></td></tr>
<tr class="highlight"><td>Sources</td><td th:text="${metric.sourceCountStr}"></td></tr>
<tr><td>Next Calls</td><td th:text="${metric.nextCountStr}"></td>
<tr class="highlight"><td>Seek Calls</td><td th:text="${metric.seekCountStr}"></td></tr>
<tr><td>Yield Count</td><td th:text="${metric.yieldCountStr}"></td></tr>
<tr class="highlight"><td>Total Page Time (ms)</td><td th:text="${metric.totalPageTimeStr}"></td></tr>
<tr><td>Total Page Call Time (ms)</td><td th:text="${metric.totalPageCallTimeStr}"></td></tr>
<tr class="highlight"><td>Total Page Serialization Time (ms)</td><td th:text="${metric.totalSerializationTimeStr}"></td>
<tr><td>Total Page Bytes Sent (uncompressed)</td><td th:text="${metric.totalBytesSentStr}"></td></tr>

<script type="text/javascript">
// Makes the metric's query and the metric's query plan interactive (highlight
Expand Down Expand Up @@ -178,8 +180,9 @@ <h1>Query Metrics</h1>
<td th:text="${page.bytesWrittenStr}"></td>
</tr>
</table>
<br th:if="metricIter.last == false"/>
<br th:if="metricIter.last == false"/>
<br th:if="${!metricIter.last}"/>
<br th:if="${!metricIter.last}"/>
<hr style="height:4px;width:100%" th:if="${!metricIter.last}"/>
</div>
<br/>
<div th:utext="${footer}"></div>
Expand Down
48 changes: 6 additions & 42 deletions microservices/services/query-metric/service/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
<relativePath>../../../microservice-service-parent/pom.xml</relativePath>
</parent>
<artifactId>query-metric-service</artifactId>
<version>4.0.10-SNAPSHOT</version>
<version>4.0.13-SNAPSHOT</version>
<description>DATAWAVE Query Metric Microservice</description>
<url>https://code.nsa.gov/datawave-query-metric-service</url>
<scm>
Expand All @@ -21,16 +21,15 @@
<spotbugs.excludes.file>${basedir}/spotbugs-exclude.xml</spotbugs.excludes.file>
<start-class>datawave.microservice.querymetric.QueryMetricService</start-class>
<version.cache-api>1.1.1</version.cache-api>
<version.datawave>7.1.3</version.datawave>
<version.datawave>7.10.1</version.datawave>
<version.datawave.accumulo-api>4.0.0</version.datawave.accumulo-api>
<version.datawave.accumulo-utils>4.0.0</version.datawave.accumulo-utils>
<version.datawave.accumulo-utils>4.0.1</version.datawave.accumulo-utils>
<version.datawave.base-rest-responses>4.0.0</version.datawave.base-rest-responses>
<version.datawave.hazelcast-common>4.0.2</version.datawave.hazelcast-common>
<version.datawave.query-metric-api>4.0.8</version.datawave.query-metric-api>
<version.datawave.starter>4.0.2</version.datawave.starter>
<version.datawave.query-metric-api>4.0.11</version.datawave.query-metric-api>
<version.datawave.starter>4.0.3</version.datawave.starter>
<version.datawave.starter-datawave-query-metric>3.0.3</version.datawave.starter-datawave-query-metric>
<version.datawave.starter-metadata>3.0.2</version.datawave.starter-metadata>
<version.datawave.type-utils>3.0.0</version.datawave.type-utils>
<version.hazelcast-kubernetes>2.2.3</version.hazelcast-kubernetes>
<version.in-memory-accumulo>4.0.0</version.in-memory-accumulo>
<version.protobuf-java>3.7.1</version.protobuf-java>
Expand All @@ -39,7 +38,7 @@
<version.webjars.jquery>3.4.1</version.webjars.jquery>
<version.webjars.leaflet>1.5.1</version.webjars.leaflet>
<version.webjars.locator-core>0.35</version.webjars.locator-core>
<version.zookeeper>3.8.0</version.zookeeper>
<version.zookeeper>3.8.3</version.zookeeper>
</properties>
<dependencyManagement>
<dependencies>
Expand Down Expand Up @@ -220,37 +219,6 @@
<artifactId>spring-boot-starter-datawave-query-metric</artifactId>
<version>${version.datawave.starter-datawave-query-metric}</version>
</dependency>
<dependency>
<groupId>gov.nsa.datawave.microservice</groupId>
<artifactId>type-utils</artifactId>
<version>${version.datawave.type-utils}</version>
<exclusions>
<exclusion>
<artifactId>log4j</artifactId>
<groupId>log4j</groupId>
</exclusion>
<exclusion>
<artifactId>slf4j-reload4j</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>reload4j</artifactId>
<groupId>ch.qos.reload4j</groupId>
</exclusion>
<exclusion>
<artifactId>avro</artifactId>
<groupId>org.apache.avro</groupId>
</exclusion>
<exclusion>
<artifactId>reload4j</artifactId>
<groupId>ch.qos.reload4j</groupId>
</exclusion>
<exclusion>
<artifactId>avro</artifactId>
<groupId>org.apache.avro</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>gov.nsa.datawave.webservices</groupId>
<artifactId>datawave-ws-common</artifactId>
Expand Down Expand Up @@ -372,10 +340,6 @@
<groupId>gov.nsa.datawave.microservice</groupId>
<artifactId>spring-boot-starter-datawave-query-metric</artifactId>
</dependency>
<dependency>
<groupId>gov.nsa.datawave.microservice</groupId>
<artifactId>type-utils</artifactId>
</dependency>
<dependency>
<groupId>gov.nsa.datawave.webservices</groupId>
<artifactId>datawave-ws-common</artifactId>
Expand Down
71 changes: 27 additions & 44 deletions ...metric/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import java.util.Comparator;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.TimeZone;
Expand Down Expand Up @@ -373,16 +374,8 @@ public BaseQueryMetricListResponse query(@AuthenticationPrincipal DatawaveUserDe
BaseQueryMetricListResponse response = queryMetricResponseFactory.createListResponse();
List<BaseQueryMetric> metricList = new ArrayList<>();
try {
BaseQueryMetric metric;
QueryMetricUpdateHolder metricUpdate = incomingQueryMetricsCache.get(queryId, QueryMetricUpdateHolder.class);
if (metricUpdate == null) {
// fetches metric from Accumulo
metric = this.handler.getQueryMetric(queryId);
} else {
// determine if the cached metric is complete, and if not then fetch metric from Accumulo
metric = isMetricComplete(metricUpdate) ? metricUpdate.getMetric() : completeMetric(metricUpdate);
}
if (metric != null) {
List<BaseQueryMetric> metricsFromAccumulo = this.handler.getQueryMetrics(queryId);
for (BaseQueryMetric metric : metricsFromAccumulo) {
boolean allowAllMetrics = false;
boolean sameUser = false;
if (currentUser != null) {
Expand All @@ -405,7 +398,8 @@ public BaseQueryMetricListResponse query(@AuthenticationPrincipal DatawaveUserDe
}
}
if (userCanSeeVisibility) {
metricList.add(metric);
// if returning this metric, then add any updates found in the incomingQueryMetricsCache
metricList.add(completeMetric(metric));
}
}
}
Expand Down Expand Up @@ -491,44 +485,33 @@ public QueryGeometryResponse map(@AuthenticationPrincipal DatawaveUserDetails cu
}

/*
* Try to determine if cached metric is complete or whether it may be missing pages because it was evicted from the incoming cache
* Pages could be missing if the metric was updated since being written to Accumulo
*/
protected boolean isMetricComplete(QueryMetricUpdateHolder metricUpdateHolderFromCache) {
BaseQueryMetric metricUpdate = metricUpdateHolderFromCache.getMetric();
if (metricUpdateHolderFromCache.isNewMetric()) {
return true;
} else {
List<PageMetric> pageMetrics = new ArrayList<>(metricUpdate.getPageTimes());
if (metricUpdate.getNumPages() != pageMetrics.size()) {
return false;
} else {
pageMetrics.sort(Comparator.comparingLong(PageMetric::getPageNumber));
// If the first page's pageNumber is equal to 1 then that's a good indicator that the metric hasn't been evicted
return (pageMetrics.isEmpty()) || (pageMetrics.get(0).getPageNumber() == 1);
}
}
}

/*
* Pages could be missing if the metricUpdate was evicted from the incomingQueryMetricsCache. If so, then retrieve the full metric from Accumulo and combine
* metrics so that the requesting user gets the complete metric.
*/
protected BaseQueryMetric completeMetric(QueryMetricUpdateHolder metricUpdateHolderFromCache) {
BaseQueryMetric metricUpdate = null;
protected BaseQueryMetric completeMetric(BaseQueryMetric metricFromAccumulo) {
String queryId = metricFromAccumulo.getQueryId();
QueryMetricUpdateHolder metricUpdateHolderFromCache = incomingQueryMetricsCache.get(queryId, QueryMetricUpdateHolder.class);
BaseQueryMetric updatedMetric = metricFromAccumulo;
if (metricUpdateHolderFromCache != null) {
metricUpdate = metricUpdateHolderFromCache.getMetric();
if (metricUpdate != null) {
try {
BaseQueryMetric metricFromAccumulo = this.handler.getQueryMetric(metricUpdate.getQueryId());
if (metricFromAccumulo != null) {
metricUpdate = this.handler.combineMetrics(metricUpdate, metricFromAccumulo, metricUpdateHolderFromCache.getMetricType());
}
} catch (Exception e) {
log.error(e.getMessage(), e);
try {
BaseQueryMetric metricFromCache = metricUpdateHolderFromCache.getMetric();
QueryMetricType metricType = metricUpdateHolderFromCache.getMetricType();
if (metricType.equals(QueryMetricType.DISTRIBUTED)) {
// These values are additive, so we don't want to count them twice
// The QueryMetricUpdateHolder resets the values when written to Accumulo
metricFromCache.setSourceCount(metricUpdateHolderFromCache.getValue("sourceCount"));
metricFromCache.setNextCount(metricUpdateHolderFromCache.getValue("nextCount"));
metricFromCache.setSeekCount(metricUpdateHolderFromCache.getValue("seekCount"));
metricFromCache.setYieldCount(metricUpdateHolderFromCache.getValue("yieldCount"));
metricFromCache.setDocSize(metricUpdateHolderFromCache.getValue("docSize"));
metricFromCache.setDocRanges(metricUpdateHolderFromCache.getValue("docRanges"));
metricFromCache.setFiRanges(metricUpdateHolderFromCache.getValue("fiRanges"));
}
updatedMetric = this.handler.combineMetrics(metricFromCache, metricFromAccumulo, metricUpdateHolderFromCache.getMetricType());
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
return metricUpdate;
return updatedMetric;
}

@Operation(summary = "Get a map for the given query represented by the query ID, if applicable.")
Expand Down
Loading

0 comments on commit 931d801

Please sign in to comment.