Add TTL E2E test (apache#3437)

* Add TTL E2E test * Add to Jenkins stage and minor bugfix * Upgrade e2e container version * Polish and minor fix * Polish
MiniSC · Sep 10, 2019 · 721f864 · 721f864
1 parent ce20fe1
commit 721f864
Show file tree

Hide file tree

Showing 20 changed files with 798 additions and 21 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -7,3 +7,6 @@
 [submodule "skywalking-ui"]
 	path = skywalking-ui
 	url = https://github.com/apache/skywalking-rocketbot-ui.git
+[submodule "test/e2e/e2e-ttl/e2e-ttl-es/src/main/proto"]
+	path = test/e2e/e2e-ttl/e2e-ttl-es/src/main/proto
+	url = https://github.com/apache/skywalking-data-collect-protocol.git
diff --git a/Jenkinsfile-E2E b/Jenkinsfile-E2E
@@ -44,12 +44,13 @@ pipeline {
                 // we're using them as a barrier here to filter out some invalid PRs (fast-fail)
                 // thus save unnecessary E2E builds(which is expensive)
                 sh './mvnw checkstyle:check apache-rat:check'
-                sh './mvnw -Dcheckstyle.skip -Drat.skip -T2 -Dmaven.compile.fork -Dmaven.compiler.maxmem=3072 -DskipTests clean package'
+                sh './mvnw -Dcheckstyle.skip -Drat.skip -T2 -Dmaven.compile.fork -Dmaven.compiler.maxmem=3072 -DskipTests clean install'
                 // Some of the tests will modify files in the distribution folder, e.g. cluster test will modify the application.yml
                 // so we give each test a separate distribution folder here
                 sh 'mkdir -p dist-for-single-node-service && tar -zxf dist/apache-skywalking-apm-bin.tar.gz -C dist-for-single-node-service'
                 sh 'mkdir -p dist-for-cluster && tar -zxf dist/apache-skywalking-apm-bin.tar.gz -C dist-for-cluster'
                 sh 'mkdir -p dist-for-agent-reboot && tar -zxf dist/apache-skywalking-apm-bin.tar.gz -C dist-for-agent-reboot'
+                sh 'mkdir -p dist-for-ttl-es && tar -zxf dist/apache-skywalking-apm-bin.tar.gz -C dist-for-ttl-es'
             }
         }
 
@@ -78,6 +79,12 @@ pipeline {
                         sh './mvnw -Dbuild.id=${BUILD_ID} -f test/e2e/pom.xml -pl e2e-agent-reboot -am verify'
                     }
                 }
+
+                stage('Run TTL Tests') {
+                    steps {
+                        sh './mvnw -Dbuild.id=${BUILD_ID} -f test/e2e/pom.xml -pl e2e-ttl/e2e-ttl-es -am verify'
+                    }
+                }
             }
         }
     }
@@ -95,7 +102,7 @@ pipeline {
             // inside the container
             //
             // Delete all distribution folder
-            sh 'docker run -v $(pwd):/sw alpine sleep 10 && rm -rf /sw/dist-for-cluster/* /sw/dist-for-single-node-service/* /sw/dist-for-agent-reboot/*'
+            sh 'docker run -v $(pwd):/sw alpine sh -c "sleep 10 && rm -rf /sw/dist-for-cluster/* /sw/dist-for-single-node-service/* /sw/dist-for-agent-reboot/* /sw/dist-for-ttl-es/*"'
             deleteDir()
         }
     }

diff --git a/docker/oap/docker-entrypoint.sh b/docker/oap/docker-entrypoint.sh
@@ -259,6 +259,7 @@ core:
     - Month
     # Set a timeout on metrics data. After the timeout has expired, the metrics data will automatically be deleted.
     enableDataKeeperExecutor: \${SW_CORE_ENABLE_DATA_KEEPER_EXECUTOR:true} # Turn it off then automatically metrics data delete will be close.
+    dataKeeperExecutePeriod: \${SW_CORE_DATA_KEEPER_EXECUTE_PERIOD:5} # How often the data keeper executor runs periodically, unit is minute
     recordDataTTL: \${SW_CORE_RECORD_DATA_TTL:90} # Unit is minute
     minuteMetricsDataTTL: \${SW_CORE_MINUTE_METRIC_DATA_TTL:90} # Unit is minute
     hourMetricsDataTTL: \${SW_CORE_HOUR_METRIC_DATA_TTL:36} # Unit is hour

diff --git a/docs/en/setup/backend/ttl.md b/docs/en/setup/backend/ttl.md
@@ -8,6 +8,7 @@ You have following settings for different types.
 ```yaml
     # Set a timeout on metrics data. After the timeout has expired, the metrics data will automatically be deleted.
     enableDataKeeperExecutor: ${SW_CORE_ENABLE_DATA_KEEPER_EXECUTOR:true} # Turn it off then automatically metrics data delete will be close.
+    dataKeeperExecutePeriod: ${SW_CORE_DATA_KEEPER_EXECUTE_PERIOD:5} # How often the data keeper executor runs periodically, unit is minute
     recordDataTTL: ${SW_CORE_RECORD_DATA_TTL:90} # Unit is minute
     minuteMetricsDataTTL: ${SW_CORE_MINUTE_METRIC_DATA_TTL:90} # Unit is minute
     hourMetricsDataTTL: ${SW_CORE_HOUR_METRIC_DATA_TTL:36} # Unit is hour

diff --git a/...ver/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java b/...ver/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java
@@ -45,6 +45,7 @@ public class CoreModuleConfig extends ModuleConfig {
      */
     @Setter private long persistentPeriod = 3;
     @Setter private boolean enableDataKeeperExecutor = true;
+    @Setter private int dataKeeperExecutePeriod = 5;
     @Setter private int recordDataTTL;
     @Setter private int minuteMetricsDataTTL;
     @Setter private int hourMetricsDataTTL;

diff --git a/...r/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java b/...r/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java
@@ -203,7 +203,7 @@ public CoreModuleProvider() {
         PersistenceTimer.INSTANCE.start(getManager(), moduleConfig);
 
         if (moduleConfig.isEnableDataKeeperExecutor()) {
-            DataTTLKeeperTimer.INSTANCE.start(getManager());
+            DataTTLKeeperTimer.INSTANCE.start(getManager(), moduleConfig);
         }
 
         CacheUpdateTimer.INSTANCE.start(getManager());

diff --git a/...e/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/DataTTLKeeperTimer.java b/...e/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/DataTTLKeeperTimer.java
@@ -18,18 +18,26 @@
 
 package org.apache.skywalking.oap.server.core.storage.ttl;
 
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.*;
 import org.apache.skywalking.apm.util.RunnableWithExceptionProtection;
 import org.apache.skywalking.oap.server.core.CoreModule;
+import org.apache.skywalking.oap.server.core.CoreModuleConfig;
 import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
-import org.apache.skywalking.oap.server.core.cluster.*;
-import org.apache.skywalking.oap.server.core.storage.*;
-import org.apache.skywalking.oap.server.core.storage.model.*;
+import org.apache.skywalking.oap.server.core.cluster.ClusterModule;
+import org.apache.skywalking.oap.server.core.cluster.ClusterNodesQuery;
+import org.apache.skywalking.oap.server.core.cluster.RemoteInstance;
+import org.apache.skywalking.oap.server.core.storage.IHistoryDeleteDAO;
+import org.apache.skywalking.oap.server.core.storage.StorageModule;
+import org.apache.skywalking.oap.server.core.storage.model.IModelGetter;
+import org.apache.skywalking.oap.server.core.storage.model.Model;
 import org.apache.skywalking.oap.server.library.module.ModuleManager;
 import org.apache.skywalking.oap.server.library.util.CollectionUtils;
-import org.slf4j.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 
 /**
  * @author peng-yongsheng
@@ -42,13 +50,13 @@ public enum DataTTLKeeperTimer {
     private ModuleManager moduleManager;
     private ClusterNodesQuery clusterNodesQuery;
 
-    public void start(ModuleManager moduleManager) {
+    public void start(ModuleManager moduleManager, CoreModuleConfig moduleConfig) {
         this.moduleManager = moduleManager;
         this.clusterNodesQuery = moduleManager.find(ClusterModule.NAME).provider().getService(ClusterNodesQuery.class);
 
         Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(
-            new RunnableWithExceptionProtection(this::delete,
-                t -> logger.error("Remove data in background failure.", t)), 5, 5, TimeUnit.MINUTES);
+            new RunnableWithExceptionProtection(this::delete, t -> logger.error("Remove data in background failure.", t)),
+            moduleConfig.getDataKeeperExecutePeriod(), moduleConfig.getDataKeeperExecutePeriod(), TimeUnit.MINUTES);
     }
 
     private void delete() {

diff --git a/oap-server/server-starter/src/main/assembly/application.yml b/oap-server/server-starter/src/main/assembly/application.yml
@@ -61,6 +61,7 @@ core:
       - Month
     # Set a timeout on metrics data. After the timeout has expired, the metrics data will automatically be deleted.
     enableDataKeeperExecutor: ${SW_CORE_ENABLE_DATA_KEEPER_EXECUTOR:true} # Turn it off then automatically metrics data delete will be close.
+    dataKeeperExecutePeriod: ${SW_CORE_DATA_KEEPER_EXECUTE_PERIOD:5} # How often the data keeper executor runs periodically, unit is minute
     recordDataTTL: ${SW_CORE_RECORD_DATA_TTL:90} # Unit is minute
     minuteMetricsDataTTL: ${SW_CORE_MINUTE_METRIC_DATA_TTL:90} # Unit is minute
     hourMetricsDataTTL: ${SW_CORE_HOUR_METRIC_DATA_TTL:36} # Unit is hour

diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml
@@ -60,6 +60,7 @@ core:
       - Month
     # Set a timeout on metrics data. After the timeout has expired, the metrics data will automatically be deleted.
     enableDataKeeperExecutor: ${SW_CORE_ENABLE_DATA_KEEPER_EXECUTOR:true} # Turn it off then automatically metrics data delete will be close.
+    dataKeeperExecutePeriod: ${SW_CORE_DATA_KEEPER_EXECUTE_PERIOD:5} # How often the data keeper executor runs periodically, unit is minute
     recordDataTTL: ${SW_CORE_RECORD_DATA_TTL:90} # Unit is minute
     minuteMetricsDataTTL: ${SW_CORE_MINUTE_METRIC_DATA_TTL:90} # Unit is minute
     hourMetricsDataTTL: ${SW_CORE_HOUR_METRIC_DATA_TTL:36} # Unit is hour

diff --git a/test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/AbstractQuery.java b/test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/AbstractQuery.java
@@ -28,6 +28,8 @@
 public abstract class AbstractQuery<T extends AbstractQuery<?>> {
     private static final DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HHmmss");
     private static final DateTimeFormatter MINUTE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HHmm");
+    private static final DateTimeFormatter DAY_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd");
+    private static final DateTimeFormatter MONTH_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM");
 
     private String start;
     private String end;
@@ -37,9 +39,19 @@ public String start() {
         if (start != null) {
             return start;
         }
-        return "SECOND".equals(step())
-            ? LocalDateTime.now(ZoneOffset.UTC).minusMinutes(15).format(TIME_FORMATTER)
-            : LocalDateTime.now(ZoneOffset.UTC).minusMinutes(15).format(MINUTE_TIME_FORMATTER);
+        if ("SECOND".equals(step())) {
+            return LocalDateTime.now(ZoneOffset.UTC).minusMinutes(15).format(TIME_FORMATTER);
+        }
+        if ("MINUTE".equals(step())) {
+            return LocalDateTime.now(ZoneOffset.UTC).minusMinutes(15).format(MINUTE_TIME_FORMATTER);
+        }
+        if ("DAY".equals(step())) {
+            return LocalDateTime.now(ZoneOffset.UTC).minusMinutes(15).format(DAY_TIME_FORMATTER);
+        }
+        if ("MONTH".equals(step())) {
+            return LocalDateTime.now(ZoneOffset.UTC).minusMinutes(15).format(MONTH_TIME_FORMATTER);
+        }
+        return null;
     }
 
     public T start(String start) {
@@ -52,6 +64,10 @@ public T start(LocalDateTime start) {
             this.start = start.format(MINUTE_TIME_FORMATTER);
         } else if ("SECOND".equals(step())) {
             this.start = start.format(TIME_FORMATTER);
+        } else if ("DAY".equals(step())) {
+            this.start = start.format(DAY_TIME_FORMATTER);
+        } else if ("MONTH".equals(step())) {
+            this.start = start.format(MONTH_TIME_FORMATTER);
         }
         return (T) this;
     }
@@ -60,9 +76,19 @@ public String end() {
         if (end != null) {
             return end;
         }
-        return "SECOND".equals(step())
-            ? LocalDateTime.now(ZoneOffset.UTC).format(TIME_FORMATTER)
-            : LocalDateTime.now(ZoneOffset.UTC).format(MINUTE_TIME_FORMATTER);
+        if ("SECOND".equals(step())) {
+            return LocalDateTime.now(ZoneOffset.UTC).format(TIME_FORMATTER);
+        }
+        if ("MINUTE".equals(step())) {
+            return LocalDateTime.now(ZoneOffset.UTC).format(MINUTE_TIME_FORMATTER);
+        }
+        if ("DAY".equals(step())) {
+            return LocalDateTime.now(ZoneOffset.UTC).format(DAY_TIME_FORMATTER);
+        }
+        if ("MONTH".equals(step())) {
+            return LocalDateTime.now(ZoneOffset.UTC).format(MONTH_TIME_FORMATTER);
+        }
+        return null;
     }
 
     public AbstractQuery end(String end) {
@@ -75,6 +101,10 @@ public T end(LocalDateTime end) {
             this.end = end.format(MINUTE_TIME_FORMATTER);
         } else if ("SECOND".equals(step())) {
             this.end = end.format(TIME_FORMATTER);
+        } else if ("DAY".equals(step())) {
+            this.end = end.format(DAY_TIME_FORMATTER);
+        } else if ("MONTH".equals(step())) {
+            this.end = end.format(MONTH_TIME_FORMATTER);
         }
         return (T) this;
     }
@@ -88,6 +118,16 @@ public T step(String step) {
         return (T) this;
     }
 
+    public T stepByMonth() {
+        this.step = "MONTH";
+        return (T) this;
+    }
+
+    public T stepByDay() {
+        this.step = "DAY";
+        return (T) this;
+    }
+
     public T stepByMinute() {
         this.step = "MINUTE";
         return (T) this;

diff --git a/test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/metrics/AllOfMetricsMatcher.java b/test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/metrics/AllOfMetricsMatcher.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.skywalking.e2e.metrics;
+
+import org.apache.skywalking.e2e.verification.AbstractMatcher;
+import org.assertj.core.api.Condition;
+
+import java.util.function.Predicate;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * @author kezhenxu94
+ */
+public class AllOfMetricsMatcher extends AbstractMatcher<Metrics> {
+    private MetricsValueMatcher value;
+
+    @Override
+    public void verify(Metrics metrics) {
+        assertThat(metrics.getValues()).isNotEmpty();
+        assertThat(metrics.getValues()).allMatch(value -> {
+            try {
+                AllOfMetricsMatcher.this.getValue().verify(value);
+                return true;
+            } catch (Throwable t) {
+                return false;
+            }
+        });
+    }
+
+    public MetricsValueMatcher getValue() {
+        return value;
+    }
+
+    public void setValue(MetricsValueMatcher value) {
+        this.value = value;
+    }
+
+    @Override
+    public String toString() {
+        return "AllOfMetricsMatcher{" +
+            "value=" + value +
+            '}';
+    }
+}
diff --git a/test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/verification/AbstractMatcher.java b/test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/verification/AbstractMatcher.java
@@ -28,6 +28,7 @@
  */
 public abstract class AbstractMatcher<T> {
     private static final Pattern NE_MATCHER = Pattern.compile("ne\\s+(?<val>.+)");
+    private static final Pattern EQ_MATCHER = Pattern.compile("eq\\s+(?<val>.+)");
     private static final Pattern GT_MATCHER = Pattern.compile("gt\\s+(?<val>.+)");
     private static final Pattern GE_MATCHER = Pattern.compile("ge\\s+(?<val>.+)");
     private static final Pattern NN_MATCHER = Pattern.compile("^not null$");
@@ -65,6 +66,15 @@ protected void doVerify(String expected, String actual) {
             return;
         }
 
+        matcher = EQ_MATCHER.matcher(expected);
+        if (matcher.find()) {
+            String val = matcher.group("val");
+
+            assertThat(val).isNotBlank();
+            assertThat(Double.parseDouble(actual)).isEqualTo(Double.parseDouble(val));
+            return;
+        }
+
         assertThat(actual).isEqualTo(expected);
     }