Merge branch 'master' into nicholas.hulston/s3-span-pointers

.circleci/config.continue.yml.j2

@@ -953,13 +953,23 @@ jobs:
           no_output_timeout: 5m
           command: |
             cd system-tests
-            DD_SITE=datadoghq.com DD_API_KEY=$SYSTEM_TESTS_E2E_DD_API_KEY DD_APPLICATION_KEY=$SYSTEM_TESTS_E2E_DD_APP_KEY ./run.sh DEBUGGER_SCENARIOS
+            export DD_API_KEY=$SYSTEM_TESTS_E2E_DD_API_KEY 
+            ./run.sh DEBUGGER_SCENARIOS
+
+      - run:
+          name: Collect log files
+          no_output_timeout: 5m
+          command: |
+            mkdir -p logs_debugger
+            for dir in system-tests/logs*/; do
+                cp -r "$dir" logs_debugger
+            done
 
       - store_test_results:
-          path: system-tests/logs_debugger*
+          path: logs_debugger
 
       - store_artifacts:
-          path: system-tests/logs_debugger*
+          path: logs_debugger
 
   parametric-tests:
     machine:

components/json/gradle.lockfile

@@ -0,0 +1,118 @@
+# This is a Gradle generated file for dependency locking.
+# Manual edits can break the build and are not advised.
+# This file is expected to be part of source control.
+ch.qos.logback:logback-classic:1.2.3=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+ch.qos.logback:logback-core:1.2.3=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.beust:jcommander:1.78=jmhRuntimeClasspath,testRuntimeClasspath
+com.github.javaparser:javaparser-core:3.25.1=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.github.spotbugs:spotbugs-annotations:4.2.0=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+com.github.spotbugs:spotbugs-annotations:4.7.3=spotbugs
+com.github.spotbugs:spotbugs:4.7.3=spotbugs
+com.google.code.findbugs:jsr305:3.0.2=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,spotbugs,testCompileClasspath,testRuntimeClasspath
+com.google.code.gson:gson:2.9.1=spotbugs
+com.thoughtworks.qdox:qdox:1.12.1=jmhRuntimeClasspath,testRuntimeClasspath
+commons-codec:commons-codec:1.15=spotbugs
+de.thetaphi:forbiddenapis:3.1=compileClasspath,jmhCompileClasspath
+info.picocli:picocli:4.6.3=jmhRuntimeClasspath,testRuntimeClasspath
+jaxen:jaxen:1.2.0=spotbugs
+jline:jline:2.14.6=jmhRuntimeClasspath,testRuntimeClasspath
+junit:junit:4.13.2=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+net.jcip:jcip-annotations:1.0=compileClasspath,jmhCompileClasspath,jmhRuntimeClasspath,spotbugs,testCompileClasspath,testRuntimeClasspath
+net.sf.jopt-simple:jopt-simple:5.0.4=jmh,jmhCompileClasspath,jmhRuntimeClasspath
+net.sf.saxon:Saxon-HE:11.4=spotbugs
+org.apache.ant:ant-antlr:1.10.12=jmhRuntimeClasspath,testRuntimeClasspath
+org.apache.ant:ant-antlr:1.9.15=codenarc
+org.apache.ant:ant-junit:1.10.12=jmhRuntimeClasspath,testRuntimeClasspath
+org.apache.ant:ant-junit:1.9.15=codenarc
+org.apache.ant:ant-launcher:1.10.12=jmhRuntimeClasspath,testRuntimeClasspath
+org.apache.ant:ant:1.10.12=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.apache.bcel:bcel:6.5.0=spotbugs
+org.apache.commons:commons-lang3:3.12.0=pitest,spotbugs
+org.apache.commons:commons-math3:3.2=jmh,jmhCompileClasspath,jmhRuntimeClasspath
+org.apache.commons:commons-text:1.10.0=pitest,spotbugs
+org.apache.httpcomponents.client5:httpclient5:5.1.3=spotbugs
+org.apache.httpcomponents.core5:httpcore5-h2:5.1.3=spotbugs
+org.apache.httpcomponents.core5:httpcore5:5.1.3=spotbugs
+org.apache.logging.log4j:log4j-api:2.19.0=spotbugs
+org.apache.logging.log4j:log4j-core:2.19.0=spotbugs
+org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath
+org.codehaus.groovy:groovy-all:3.0.17=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-ant:2.5.14=codenarc
+org.codehaus.groovy:groovy-ant:3.0.17=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-astbuilder:3.0.17=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-cli-picocli:3.0.17=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-console:3.0.17=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-datetime:3.0.17=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-docgenerator:3.0.17=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-groovydoc:2.5.14=codenarc
+org.codehaus.groovy:groovy-groovydoc:3.0.17=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-groovysh:3.0.17=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-jmx:3.0.17=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-json:2.5.14=codenarc
+org.codehaus.groovy:groovy-json:3.0.17=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-jsr223:3.0.17=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-macro:3.0.17=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-nio:3.0.17=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-servlet:3.0.17=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-sql:3.0.17=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-swing:3.0.17=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-templates:2.5.14=codenarc
+org.codehaus.groovy:groovy-templates:3.0.17=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-test-junit5:3.0.17=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-test:3.0.17=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-testng:3.0.17=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy-xml:2.5.14=codenarc
+org.codehaus.groovy:groovy-xml:3.0.17=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.codehaus.groovy:groovy:2.5.14=codenarc
+org.codehaus.groovy:groovy:3.0.17=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.codenarc:CodeNarc:2.2.0=codenarc
+org.dom4j:dom4j:2.1.3=spotbugs
+org.gmetrics:GMetrics:1.1=codenarc
+org.hamcrest:hamcrest-core:1.3=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.hamcrest:hamcrest:2.2=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.jacoco:org.jacoco.agent:0.8.5=jacocoAgent,jacocoAnt
+org.jacoco:org.jacoco.ant:0.8.5=jacocoAnt
+org.jacoco:org.jacoco.core:0.8.5=jacocoAnt
+org.jacoco:org.jacoco.report:0.8.5=jacocoAnt
+org.junit.jupiter:junit-jupiter-api:5.9.2=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.junit.jupiter:junit-jupiter-engine:5.9.2=jmhRuntimeClasspath,testRuntimeClasspath
+org.junit.jupiter:junit-jupiter-params:5.9.2=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.junit.jupiter:junit-jupiter:5.9.2=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.junit.platform:junit-platform-commons:1.9.2=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.junit.platform:junit-platform-engine:1.9.2=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.junit.platform:junit-platform-launcher:1.9.2=jmhRuntimeClasspath,testRuntimeClasspath
+org.junit:junit-bom:5.9.1=spotbugs
+org.junit:junit-bom:5.9.2=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.objenesis:objenesis:3.3=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.openjdk.jmh:jmh-core:1.36=jmh,jmhCompileClasspath,jmhRuntimeClasspath
+org.openjdk.jmh:jmh-generator-asm:1.36=jmh,jmhCompileClasspath,jmhRuntimeClasspath
+org.openjdk.jmh:jmh-generator-bytecode:1.36=jmh,jmhCompileClasspath,jmhRuntimeClasspath
+org.openjdk.jmh:jmh-generator-reflection:1.36=jmh,jmhCompileClasspath,jmhRuntimeClasspath
+org.opentest4j:opentest4j:1.2.0=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.ow2.asm:asm-analysis:7.2=jacocoAnt
+org.ow2.asm:asm-analysis:9.4=spotbugs
+org.ow2.asm:asm-commons:7.2=jacocoAnt
+org.ow2.asm:asm-commons:9.4=spotbugs
+org.ow2.asm:asm-tree:7.2=jacocoAnt
+org.ow2.asm:asm-tree:9.4=spotbugs
+org.ow2.asm:asm-util:9.4=spotbugs
+org.ow2.asm:asm:7.2=jacocoAnt
+org.ow2.asm:asm:9.0=jmh,jmhCompileClasspath,jmhRuntimeClasspath
+org.ow2.asm:asm:9.4=spotbugs
+org.pitest:pitest-command-line:1.9.11=pitest
+org.pitest:pitest-entry:1.9.11=pitest
+org.pitest:pitest:1.9.11=pitest
+org.slf4j:jcl-over-slf4j:1.7.30=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.slf4j:jul-to-slf4j:1.7.30=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.slf4j:log4j-over-slf4j:1.7.30=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.slf4j:slf4j-api:1.7.30=testCompileClasspath
+org.slf4j:slf4j-api:1.7.32=jmhRuntimeClasspath,testRuntimeClasspath
+org.slf4j:slf4j-api:2.0.0=spotbugs,spotbugsSlf4j
+org.slf4j:slf4j-simple:2.0.0=spotbugsSlf4j
+org.spockframework:spock-core:2.2-groovy-3.0=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.spockframework:spock-junit4:2.2-groovy-3.0=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.testng:testng:7.5=jmhRuntimeClasspath,testRuntimeClasspath
+org.webjars:jquery:3.5.1=jmhRuntimeClasspath,testRuntimeClasspath
+org.xmlresolver:xmlresolver:4.4.3=spotbugs
+xml-apis:xml-apis:1.4.01=spotbugs
+empty=annotationProcessor,jmhAnnotationProcessor,runtimeClasspath,spotbugsPlugins,testAnnotationProcessor

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationApiImpl.java

@@ -15,6 +15,7 @@
 import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector;
 import datadog.trace.api.civisibility.telemetry.tag.CoverageEnabled;
 import datadog.trace.api.civisibility.telemetry.tag.EarlyFlakeDetectionEnabled;
+import datadog.trace.api.civisibility.telemetry.tag.FlakyTestRetriesEnabled;
 import datadog.trace.api.civisibility.telemetry.tag.ItrEnabled;
 import datadog.trace.api.civisibility.telemetry.tag.ItrSkipEnabled;
 import datadog.trace.api.civisibility.telemetry.tag.RequireGit;
@@ -132,6 +133,7 @@ public CiVisibilitySettings getSettings(TracerEnvironment tracerEnvironment) thr
         settings.getEarlyFlakeDetectionSettings().isEnabled()
             ? EarlyFlakeDetectionEnabled.TRUE
             : null,
+        settings.isFlakyTestRetriesEnabled() ? FlakyTestRetriesEnabled.TRUE : null,
         settings.isGitUploadRequired() ? RequireGit.TRUE : null);
 
     return settings;
@@ -166,7 +168,8 @@ public SkippableTests getSkippableTests(TracerEnvironment tracerEnvironment) thr
     for (DataDto<TestIdentifierJson> dataDto : response.data) {
       TestIdentifierJson testIdentifierJson = dataDto.getAttributes();
       Configurations conf = testIdentifierJson.getConfigurations();
-      String moduleName = (conf != null ? conf : requestConf).getTestBundle();
+      String moduleName =
+          (conf != null && conf.getTestBundle() != null ? conf : requestConf).getTestBundle();
       testIdentifiersByModule
           .computeIfAbsent(moduleName, k -> new HashMap<>())
           .put(testIdentifierJson.toTestIdentifier(), testIdentifierJson.toTestMetadata());
@@ -185,6 +188,14 @@ public SkippableTests getSkippableTests(TracerEnvironment tracerEnvironment) thr
   @Override
   public Map<String, Collection<TestIdentifier>> getFlakyTestsByModule(
       TracerEnvironment tracerEnvironment) throws IOException {
+    OkHttpUtils.CustomListener telemetryListener =
+        new TelemetryListener.Builder(metricCollector)
+            .requestCount(CiVisibilityCountMetric.FLAKY_TESTS_REQUEST)
+            .requestErrors(CiVisibilityCountMetric.FLAKY_TESTS_REQUEST_ERRORS)
+            .requestDuration(CiVisibilityDistributionMetric.FLAKY_TESTS_REQUEST_MS)
+            .responseBytes(CiVisibilityDistributionMetric.FLAKY_TESTS_RESPONSE_BYTES)
+            .build();
+
     String uuid = uuidGenerator.get();
     EnvelopeDto<TracerEnvironment> request =
         new EnvelopeDto<>(
@@ -196,20 +207,27 @@ public Map<String, Collection<TestIdentifier>> getFlakyTestsByModule(
             FLAKY_TESTS_URI,
             requestBody,
             is -> testIdentifiersResponseAdapter.fromJson(Okio.buffer(Okio.source(is))).data,
-            null,
+            telemetryListener,
             false);
 
     LOGGER.debug("Received {} flaky tests in total", response.size());
 
+    Configurations requestConf = tracerEnvironment.getConfigurations();
+
+    int flakyTestsCount = 0;
     Map<String, Collection<TestIdentifier>> testIdentifiers = new HashMap<>();
     for (DataDto<TestIdentifierJson> dataDto : response) {
       TestIdentifierJson testIdentifierJson = dataDto.getAttributes();
-      Configurations configurations = testIdentifierJson.getConfigurations();
-      String moduleName = configurations.getTestBundle();
+      Configurations conf = testIdentifierJson.getConfigurations();
+      String moduleName =
+          (conf != null && conf.getTestBundle() != null ? conf : requestConf).getTestBundle();
       testIdentifiers
           .computeIfAbsent(moduleName, k -> new HashSet<>())
           .add(testIdentifierJson.toTestIdentifier());
+      flakyTestsCount++;
     }
+
+    metricCollector.add(CiVisibilityDistributionMetric.FLAKY_TESTS_RESPONSE_TESTS, flakyTestsCount);
     return testIdentifiers;
   }
 

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettings.java

@@ -114,7 +114,6 @@ public Collection<TestIdentifier> getKnownTests() {
 
   @Nullable
   public Collection<TestIdentifier> getFlakyTests() {
-    // backend does not store module info for flaky tests yet
     return flakyTests;
   }
 

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettingsFactoryImpl.java

@@ -1,6 +1,7 @@
 package datadog.trace.civisibility.config;
 
 import datadog.trace.api.Config;
+import datadog.trace.api.civisibility.CIConstants;
 import datadog.trace.api.civisibility.CiVisibilityWellKnownTags;
 import datadog.trace.api.civisibility.config.TestIdentifier;
 import datadog.trace.api.civisibility.config.TestMetadata;
@@ -139,11 +140,17 @@ private TracerEnvironment buildTracerEnvironment(
 
     Map<String, Collection<TestIdentifier>> flakyTestsByModule =
         flakyTestRetriesEnabled && config.isCiVisibilityFlakyRetryOnlyKnownFlakes()
+                || CIConstants.FAIL_FAST_TEST_ORDER.equalsIgnoreCase(
+                    config.getCiVisibilityTestOrder())
             ? getFlakyTestsByModule(tracerEnvironment)
             : null;
 
     Map<String, Collection<TestIdentifier>> knownTestsByModule =
-        earlyFlakeDetectionEnabled ? getKnownTestsByModule(tracerEnvironment) : null;
+        earlyFlakeDetectionEnabled
+                || CIConstants.FAIL_FAST_TEST_ORDER.equalsIgnoreCase(
+                    config.getCiVisibilityTestOrder())
+            ? getKnownTestsByModule(tracerEnvironment)
+            : null;
 
     Set<String> moduleNames = new HashSet<>(Collections.singleton(DEFAULT_SETTINGS));
     moduleNames.addAll(skippableTestIdentifiers.keySet());

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/TestIdentifierSerializer.java

@@ -13,8 +13,10 @@ public static void serialize(Serializer serializer, TestIdentifier testIdentifie
   }
 
   public static TestIdentifier deserialize(ByteBuffer buffer) {
+    String suiteName = Serializer.readString(buffer);
     return new TestIdentifier(
-        Serializer.readString(buffer),
+        // suite name repeats a lot; interning it to save memory
+        suiteName != null ? suiteName.intern() : null,
         Serializer.readString(buffer),
         Serializer.readString(buffer));
   }

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/AbstractTestSession.java

@@ -5,11 +5,14 @@
 import datadog.trace.api.Config;
 import datadog.trace.api.DDTraceId;
 import datadog.trace.api.IdGenerationStrategy;
+import datadog.trace.api.civisibility.CIConstants;
 import datadog.trace.api.civisibility.telemetry.CiVisibilityCountMetric;
 import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector;
 import datadog.trace.api.civisibility.telemetry.TagValue;
+import datadog.trace.api.civisibility.telemetry.tag.AgentlessLogSubmissionEnabled;
 import datadog.trace.api.civisibility.telemetry.tag.AutoInjected;
 import datadog.trace.api.civisibility.telemetry.tag.EventType;
+import datadog.trace.api.civisibility.telemetry.tag.FailFastTestOrderEnabled;
 import datadog.trace.api.civisibility.telemetry.tag.HasCodeowner;
 import datadog.trace.api.civisibility.telemetry.tag.IsHeadless;
 import datadog.trace.api.civisibility.telemetry.tag.IsUnsupportedCI;
@@ -109,7 +112,11 @@ public AbstractTestSession(
         CiVisibilityCountMetric.TEST_SESSION,
         1,
         ciProvider,
-        config.isCiVisibilityAutoInjected() ? AutoInjected.TRUE : null);
+        config.isCiVisibilityAutoInjected() ? AutoInjected.TRUE : null,
+        config.isAgentlessLogSubmissionEnabled() ? AgentlessLogSubmissionEnabled.TRUE : null,
+        CIConstants.FAIL_FAST_TEST_ORDER.equalsIgnoreCase(config.getCiVisibilityTestOrder())
+            ? FailFastTestOrderEnabled.TRUE
+            : null);
 
     if (instrumentationType == InstrumentationType.MANUAL_API) {
       metricCollector.add(CiVisibilityCountMetric.MANUAL_API_EVENTS, 1, EventType.SESSION);

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestFrameworkModule.java

@@ -25,6 +25,8 @@ TestSuiteImpl testSuiteStart(
    */
   boolean isNew(TestIdentifier test);
 
+  boolean isFlaky(TestIdentifier test);
+
   /**
    * Checks if a given test should be skipped with Intelligent Test Runner or not
    *

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/ProxyTestModule.java

@@ -87,6 +87,11 @@ public boolean isNew(TestIdentifier test) {
     return executionStrategy.isNew(test);
   }
 
+  @Override
+  public boolean isFlaky(TestIdentifier test) {
+    return executionStrategy.isFlaky(test);
+  }
+
   @Override
   public boolean shouldBeSkipped(TestIdentifier test) {
     return executionStrategy.shouldBeSkipped(test);

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestModule.java

@@ -72,6 +72,11 @@ public boolean isNew(TestIdentifier test) {
     return executionStrategy.isNew(test);
   }
 
+  @Override
+  public boolean isFlaky(TestIdentifier test) {
+    return executionStrategy.isFlaky(test);
+  }
+
   @Override
   public boolean shouldBeSkipped(TestIdentifier test) {
     return executionStrategy.shouldBeSkipped(test);

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java

@@ -272,6 +272,16 @@ public TestRetryPolicy retryPolicy(TestIdentifier test) {
     return testModule.retryPolicy(test);
   }
 
+  @Override
+  public boolean isNew(TestIdentifier test) {
+    return testModule.isNew(test);
+  }
+
+  @Override
+  public boolean isFlaky(TestIdentifier test) {
+    return testModule.isFlaky(test);
+  }
+
   @Override
   public void close() {
     testModule.end(null);

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java

@@ -43,6 +43,11 @@ public boolean isNew(TestIdentifier test) {
     return knownTests != null && !knownTests.contains(test.withoutParameters());
   }
 
+  public boolean isFlaky(TestIdentifier test) {
+    Collection<TestIdentifier> flakyTests = executionSettings.getFlakyTests();
+    return flakyTests != null && flakyTests.contains(test.withoutParameters());
+  }
+
   public boolean shouldBeSkipped(TestIdentifier test) {
     if (test == null) {
       return false;