[WIP] First stab at adding PCT ITs for the Docker image

Jenkinsfile

@@ -53,3 +53,30 @@ for (int i = 0; i < platforms.size(); ++i) {
 
 /* Execute our platforms in parallel */
 parallel(branches)
+
+node('docker') {
+    checkout scm
+
+    stage('Build Docker Image') {
+        sh 'make docker'
+    }
+
+    stage('Download Jenkins 2.164.1') {
+        sh '''curl -sL http://mirrors.jenkins.io/war-stable/2.164.1/jenkins.war --output jenkins.war
+        echo "65543f5632ee54344f3351b34b305702df12393b3196a95c3771ddb3819b220b jenkins.war" | sha256sum --check
+        '''
+    }
+
+    stage("Run known successful case(s)") {
+        sh '''docker run --rm \
+                         -v $(pwd)/jenkins.war:/pct/jenkins.war:ro \
+                         -v $(pwd)/out:/pct/out -e JDK_VERSION=11 \
+                         -e ARTIFACT_ID=buildtriggerbadge -e VERSION=buildtriggerbadge-2.10 \
+                         jenkins/pct
+        '''
+        archiveArtifacts artifacts: "out/**"
+
+        sh 'cat out/pct-report.html | grep "Tests : Success"'
+
+    }
+}

README.md

@@ -39,21 +39,6 @@ The command below will run PCT for a branch in a custom repository.
 docker run --rm -v maven-repo:/root/.m2 -v $(pwd)/out:/pct/out -e CHECKOUT_SRC=https://github.com/oleg-nenashev/job-restrictions-plugin.git -e VERSION=JENKINS-26374 jenkins/pct
 ```
 
-#### Full PCT runs using a war file as input with Docker
-
-The PCT cli supports to pass a war file containing plugins (generated with Custom war Packager for example) as input, in this case the version of
-the plugins is infered from the war file contents.
-
-In such scenarios is tipical to want to run the PCT for all plugins contained in the war file, to avoid having to spawn a new docker container for each plugin
-you can use the env variables `DO_NOT_OVERRIDE_PCT_CHECKOUT=true` and `FAIL_ON_ERROR=false` to let the PCT CLI (instead of the `run-pct` script) checkout the proper
-versions of the plugins and make sure the PCT does not stop before testing all plugins.
-
-By using those env variables you can pass a comma separated list of plugins ids using the `ARTIFACT_ID` env variable
-
-```shell
-docker run -ti --rm -v maven-repo:/root/.m2 -v $(pwd)/out:/pct/out -v my/jenkins.war:/pct/jenkins.war:ro -e ARTIFACT_ID=ssh-slaves,credentials -e DO_NOT_OVERRIDE_PCT_CHECKOUT=true -e FAIL_ON_ERROR=false jenkins/pct
-```
-
 #### Configuration
 
 Environment variables:

plugins-compat-tester/src/main/java/org/jenkins/tools/test/PluginCompatTester.java

@@ -494,7 +494,7 @@ private TestExecutionResult testPluginAgainst(MavenCoordinates coreCoordinates,
             // -Dmaven-surefire-plugin.version=2.15 -Dmaven.test.dependency.excludes=org.jenkins-ci.main:jenkins-war -Dmaven.test.additionalClasspath=/…/org/jenkins-ci/main/jenkins-war/1.580.1/jenkins-war-1.580.1.war clean test
             // (2.15+ required for ${maven.test.dependency.excludes} and ${maven.test.additionalClasspath} to be honored from CLI)
             // but it does not work; there are lots of linkage errors as some things are expected to be in the test classpath which are not.
-            // Much simpler to do use the parent POM to set up the test classpath. 
+            // Much simpler to do use the parent POM to set up the test classpath.
             MavenPom pom = new MavenPom(pluginCheckoutDir);
             try {
                 addSplitPluginDependencies(plugin.name, mconfig, pluginCheckoutDir, pom, otherPlugins, pluginGroupIds, coreCoordinates.version, overridenPlugins);
@@ -618,8 +618,9 @@ private UpdateSite.Data scanWAR(File war, Map<String, String> pluginGroupIds, St
                     // We do not really care about the value
                     top.put("core", new JSONObject().accumulate("name", "core").accumulate("version", m.group(1)).accumulate("url", "https://foobar"));
                 }
-
-                m = Pattern.compile(pluginRegExp).matcher(name);
+
+                // TODO: should it also scan detached plugins info?
+                m = Pattern.compile("WEB-INF/(?:optional-)?plugins/([^/.]+)[.][hj]pi").matcher(name);
                 if (m.matches()) {
                     JSONObject plugin = new JSONObject().accumulate("url", "");
                     InputStream is = jf.getInputStream(entry);

src/main/docker/run-pct.sh

@@ -26,16 +26,6 @@ fi
 
 CUSTOM_MAVEN_SETTINGS=${M2_SETTINGS_FILE:-"/pct/m2-settings.xml"}
 
-# Set failOnError by default unless env FAIL_ON_ERROR is false
-FAIL_ON_ERROR_ARG=""
-if [ -n "${FAIL_ON_ERROR}" ]; then
-    if [ "${FAIL_ON_ERROR}" != "false" ] ; then
-        FAIL_ON_ERROR_ARG="-failOnError"
-    fi
-else
-    FAIL_ON_ERROR_ARG="-failOnError"
-fi
-
 if [ -f "${CUSTOM_MAVEN_SETTINGS}" ] ; then
     echo "Using a custom Maven settings file"
     MVN_SETTINGS_FILE="${CUSTOM_MAVEN_SETTINGS}"
@@ -115,7 +105,7 @@ if [[ "$DEBUG" ]] ; then
 fi
 
 LOCAL_CHECKOUT_ARG=""
-if [ "${SHOULD_CHECKOUT}" -eq 1 ] && [ -z "${DO_NOT_OVERRIDE_PCT_CHECKOUT}" ] ; then
+if [ "${SHOULD_CHECKOUT}" -eq 1 ] ; then
   ###
   # Checkout sources
   ###
@@ -165,7 +155,7 @@ echo java ${JAVA_OPTS} ${extra_java_opts[@]} \
   -reportFile ${PCT_OUTPUT_DIR}/pct-report.xml \
   -workDirectory "${PCT_TMP}/work" ${WAR_PATH_OPT} \
   -skipTestCache true \
-  ${FAIL_ON_ERROR_ARG}\
+  -failOnError \
   ${LOCAL_CHECKOUT_ARG} \
   -includePlugins "${ARTIFACT_ID}" \
   -mvn "/usr/bin/mvn" \