#917 - Add test-harness module + change json schema test API to return list of errors

docs/PLUGINS.md

@@ -212,10 +212,9 @@ Add the Configuration as Code plugin as a test dependency in your pom.xml:
     <scope>test</scope>
 </dependency>
 <dependency>
-    <groupId>io.jenkins</groupId>
-    <artifactId>configuration-as-code</artifactId>
+    <groupId>io.jenkins.configuration-as-code</groupId>
+    <artifactId>test-harness</artifactId>
     <version>${configuration-as-code.version}</version>
-    <classifier>tests</classifier>
     <scope>test</scope>
 </dependency>
 ```
@@ -337,6 +336,6 @@ Add a test for the YAML file
 ```java
     @Test
     public void validSchemaShouldSucceed() throws Exception {
-        assertTrue(validateSchema(convertYamlFileToJson(this, "validJenkinsConfigurator.yml")));
+        assertThat(validateSchema(convertYamlFileToJson(this, "validSchemaConfig.yml")), empty());
     }
 ```

integrations/pom.xml

@@ -14,6 +14,7 @@
   <properties>
     <jenkins.version>2.176.3</jenkins.version> <!-- Remove when parent base is updated. -->
     <maven.deploy.skip>true</maven.deploy.skip> <!-- no need to be deployed during release, this is a test-only module -->
+    <jackson.version>2.10.1</jackson.version>
   </properties>
 
   <dependencies>
@@ -28,18 +29,10 @@
         </exclusion>
       </exclusions>
     </dependency>
-
     <dependency>
-      <groupId>io.jenkins</groupId>
-      <artifactId>configuration-as-code</artifactId>
+      <groupId>io.jenkins.configuration-as-code</groupId>
+      <artifactId>test-harness</artifactId>
       <version>${project.version}</version>
-      <type>test-jar</type>
-      <exclusions>
-        <exclusion>
-          <groupId>org.json</groupId>
-          <artifactId>json</artifactId>
-        </exclusion>
-      </exclusions>
     </dependency>
 
     <dependency>
@@ -223,7 +216,7 @@
       <version>2.3.0</version>
       <scope>test</scope>
     </dependency>
-  
+
     <dependency>
       <groupId>org.jenkins-ci.plugins</groupId>
       <artifactId>mesos</artifactId>
@@ -334,7 +327,7 @@
     <dependency>
       <groupId>org.jenkins-ci.plugins</groupId>
       <artifactId>jackson2-api</artifactId>
-      <version>2.9.10</version>
+      <version>${jackson.version}</version>
     </dependency>
 
     <dependency>
@@ -484,27 +477,27 @@
       <dependency>
         <groupId>com.fasterxml.jackson.core</groupId>
         <artifactId>jackson-core</artifactId>
-        <version>2.9.10</version>
+        <version>${jackson.version}</version>
       </dependency>
       <dependency>
         <groupId>com.fasterxml.jackson.core</groupId>
         <artifactId>jackson-databind</artifactId>
-        <version>2.9.10</version>
+        <version>${jackson.version}</version>
       </dependency>
       <dependency>
         <groupId>com.fasterxml.jackson.jaxrs</groupId>
         <artifactId>jackson-jaxrs-json-provider</artifactId>
-        <version>2.9.7</version>
+        <version>${jackson.version}</version>
       </dependency>
       <dependency>
         <groupId>com.fasterxml.jackson.jaxrs</groupId>
         <artifactId>jaxrs-json-provider</artifactId>
-        <version>2.9.7</version>
+        <version>${jackson.version}</version>
       </dependency>
       <dependency>
         <groupId>com.fasterxml.jackson.module</groupId>
         <artifactId>jackson-module-jaxb-annotations</artifactId>
-        <version>2.9.9</version>
+        <version>${jackson.version}</version>
       </dependency>
       <dependency>
         <groupId>com.google.errorprone</groupId>

plugin/pom.xml

@@ -14,7 +14,7 @@
   <packaging>hpi</packaging>
 
   <name>Configuration as Code Plugin</name>
-  <description>Manage Jenkins master configuration as code</description>	
+  <description>Manage Jenkins master configuration as code</description>
   <url>https://github.com/jenkinsci/configuration-as-code-plugin</url>
 
   <properties>
@@ -46,29 +46,11 @@
   </developers>
 
   <dependencies>
-    <dependency>
-      <groupId>org.everit.json</groupId>
-      <artifactId>org.everit.json.schema</artifactId>
-      <version>1.5.1</version>
-      <scope>test</scope>
-      <exclusions>
-        <exclusion>
-          <groupId>commons-beanutils</groupId>
-          <artifactId>commons-beanutils</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
     <dependency>
       <groupId>org.json</groupId>
       <artifactId>json</artifactId>
       <version>20190722</version>
     </dependency>
-    <dependency>
-      <groupId>com.fasterxml.jackson.dataformat</groupId>
-      <artifactId>jackson-dataformat-yaml</artifactId>
-      <version>2.7.7</version>
-      <scope>test</scope>
-    </dependency>
     <dependency>
       <groupId>io.jenkins.configuration-as-code</groupId>
       <artifactId>snakeyaml</artifactId>
@@ -93,32 +75,10 @@
       <version>1.19.0</version>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>com.vladsch.flexmark</groupId>
-      <artifactId>flexmark-all</artifactId>
-      <version>0.50.42</version>
-      <scope>test</scope>
-    </dependency>
   </dependencies>
 
   <build>
     <plugins>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-jar-plugin</artifactId>
-        <executions>
-          <execution>
-            <goals>
-              <goal>test-jar</goal>
-            </goals>
-            <configuration>
-              <includes>
-                <include>io/jenkins/plugins/casc/misc/**</include>
-              </includes>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
       <plugin>
         <artifactId>maven-javadoc-plugin</artifactId>
         <configuration>

pom.xml

@@ -16,6 +16,7 @@
     <module>snakeyaml</module>
     <module>plugin</module>
     <module>integrations</module>
+    <module>test-harness</module>
   </modules>
 
   <properties>

test-harness/pom.xml

@@ -0,0 +1,66 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <artifactId>parent</artifactId>
+    <groupId>io.jenkins.configuration-as-code</groupId>
+    <version>${revision}${changelist}</version>
+    <relativePath>..</relativePath>
+  </parent>
+
+  <artifactId>test-harness</artifactId>
+  <name>Configuration as Code Test harness</name>
+  <description>Functionality to make testing jcasc integration easier</description>
+
+  <properties>
+    <jackson.version>2.10.1</jackson.version>
+    <spotbugs.skip>true</spotbugs.skip>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.jenkins</groupId>
+      <artifactId>configuration-as-code</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.everit.json</groupId>
+      <artifactId>org.everit.json.schema</artifactId>
+      <version>1.5.1</version>
+      <exclusions>
+        <exclusion>
+          <groupId>commons-beanutils</groupId>
+          <artifactId>commons-beanutils</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.jenkins-ci.main</groupId>
+      <artifactId>jenkins-test-harness</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+      <version>${jackson.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.dataformat</groupId>
+      <artifactId>jackson-dataformat-yaml</artifactId>
+      <version>${jackson.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.vladsch.flexmark</groupId>
+      <artifactId>flexmark-all</artifactId>
+      <version>0.50.42</version>
+    </dependency>
+    <dependency>
+      <groupId>com.github.stefanbirkner</groupId>
+      <artifactId>system-rules</artifactId>
+      <version>1.19.0</version>
+    </dependency>
+  </dependencies>
+
+</project>

plugin/src/test/java/io/jenkins/plugins/casc/misc/ConfiguredWithCode.java → test-harness/src/main/java/io/jenkins/plugins/casc/misc/ConfiguredWithCode.java

@@ -16,7 +16,8 @@
 public @interface ConfiguredWithCode {
 
     /**
-     * resource path in classpath
+     * Resource path in classpath
+     * @return resources to configure the test case with.
      */
     String[] value();
 

plugin/src/test/java/io/jenkins/plugins/casc/misc/ConfiguredWithReadme.java → test-harness/src/main/java/io/jenkins/plugins/casc/misc/ConfiguredWithReadme.java

@@ -16,7 +16,8 @@
 public @interface ConfiguredWithReadme {
 
     /**
-     * resource path in the project
+     * Resource path in classpath
+     * @return resources to configure the test case with
      */
     String[] value();
 

plugin/src/test/java/io/jenkins/plugins/casc/misc/RoundTripAbstractTest.java → test-harness/src/main/java/io/jenkins/plugins/casc/misc/RoundTripAbstractTest.java

@@ -57,6 +57,8 @@ public abstract class RoundTripAbstractTest {
     /**
      * A method to assert if the configuration was correctly loaded. The Jenkins rule and the content of the config
      * supposedly loaded are passed.
+     * @param j a RestartableJenkinsRule instance.
+     * @param configContent expected configuration.
      */
     protected abstract void assertConfiguredAsExpected(RestartableJenkinsRule j, String configContent);