Merge branch 'dev'

Author: Scott Stafford

CONTRIBUTING.md

@@ -13,17 +13,24 @@ Each contributor should be knowledgeable and proficient at the following project
 * [MarkLogic JUnit Library](https://github.com/rjrudin/ml-junit)
 * [MarkLogic App Deployer](https://github.com/rjrudin/ml-app-deployer)
 * [Git](http://git-scm.com/doc)
+  * [Git Large File System](https://git-lfs.github.com/) - this is required to pull down large binary files used for the Entity Extraction examples
 
 # What software do I need?
 
 * [MarkLogic 8+](http://developer.marklogic.com/products)
 * [Java Development Kit 1.8+](http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html)
-* Recommended: [Gradle 3.+](http://gradle.org/gradle-download/)
+* (Recommended) [Gradle 3.+](http://gradle.org/gradle-download/)
 
 You will need to first install MarkLogic if you haven't already.  It is recommended to use a virtual machine or remote server to run MarkLogic to partition our resources (MarkLogic likes to use a lot of memory).  
 
+# Git Large File System
+
+Make sure that after you clone this repository that you also pull from Git LFS.  Otherwise your tests may not pass. 
+
 # What code baseline should I use?
-We use a [fork-and-Branch Git workflow](http://blog.scottlowe.org/2015/01/27/using-fork-branch-git-workflow/).  The dev branch contains the latest and greatest code, while master represents the latest published version.  **Always branch from DEV**.  If you issue a pull request, make sure to compare against the DEV branch.  
+The dev branch contains the latest and greatest code, while master represents the latest published version.  **Always branch from DEV**.  If you issue a pull request, make sure to compare against the DEV branch.
+
+We use a [fork-and-Branch Git workflow](http://blog.scottlowe.org/2015/01/27/using-fork-branch-git-workflow/).
 
 # Should I use an IDE? 
 Yes, it is recommended to use a Java friendly IDE to make one's life easier.  Of course, you can use a simple text editor if that is your preference.  The lead authors have a recent preference for [IntelliJ](https://www.jetbrains.com/idea/).  We also use Gradle for our swiss army knife of building, testing, deploying, etc.  One of the cool features of Gradle is the ability to create the project files for two popular IDE's, IntelliJ and Eclipse.    

build.gradle

@@ -22,7 +22,7 @@ subprojects {
 
     dependencies {
         compile "org.springframework.batch:spring-batch-core:3.0.7.RELEASE"
-        compile "com.marklogic:ml-javaclient-util:2.10.0-alpha"
+        compile "com.marklogic:ml-javaclient-util:2.10.0"
     }
 
     task wrapper(type: Wrapper) {
@@ -36,7 +36,7 @@ subprojects {
 
     checkstyle {
         configFile = file("${project.rootDir}/dev-tools/checkstyle/checkstyle.xml")
-        toolVersion = '6.19'
+        toolVersion = '7.1.2'
     }
 
     task sourcesJar(type: Jar, dependsOn: classes) {

core/gradle/wrapper/gradle-wrapper.properties

@@ -1,4 +1,4 @@
-#Fri Sep 30 20:57:34 EDT 2016
+#Tue Oct 04 14:55:15 EDT 2016
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME

core/src/main/java/com/marklogic/client/io/MarkLogicWriteHandle.java

@@ -1,6 +1,5 @@
 package com.marklogic.client.io;
 
-import com.marklogic.client.document.DocumentUriTemplate;
 import com.marklogic.client.document.DocumentWriteOperation;
 import com.marklogic.client.io.marker.AbstractWriteHandle;
 import com.marklogic.client.io.marker.DocumentMetadataWriteHandle;
@@ -15,6 +14,10 @@ public class MarkLogicWriteHandle implements DocumentWriteOperation {
     private DocumentMetadataHandle metadataHandle;
     private AbstractWriteHandle handle;
     private OperationType opType;
+
+    public MarkLogicWriteHandle() {
+
+    }
 
     public MarkLogicWriteHandle(String uri, DocumentMetadataHandle metadata, AbstractWriteHandle handle) {
         this.uri = uri;

core/src/main/java/com/marklogic/spring/batch/Main.java

@@ -22,7 +22,6 @@
 import org.springframework.context.annotation.Configuration;
 import org.springframework.core.env.JOptCommandLinePropertySource;
 import org.springframework.core.type.filter.AnnotationTypeFilter;
-import org.springframework.core.type.filter.RegexPatternTypeFilter;
 
 import java.io.File;
 import java.io.FileReader;

core/src/main/java/com/marklogic/spring/batch/core/repository/dao/AbstractMarkLogicBatchMetadataDao.java

@@ -7,25 +7,25 @@
 
 public abstract class AbstractMarkLogicBatchMetadataDao extends LoggingObject {
 
-	protected DatabaseClient databaseClient;
-	
-	protected DataFieldMaxValueIncrementer incrementer;
+    protected DatabaseClient databaseClient;
 
-	protected final long LOWER_RANGE = 0;
+    protected DataFieldMaxValueIncrementer incrementer;
+
+    protected final long LOWER_RANGE = 0;
     protected final long UPPER_RANGE = 9999999;
-    
+
     public final String SEARCH_OPTIONS_NAME = "spring-batch";
-	
-	public final String SPRING_BATCH_DIR = "/projects.spring.io/spring-batch/";
-	
-	public final String COLLECTION_JOB_INSTANCE = "http://marklogic.com/spring-batch/job-instance";
-
-	public DatabaseClient getDatabaseClient() {
-		return databaseClient;
-	}
-	
-	public void setIncrementer(DataFieldMaxValueIncrementer incrementer) {
-		this.incrementer = incrementer;
-	}
+
+    public final String SPRING_BATCH_DIR = "/projects.spring.io/spring-batch/";
+
+    public final String COLLECTION_JOB_INSTANCE = "http://marklogic.com/spring-batch/job-instance";
+
+    public DatabaseClient getDatabaseClient() {
+        return databaseClient;
+    }
+
+    public void setIncrementer(DataFieldMaxValueIncrementer incrementer) {
+        this.incrementer = incrementer;
+    }
 
 }

core/src/main/java/com/marklogic/spring/batch/item/AbstractDocumentWriter.java

@@ -7,8 +7,6 @@
 import com.marklogic.client.io.DocumentMetadataHandle;
 import com.marklogic.client.io.DocumentMetadataHandle.Capability;
 
-import java.io.File;
-
 /**
  * Base class for writing documents. Should be able to support both the Client API and XCC.
  */

core/src/main/java/com/marklogic/spring/batch/item/MarkLogicItemWriter.java

@@ -1,14 +1,14 @@
 package com.marklogic.spring.batch.item;
 
 import com.marklogic.client.DatabaseClient;
-import com.marklogic.client.document.DocumentWriteOperation;
-import com.marklogic.client.document.DocumentWriteSet;
-import com.marklogic.client.document.GenericDocumentManager;
+import com.marklogic.client.document.*;
+import com.marklogic.client.io.Format;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.batch.item.ItemWriter;
 
 import java.util.List;
+import java.util.Map;
 
 /*
 The MarkLogicItemWriter is an ItemWriter used to write any type of document to MarkLogic.  It expects a
@@ -24,6 +24,9 @@ public class MarkLogicItemWriter implements ItemWriter<DocumentWriteOperation> {
     private String outputUriPrefix;
     private String outputUriReplace;
     private String outputUriSuffix;
+    private ServerTransform serverTransform;
+    private Format format;
+    private boolean transformOn = false;
 
     public MarkLogicItemWriter(DatabaseClient client) {
         this.client = client;
@@ -40,9 +43,13 @@ public void write(List<? extends DocumentWriteOperation> items) throws Exception
             uri = (outputUriSuffix != null) ? uri + outputUriSuffix : uri;
             batch.add(uri, item.getMetadata(), item.getContent());
         }
-        docMgr.write(batch);
+        if (!transformOn) {
+            docMgr.write(batch);
+        } else {
+            docMgr.write(batch, serverTransform);
+        }
     }
-    
+
     private String applyOutputUriReplace(String uri, String outputUriReplace) {
         String[] regexReplace = outputUriReplace.split(",");
         for (int i = 0; i < regexReplace.length; i=i+2) {
@@ -74,6 +81,19 @@ public void setOutputUriReplace(String outputUriReplace) {
     public void setOutputUriSuffix(String outputUriSuffix) {
         this.outputUriSuffix = outputUriSuffix;
     }
-    
-    
+
+    /*
+    Applies transform for every document
+    */
+    public void setTransform(Format format, String transformName, Map<String, String> transformParameters) {
+        this.format = format;
+        docMgr.setContentFormat(format);
+        this.serverTransform = new ServerTransform(transformName);
+        if (transformParameters != null) {
+            for (String key : transformParameters.keySet()) {
+                serverTransform.put(key, transformParameters.get(key));
+            }
+        }
+        transformOn = true;
+    }
 }

core/src/main/java/com/marklogic/uri/DefaultUriGenerator.java

@@ -1,6 +1,5 @@
 package com.marklogic.uri;
 
-import java.io.File;
 import java.util.UUID;
 
 public class DefaultUriGenerator implements UriGenerator<String>{

core/src/test/java/com/marklogic/spring/batch/bind/MarshallSpringBatchPojoToXmlTest.java

@@ -1,6 +1,5 @@
 package com.marklogic.spring.batch.bind;
 
-import com.marklogic.client.spring.BasicConfig;
 import com.marklogic.junit.Fragment;
 import com.marklogic.junit.NamespaceProvider;
 import com.marklogic.spring.batch.*;