Verify tests and update documentation

Author: Scott Stafford

examples/rowToDoc/README.md

@@ -1,9 +1,40 @@
 # SQL Example #1 - Import Invoice Documents
 
-This is an batch processing job example that migrates the tabular rows of a SQL Query, transforms them into either JSON or XML and inserts the data into MarkLogic.  
+This is an batch processing job example that migrates the tabular rows of a SQL Query, transforms them into either JSON or XML and inserts the data into MarkLogic.
 
 The RDBMS data set is using the [Invoice data](invoices-sql-diagram.jpg) from the [HSQL relational database](http://www.hsqldb.org/).  
 
+This job can take two tables and create child elements from the many relationship.  For this example, we will examine the
+customer and invoice tables.  These two tables are joined via a one (customer) to 
+many (invoice) relationship.  We have decided that the root element is going to be based on the customer
+and many invoice children will be created.  By renaming the column names using a 
+**[parent element]/[child element]** naming convention in your SQL query, this batch processing job
+ will create the document accordingly.  
+
+    SELECT customer.*, invoice.id as \"invoice/id\", invoice.total as \"invoice/total\" 
+    FROM invoice LEFT JOIN customer on invoice.customerId = customer.id 
+    ORDER BY customer.id
+
+In this example, this would generate the following sample document. 
+
+    <invoice>
+      <ID>13</ID>
+      <FIRSTNAME>Laura</FIRSTNAME>
+      <LASTNAME>Ringer</LASTNAME>
+      <STREET>38 College Av.</STREET>
+      <CITY>New York</CITY>
+      <invoice>
+        <id>43</id>
+        <total>3215</total>
+      </invoice>
+      <invoice>
+        <id>30</id>
+        <total>1376</total>
+      </invoice>
+    </invoice>
+
+Please note that any child elements beyond the second level are not yet supported.  
+
 ## Create RowToDoc Distribution Program
   1) gradle installDist 
   2) Verify the program that was installed under ./build/install/rowToDoc
@@ -21,11 +52,8 @@ This command uses the [gradle application plugin](https://docs.gradle.org/curren
   1) Open rowToDoc.bat and check the host, port, username, and password parameters to make sure that they point to a valid MarkLogic application server
   2) Execute: rowToDoc.bat
 
-For Unix based systems, execute build\install\rowToJson\bin\sql
-
-The program can also be executed via a Gradle JavaExec task
+For Unix based systems, execute build\install\rowToJson\bin\rowToDoc
 
-    gradle importInvoices
 
 
 

examples/rowToDoc/build.gradle

@@ -29,22 +29,3 @@ task runManager(type: JavaExec) {
             "--user", "SA"
     ]
 }
-
-task importInvoices(type: JavaExec) {
-    classpath = sourceSets.main.runtimeClasspath
-    main = "com.marklogic.spring.batch.Main"
-    args = [
-            "--host", "localhost",
-            "--port", "8200",
-            "--username", "admin",
-            "--password", "admin",
-            "--config", "com.marklogic.spring.batch.config.MigrateColumnMapsConfig",
-            "--jdbc_driver", "org.hsqldb.jdbc.JDBCDriver",
-            "--jdbc_url", "jdbc:hsqldb:file:data\\invoices",
-            "--sql", "SELECT * FROM invoice LEFT JOIN customer on invoice.customerId=customer.id LEFT JOIN item on invoice.id=item.invoiceId LEFT JOIN product on product.id=item.productId ORDER BY invoice.id asc",
-            "--jdbc_username", "sa",
-            "--format", "xml",
-            "--root_local_name", "invoice",
-            "--collections", "invoice"
-    ]
-}

examples/rowToDoc/src/main/java/com/marklogic/spring/batch/config/RowToDocConfig.java

@@ -16,7 +16,6 @@
 import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
 import org.springframework.batch.core.configuration.annotation.JobScope;
 import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
-import org.springframework.batch.item.ItemProcessor;
 import org.springframework.batch.item.database.JdbcCursorItemReader;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;

examples/rowToDoc/src/test/java/com/marklogic/spring/batch/config/sql/RowToDocConfigTest.java

@@ -1,5 +1,11 @@
 package com.marklogic.spring.batch.config.sql;
 
+import com.marklogic.client.admin.TransformExtensionsManager;
+import com.marklogic.client.document.XMLDocumentManager;
+import com.marklogic.client.io.FileHandle;
+import com.marklogic.client.io.Format;
+import com.marklogic.client.io.StringHandle;
+import com.marklogic.junit.Fragment;
 import com.marklogic.spring.batch.config.PathAwareColumnMapRowMapper;
 import com.marklogic.spring.batch.config.RowToDocConfig;
 import com.marklogic.spring.batch.test.AbstractJobTest;
@@ -11,11 +17,13 @@
 import org.springframework.batch.item.ExecutionContext;
 import org.springframework.batch.item.database.JdbcCursorItemReader;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.core.io.Resource;
 import org.springframework.jdbc.datasource.embedded.EmbeddedDatabase;
 import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder;
 import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType;
 
 import javax.sql.DataSource;
+import java.io.IOException;
 import java.util.Map;
 
 
@@ -36,20 +44,20 @@ public class RowToDocConfigTest extends AbstractJobTest {
 
     protected static EmbeddedDatabase embeddedDatabase;
     protected final Logger logger = LoggerFactory.getLogger(getClass());
+    private TransformExtensionsManager transMgr;
+    private String sql = "SELECT customer.*, invoice.id as \"invoice/id\", invoice.total as \"invoice/total\" FROM invoice LEFT JOIN customer on invoice.customerId = customer.id ORDER BY customer.id";
 
     @Before
-    public void createDb() {
+    public void createDb() throws IOException {
         EmbeddedDatabaseBuilder builder = new EmbeddedDatabaseBuilder().setType(EmbeddedDatabaseType.HSQL);
         builder.addScripts("db/sampledata_ddl.sql", "db/sampledata_insert.sql");
         embeddedDatabase = builder.build();
-    }
 
-    @After
-    public void teardown() {
-        if (embeddedDatabase != null) {
-            embeddedDatabase.shutdown();
-        }
-        embeddedDatabase = null;
+        Resource transform = getApplicationContext().getResource("classpath:/transforms/simple.xqy");
+        transMgr = getClient().newServerConfigManager().newTransformExtensionsManager();
+        FileHandle fileHandle = new FileHandle(transform.getFile());
+        fileHandle.setFormat(Format.XML);
+        transMgr.writeXQueryTransform("simple", fileHandle);
     }
 
     @Test
@@ -69,7 +77,7 @@ public void jdbcCursorItemReaderTest() throws Exception {
     }
 
     @Test
-    public void runRowToDocJobTest() {
+    public void runRowToDocJobWithTransformTest() {
         runJob(RowToDocTestConfig.class,
                 "--sql", "SELECT customer.*, invoice.id as \"invoice/id\", invoice.total as \"invoice/total\" FROM invoice LEFT JOIN customer on invoice.customerId = customer.id ORDER BY customer.id",
                 "--jdbc_username", "sa",
@@ -78,7 +86,29 @@ public void runRowToDocJobTest() {
                 "--collections", "invoice",
                 "--transform_name", "simple",
                 "--transform_parameters", "monster,grover,trash,oscar");
+        Fragment f = loadInvoice();
+        f.assertElementValue("/invoice/invoice/ID", "13");
+        f.assertElementValue("/invoice/invoice/LASTNAME", "Ringer");
+        f.assertElementExists("/invoice/invoice/invoice[1]/total[. = '3215']");
+        f.assertElementExists("/invoice/invoice/invoice[2]/total[. = '1376']");
+        f.assertElementExists("/invoice/transform");
+        f.assertElementExists("/invoice/monster[. = 'grover']");
+        f.assertElementExists("/invoice/trash[. = 'oscar']");
+    }
 
+    private Fragment loadInvoice() {
+        XMLDocumentManager mgr = getClient().newXMLDocumentManager();
+        String xml = mgr.read("/invoice/13.xml", new StringHandle()).get();
+        return parse(xml);
+    }
+
+    @After
+    public void teardown() {
+        if (embeddedDatabase != null) {
+            embeddedDatabase.shutdown();
+        }
+        embeddedDatabase = null;
+        transMgr.deleteTransform("simple");
     }
 
     @Configuration

examples/rowToDoc/src/test/resources/transforms/simple.xqy

@@ -0,0 +1,19 @@
+xquery version "1.0-ml";
+
+module namespace x = "http://marklogic.com/rest-api/transform/simple";
+
+declare function x:transform(
+  $context as map:map,
+  $params as map:map,
+  $content as document-node())
+as document-node() {
+  document {
+    element { fn:local-name($content/element()) } {
+      $content/element(),
+      element transform {},
+      for $param in map:keys($params)
+      return element { $param } { map:get($params, $param) }
+    }
+  }
+
+};