Merge pull request #1882 from IBM/issue-1880

Signed-off-by: Lee Surprenant <[email protected]>

Author: lmsurpre

docs/src/pages/guides/FHIRServerUsersGuide.md

@@ -3,7 +3,7 @@ layout: post
 title:  IBM FHIR Server User's Guide
 description: IBM FHIR Server User's Guide
 Copyright: years 2017, 2021
-lastupdated: "2021-01-21"
+lastupdated: "2021-02-22"
 permalink: /FHIRServerUsersGuide/
 ---
 
@@ -2223,6 +2223,7 @@ This section contains reference information about each of the configuration prop
 |`fhirServer/bulkdata/patientExportPageSize`|int| The search page size for patient/group export, the default value is 200 |
 |`fhirServer/bulkdata/useFhirServerTrustStore`|boolean| If the COS Client should use the IBM FHIR Server's TrustStore to access S3/IBMCOS service |
 |`fhirServer/bulkdata/enableParquet`|boolean| Whether or not the server is configured to support export to parquet; to properly enable it the administrator must first make spark and stocator available to the fhir-bulkimportexport-webapp (e.g through the shared lib at `wlp/user/shared/resources/lib`) |
+|`fhirServer/bulkdata/ignoreImportOutcomes`|boolean| Control if push OperationOutcomes to COS/S3. |
 
 ### 5.1.2 Default property values
 | Property Name                 | Default value   |
@@ -2301,6 +2302,7 @@ This section contains reference information about each of the configuration prop
 |`fhirServer/bulkdata/patientExportPageSize`|200|
 |`fhirServer/bulkdata/useFhirServerTrustStore`|false|
 |`fhirServer/bulkdata/enableParquet`|false|
+|`fhirServer/bulkdata/ignoreImportOutcomes`|false|
 
 ### 5.1.3 Property attributes
 Depending on the context of their use, config properties can be:
@@ -2400,6 +2402,7 @@ must restart the server for that change to take effect.
 |`fhirServer/bulkdata/patientExportPageSize`|Y|Y|
 |`fhirServer/bulkdata/useFhirServerTrustStore`|Y|Y|
 |`fhirServer/bulkdata/enableParquet`|Y|Y|
+|`fhirServer/bulkdata/ignoreImportOutcomes`|Y|Y|
 
 ## 5.2 Keystores, truststores, and the FHIR server
 

fhir-bulkimportexport-webapp/src/main/java/META-INF/batch-jobs/FhirBulkImportChunkJob.xml

@@ -42,6 +42,7 @@
                     <property name="cos.credential.ibm" value="#{jobParameters['cos.credential.ibm']}"/>
                     <property name="cos.operationoutcomes.bucket.name" value="#{jobParameters['cos.operationoutcomes.bucket.name']}"/>
                     <property name="import.fhir.validation" value="#{jobParameters['import.fhir.validation']}"/>
+                    <property name="incomingUrl" value="#{jobParameters['incomingUrl']}"/>
                 </properties>
             </writer>
         </chunk>

fhir-bulkimportexport-webapp/src/main/java/com/ibm/fhir/jbatch/bulkdata/common/Constants.java

@@ -1,14 +1,13 @@
 /*
- * (C) Copyright IBM Corp. 2019, 2020
+ * (C) Copyright IBM Corp. 2019, 2021
  *
  * SPDX-License-Identifier: Apache-2.0
  */
 
 package com.ibm.fhir.jbatch.bulkdata.common;
 
 /**
- * Constants for BulkExportImport.
- *
+ * Constants for BulkExportImport
  */
 public class Constants {
     public static final String DEFAULT_FHIR_TENANT = "default";
@@ -85,8 +84,6 @@ public class Constants {
     public static final String IMPORT_PARTITTION_WORKITEM = "import.partition.workitem";
     public static final String PARTITION_RESOURCE_TYPE = "partition.resourcetype";
 
-    // Control if push OperationOutcomes to COS/S3.
-    public static final boolean IMPORT_IS_COLLECT_OPERATIONOUTCOMES = true;
     // Retry times when https or amazon s3 client timeout or other error happens, e.g, timeout can happen if the batch write to DB takes
     // longer than the socket timeout, set to retry once for now.
     public static final int IMPORT_RETRY_TIMES = 1;

fhir-bulkimportexport-webapp/src/main/java/com/ibm/fhir/jbatch/bulkdata/load/ChunkReader.java

@@ -1,11 +1,24 @@
 /*
- * (C) Copyright IBM Corp. 2019, 2020
+ * (C) Copyright IBM Corp. 2019, 2021
  *
  * SPDX-License-Identifier: Apache-2.0
  */
 
 package com.ibm.fhir.jbatch.bulkdata.load;
 
+import static com.ibm.fhir.jbatch.bulkdata.common.Constants.COS_API_KEY;
+import static com.ibm.fhir.jbatch.bulkdata.common.Constants.COS_BUCKET_NAME;
+import static com.ibm.fhir.jbatch.bulkdata.common.Constants.COS_ENDPOINT_URL;
+import static com.ibm.fhir.jbatch.bulkdata.common.Constants.COS_IS_IBM_CREDENTIAL;
+import static com.ibm.fhir.jbatch.bulkdata.common.Constants.COS_LOCATION;
+import static com.ibm.fhir.jbatch.bulkdata.common.Constants.COS_SRVINST_ID;
+import static com.ibm.fhir.jbatch.bulkdata.common.Constants.DEFAULT_FHIR_TENANT;
+import static com.ibm.fhir.jbatch.bulkdata.common.Constants.FHIR_DATASTORE_ID;
+import static com.ibm.fhir.jbatch.bulkdata.common.Constants.FHIR_TENANT;
+import static com.ibm.fhir.jbatch.bulkdata.common.Constants.IMPORT_FHIR_STORAGE_TYPE;
+import static com.ibm.fhir.jbatch.bulkdata.common.Constants.IMPORT_PARTITTION_WORKITEM;
+import static com.ibm.fhir.jbatch.bulkdata.common.Constants.PARTITION_RESOURCE_TYPE;
+
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
@@ -24,18 +37,14 @@
 import com.ibm.fhir.config.FHIRConfiguration;
 import com.ibm.fhir.config.FHIRRequestContext;
 import com.ibm.fhir.jbatch.bulkdata.common.BulkDataUtils;
-import com.ibm.fhir.jbatch.bulkdata.common.Constants;
 import com.ibm.fhir.model.resource.Resource;
 
 /**
- * Bulk import Chunk implementation - the Reader.
- *
+ * BulkData Import - ChunkReader
  */
 @Dependent
 public class ChunkReader extends AbstractItemReader {
     private static final Logger logger = Logger.getLogger(ChunkReader.class.getName());
-    private AmazonS3 cosClient = null;
-    private int numOfLinesToSkip = 0;
 
     @Inject
     StepContext stepCtx;
@@ -44,79 +53,82 @@ public class ChunkReader extends AbstractItemReader {
      * The data source storage type.
      */
     @Inject
-    @BatchProperty(name = Constants.IMPORT_FHIR_STORAGE_TYPE)
+    @BatchProperty(name = IMPORT_FHIR_STORAGE_TYPE)
     String dataSourceStorageType;
 
     /**
      * The IBM COS API key or S3 access key.
      */
     @Inject
-    @BatchProperty(name = Constants.COS_API_KEY)
+    @BatchProperty(name = COS_API_KEY)
     String cosApiKeyProperty;
 
     /**
      * The IBM COS service instance id or S3 secret key.
      */
     @Inject
-    @BatchProperty(name = Constants.COS_SRVINST_ID)
+    @BatchProperty(name = COS_SRVINST_ID)
     String cosSrvinstId;
 
     /**
      * The IBM COS or S3 End point URL.
      */
     @Inject
-    @BatchProperty(name = Constants.COS_ENDPOINT_URL)
+    @BatchProperty(name = COS_ENDPOINT_URL)
     String cosEndpointUrl;
 
     /**
      * The IBM COS or S3 location.
      */
     @Inject
-    @BatchProperty(name = Constants.COS_LOCATION)
+    @BatchProperty(name = COS_LOCATION)
     String cosLocation;
 
     /**
      * The IBM COS or S3 bucket name to import from.
      */
     @Inject
-    @BatchProperty(name = Constants.COS_BUCKET_NAME)
+    @BatchProperty(name = COS_BUCKET_NAME)
     String cosBucketName;
 
     /**
      * If use IBM credential or S3 secret keys.
      */
     @Inject
-    @BatchProperty(name = Constants.COS_IS_IBM_CREDENTIAL)
+    @BatchProperty(name = COS_IS_IBM_CREDENTIAL)
     String cosCredentialIbm;
 
     /**
      * Work item to process.
      */
     @Inject
-    @BatchProperty(name = Constants.IMPORT_PARTITTION_WORKITEM)
+    @BatchProperty(name = IMPORT_PARTITTION_WORKITEM)
     String importPartitionWorkitem;
 
     /**
-     * Fhir resource type to process.
+     * Resource type to process.
      */
     @Inject
-    @BatchProperty(name = Constants.PARTITION_RESOURCE_TYPE)
+    @BatchProperty(name = PARTITION_RESOURCE_TYPE)
     String importPartitionResourceType;
-    
+
     /**
-     * Fhir tenant id.
+     * Tenant id.
      */
     @Inject
-    @BatchProperty(name = Constants.FHIR_TENANT)
+    @BatchProperty(name = FHIR_TENANT)
     String fhirTenant;
 
     /**
-     * Fhir data store id.
+     * Data store id.
      */
     @Inject
-    @BatchProperty(name = Constants.FHIR_DATASTORE_ID)
+    @BatchProperty(name = FHIR_DATASTORE_ID)
     String fhirDatastoreId;
 
+    private AmazonS3 cosClient = null;
+    private int numOfLinesToSkip = 0;
+
     public ChunkReader() {
         super();
     }
@@ -127,7 +139,7 @@ public Object readItem() throws Exception {
         if (!stepCtx.getBatchStatus().equals(BatchStatus.STARTED)) {
             return null;
         }
-        List<Resource> loadedFhirResources = new ArrayList<Resource>();
+        List<Resource> loadedFhirResources = new ArrayList<>();
         if (logger.isLoggable(Level.FINE)) {
             logger.fine("readItem: get work item:" + importPartitionWorkitem + " resource type: " + importPartitionResourceType);
         }
@@ -179,7 +191,7 @@ public void open(Serializable checkpoint) throws Exception {
                 logger.info("open: Set tenant to default!");
             }
             if (fhirDatastoreId == null) {
-                fhirDatastoreId = Constants.DEFAULT_FHIR_TENANT;
+                fhirDatastoreId = DEFAULT_FHIR_TENANT;
                 logger.info("open: Set DatastoreId to default!");
             }
 
@@ -236,18 +248,15 @@ public void open(Serializable checkpoint) throws Exception {
             chunkData.setInFlyRateBeginMilliSeconds(System.currentTimeMillis());
             stepCtx.setTransientUserData(chunkData);
         }
-
-
     }
 
     @Override
     public void close() throws Exception {
-
+        // No Operation
     }
 
     @Override
     public Serializable checkpointInfo() throws Exception {
         return ImportCheckPointData.fromImportTransientUserData((ImportTransientUserData)stepCtx.getTransientUserData());
     }
-
-}
+}