diff --git a/integration-tests/transforms/0029-parquet-input-no-files.hpl b/integration-tests/transforms/0029-parquet-input-no-files.hpl new file mode 100644 index 00000000000..252f6660a8d --- /dev/null +++ b/integration-tests/transforms/0029-parquet-input-no-files.hpl @@ -0,0 +1,232 @@ + + + + + 0029-parquet-input-no-files + Y + + + + Normal + + + N + 1000 + 100 + - + 2024/12/18 10:21:38.301 + - + 2024/12/18 10:21:38.301 + + + + + + No filenames + Parquet File Input + Y + + + Parquet File Input + Detect empty stream + Y + + + Detect empty stream + Count + Y + + + Count + Output + Y + + + + Count + GroupBy + + Y + + 1 + + none + + + N + Y + ${java.io.tmpdir} + + + count + id + COUNT_ALL + + + N + + + N + grp + + + 544 + 80 + + + + Detect empty stream + DetectEmptyStream + + Y + + 1 + + none + + + + + 416 + 80 + + + + No filenames + DataGrid + + Y + + 1 + + none + + + + + + + + + + -1 + -1 + N + filename + String + + + + + 112 + 80 + + + + Parquet File Input + ParquetFileInput + + Y + + 1 + + none + + + + + id + id + Integer + + + lastname + lastname + String + + + firstname + firstname + String + + + zip + zip + Integer + + + city + city + String + + + birthdate + birthdate + Integer + + + street + street + String + + + housenr + housenr + Integer + + + stateCode + stateCode + String + + + stateDesc + stateDesc + String + + + filename + + + 272 + 80 + + + + Output + Dummy + + Y + + 1 + + none + + + + + 672 + 80 + + + + + + diff --git a/integration-tests/transforms/datasets/golden-parquet-input-no-files.csv b/integration-tests/transforms/datasets/golden-parquet-input-no-files.csv new file mode 100644 index 00000000000..1466dc703c4 --- /dev/null +++ b/integration-tests/transforms/datasets/golden-parquet-input-no-files.csv @@ -0,0 +1,2 @@ +filename,id,lastname,firstname,zip,city,birthdate,street,housenr,stateCode,stateDesc,count +"",,,,,,,,,,," 0000000000" diff --git a/integration-tests/transforms/main-0029-parquet.hwf b/integration-tests/transforms/main-0029-parquet.hwf index f2aff8f5586..f6dcd9d6815 100644 --- a/integration-tests/transforms/main-0029-parquet.hwf +++ b/integration-tests/transforms/main-0029-parquet.hwf @@ -35,14 +35,14 @@ limitations under the License. SPECIAL - N - 0 - 0 - 60 + 1 12 + 60 + 0 0 + N + 0 1 - 1 N 128 80 @@ -53,24 +53,22 @@ limitations under the License. PIPELINE - ${PROJECT_HOME}/0029-parquet-output.hpl - N - N - N - N - N - - N N - Basic - N - Y + N + N N - local + N + ${PROJECT_HOME}/0029-parquet-output.hpl + Basic Y + N + local + N + N + Y N 272 80 @@ -88,6 +86,9 @@ limitations under the License. 0029-parquet-input-nulls UNIT + + 0029-parquet-input-no-files UNIT + N 640 @@ -99,24 +100,22 @@ limitations under the License. PIPELINE - ${PROJECT_HOME}/0029-parquet-output-nulls.hpl - N - N - N - N - N - - N N - Basic - N - Y + N + N N - local + N + ${PROJECT_HOME}/0029-parquet-output-nulls.hpl + Basic Y + N + local + N + N + Y N 448 80 diff --git a/integration-tests/transforms/metadata/dataset/golden-parquet-input-no-files.json b/integration-tests/transforms/metadata/dataset/golden-parquet-input-no-files.json new file mode 100644 index 00000000000..3c1f7f30f8c --- /dev/null +++ b/integration-tests/transforms/metadata/dataset/golden-parquet-input-no-files.json @@ -0,0 +1,104 @@ +{ + "base_filename": "golden-parquet-input-no-files.csv", + "name": "golden-parquet-input-no-files", + "description": "", + "dataset_fields": [ + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_name": "filename", + "field_format": "" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 5, + "field_precision": 0, + "field_name": "id", + "field_format": "####0;-####0" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_name": "lastname", + "field_format": "" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_name": "firstname", + "field_format": "" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 5, + "field_precision": 0, + "field_name": "zip", + "field_format": "####0;-####0" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_name": "city", + "field_format": "" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 5, + "field_precision": 0, + "field_name": "birthdate", + "field_format": "####0;-####0" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_name": "street", + "field_format": "" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 5, + "field_precision": 0, + "field_name": "housenr", + "field_format": "####0;-####0" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_name": "stateCode", + "field_format": "" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_name": "stateDesc", + "field_format": "" + }, + { + "field_comment": "", + "field_length": 10, + "field_type": 5, + "field_precision": 0, + "field_name": "count", + "field_format": " 0000000000;-0000000000" + } + ], + "folder_name": "" +} \ No newline at end of file diff --git a/integration-tests/transforms/metadata/unit-test/0029-parquet-input-no-files UNIT.json b/integration-tests/transforms/metadata/unit-test/0029-parquet-input-no-files UNIT.json new file mode 100644 index 00000000000..26192e82b48 --- /dev/null +++ b/integration-tests/transforms/metadata/unit-test/0029-parquet-input-no-files UNIT.json @@ -0,0 +1,83 @@ +{ + "database_replacements": [], + "autoOpening": true, + "description": "", + "persist_filename": "", + "test_type": "UNIT_TEST", + "variableValues": [], + "basePath": "", + "golden_data_sets": [ + { + "field_mappings": [ + { + "transform_field": "filename", + "data_set_field": "filename" + }, + { + "transform_field": "id", + "data_set_field": "id" + }, + { + "transform_field": "lastname", + "data_set_field": "lastname" + }, + { + "transform_field": "firstname", + "data_set_field": "firstname" + }, + { + "transform_field": "zip", + "data_set_field": "zip" + }, + { + "transform_field": "city", + "data_set_field": "city" + }, + { + "transform_field": "birthdate", + "data_set_field": "birthdate" + }, + { + "transform_field": "street", + "data_set_field": "street" + }, + { + "transform_field": "housenr", + "data_set_field": "housenr" + }, + { + "transform_field": "stateCode", + "data_set_field": "stateCode" + }, + { + "transform_field": "stateDesc", + "data_set_field": "stateDesc" + }, + { + "transform_field": "count", + "data_set_field": "count" + } + ], + "field_order": [ + "filename", + "id", + "lastname", + "firstname", + "zip", + "city", + "birthdate", + "street", + "housenr", + "stateCode", + "stateDesc", + "count" + ], + "data_set_name": "golden-parquet-input-no-files", + "transform_name": "Output" + } + ], + "input_data_sets": [], + "name": "0029-parquet-input-no-files UNIT", + "trans_test_tweaks": [], + "pipeline_filename": "./0029-parquet-input-no-files.hpl" +} \ No newline at end of file diff --git a/plugins/misc/testing/src/main/java/org/apache/hop/testing/actions/runtests/RunPipelineTests.java b/plugins/misc/testing/src/main/java/org/apache/hop/testing/actions/runtests/RunPipelineTests.java index fb46c051bc8..dadb9436f10 100644 --- a/plugins/misc/testing/src/main/java/org/apache/hop/testing/actions/runtests/RunPipelineTests.java +++ b/plugins/misc/testing/src/main/java/org/apache/hop/testing/actions/runtests/RunPipelineTests.java @@ -142,7 +142,7 @@ public Result execute(Result prevResult, int nr) throws HopException { public String[] getReferencedObjectDescriptions() { String[] descriptions = new String[testNames.size()]; for (int i = 0; i < descriptions.length; i++) { - descriptions[i] = "Pipeline of unit test : " + testNames.get(i); + descriptions[i] = "Pipeline of unit test : " + testNames.get(i).getTestName(); } return descriptions; } diff --git a/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/input/ParquetInput.java b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/input/ParquetInput.java index 249e7e228ea..3629f8f7c92 100644 --- a/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/input/ParquetInput.java +++ b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/input/ParquetInput.java @@ -102,7 +102,7 @@ public boolean processRow() throws HopException { } public void closeFile() { - if (!data.readerClosed) { + if (!data.readerClosed && data.reader != null && data.inputStream != null) { try { data.reader.close(); data.inputStream.close();