From cc7e110e3e8801d28b9d32487cd2479c0b58d393 Mon Sep 17 00:00:00 2001 From: PXF Contiuous Integration Date: Thu, 13 Jul 2023 19:44:00 +0000 Subject: [PATCH 01/24] Bump version to 6.7.1-SNAPSHOT [skip ci] --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index f0e13c5090..e6ab902a4c 100644 --- a/version +++ b/version @@ -1 +1 @@ -6.7.0 +6.7.1-SNAPSHOT From 70fab0bebea9b7f11bfbb3488e785c452429b1f3 Mon Sep 17 00:00:00 2001 From: Lisa Owen Date: Tue, 18 Jul 2023 09:06:44 -0600 Subject: [PATCH 02/24] docs - minor edits to correct formatting (#1002) --- docs/content/hdfs_json.html.md.erb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/content/hdfs_json.html.md.erb b/docs/content/hdfs_json.html.md.erb index d6d830d757..7fc683da35 100644 --- a/docs/content/hdfs_json.html.md.erb +++ b/docs/content/hdfs_json.html.md.erb @@ -527,6 +527,7 @@ CREATE EXTERNAL TABLE jso_tbl( ) LOCATION('pxf://data/pxf_examples/jso?PROFILE=hdfs:json&IDENTIFIER=created_at') FORMAT 'CUSTOM' (FORMATTER='pxfwritable_import'); +``` The column names that you specify in the create command must match those of the writable external table. And recall that to read a JSON file that contains a single object, you must specify the `IDENTIFIER` option. @@ -535,10 +536,8 @@ Query the table to read the data: ``` sql SELECT * FROM jso_tbl; - created_at | id_str | id | location | coo -rdinates ----------------------------+--------------------+-----------+--------------+---- ---------- + created_at | id_str | id | location | coordinates +---------------------------+--------------------+-----------+--------------+------------- WedJun1212:37:02+00002013 | 333333333333333333 | 333333333 | NetherWorld | {9,63} SunJun0912:59:07+00002013 | 343136551111111111 | 311111111 | FarAway | {6,50} (2 rows) From 4e490b0e68957410aa1ac80211b4eb8dd9ef15b4 Mon Sep 17 00:00:00 2001 From: "Bradford D. Boyle" Date: Tue, 18 Jul 2023 13:02:22 -0700 Subject: [PATCH 03/24] fix rpmrebuild-rocky8 container image (#1000) There is a compatibility issue between the version of coreutils (v8.30-15) included in rockylinux:8 and the version of Concourse (v5.7.1) used for PXF's CI. This causes the command `ls ` to fail with the error message ```sh ls: cannot access '': Operation not permitted ``` The same comand in the same container image running on Concourse 7.8.3 works successfully. The Greenplum release engineering team previsouly encountered this same issue and resolved it by downgrading coreutils-single to v8.30-8. This version is no longer available from upstream repos for downgrading in the rpmrebuild images. Instead, this commit switches the base image from upstream rockylinux:8 to gpdb6-rocky8-build which is maintained by the Greenplum release engineering team and includes the older version of coreutils-single. For consistency, this commit also changes upstream centos:7 to gpdb6-centos7-build. Other changes: * add upstream base images to triggers * update container image diagram Authored-by: Bradford D. Boyle --- concourse/docker/README.md | 15 ++++----------- concourse/docker/rpmrebuild/centos/Dockerfile | 2 +- concourse/docker/rpmrebuild/cloudbuild.yaml | 4 ++-- concourse/docker/rpmrebuild/rocky/Dockerfile | 2 +- concourse/pipelines/cloudbuild_pipeline.yml | 2 ++ 5 files changed, 10 insertions(+), 15 deletions(-) diff --git a/concourse/docker/README.md b/concourse/docker/README.md index 84893608ad..951bb7df8c 100644 --- a/concourse/docker/README.md +++ b/concourse/docker/README.md @@ -46,21 +46,14 @@ flowchart TD classDef latestStyle fill:#6c71c4,color:white,stroke:none classDef plainStyle fill:none,stroke:black - subgraph dockerhub [Official DockerHub] - centos7[centos:7] - rocky8[rockylinux:8] - class centos7 dockerhubStyle - class rocky8 dockerhubStyle - - end - class dockerhub subgraphStyle - subgraph gcr_images ["GP RelEng Images (gcr.io/data-gpdb-public-images)"] gp5_centos7_latest[centos-gpdb-dev:7-gcc6.2-llvm3.7] gp6_centos7_latest[gpdb6-centos7-test:latest] + gp6_centos7_build[gpdb6-centos7-build:latest] gp6_ubuntu18_latest[gpdb6-ubuntu18.04-test:latest] gp6_oel7_latest[gpdb6-oel7-test:latest] gp6_rocky8_latest[gpdb6-rocky8-test:latest] + gp6_rocky8_build[gpdb6-rocky8-build:latest] gp7_rocky8_latest[gpdb7-rocky8-test:latest] class gp5_centos7_latest gcrPublicStyle @@ -210,9 +203,9 @@ flowchart TD gp7_rocky8_dockerfile -- CloudBuild --> gp7_rocky8_pxf_sha gp7_rocky8_pxf_sha -- "tag (concourse pipeline)" --> gp7_rocky8_pxf_latest - centos7 --> rpm_docker_centos7 + gp6_centos7_build --> rpm_docker_centos7 rpm_docker_centos7 --> rpm_centos7_latest - rocky8 --> rpm_docker_rocky8 + gp6_rocky8_build --> rpm_docker_rocky8 rpm_docker_rocky8 --> rpm_rocky8_latest gp5_centos7_pxf_latest --> mapr_dockerfile diff --git a/concourse/docker/rpmrebuild/centos/Dockerfile b/concourse/docker/rpmrebuild/centos/Dockerfile index 0b63324498..e4431b77dc 100644 --- a/concourse/docker/rpmrebuild/centos/Dockerfile +++ b/concourse/docker/rpmrebuild/centos/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=centos:7 +ARG BASE_IMAGE=gcr.io/data-gpdb-public-images/gpdb6-centos7-build FROM ${BASE_IMAGE} diff --git a/concourse/docker/rpmrebuild/cloudbuild.yaml b/concourse/docker/rpmrebuild/cloudbuild.yaml index 23e0f671ef..5839851b60 100644 --- a/concourse/docker/rpmrebuild/cloudbuild.yaml +++ b/concourse/docker/rpmrebuild/cloudbuild.yaml @@ -8,7 +8,7 @@ steps: id: rpmrebuild-centos7-image args: - 'build' - - '--build-arg=BASE_IMAGE=centos:7' + - '--build-arg=BASE_IMAGE=${_BASE_IMAGE_REPOSITORY}/gpdb6-centos7-build:latest' - '--tag=gcr.io/$PROJECT_ID/rpmrebuild-centos7:latest' - '-f' - 'concourse/docker/rpmrebuild/centos/Dockerfile' @@ -20,7 +20,7 @@ steps: id: rpmrebuild-rocky8-image args: - 'build' - - '--build-arg=BASE_IMAGE=rockylinux:8' + - '--build-arg=BASE_IMAGE=${_BASE_IMAGE_REPOSITORY}/gpdb6-rocky8-build:latest' - '--tag=gcr.io/$PROJECT_ID/rpmrebuild-rocky8:latest' - '-f' - 'concourse/docker/rpmrebuild/rocky/Dockerfile' diff --git a/concourse/docker/rpmrebuild/rocky/Dockerfile b/concourse/docker/rpmrebuild/rocky/Dockerfile index ced3b26c2c..6495caee92 100644 --- a/concourse/docker/rpmrebuild/rocky/Dockerfile +++ b/concourse/docker/rpmrebuild/rocky/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=rockylinux:8 +ARG BASE_IMAGE=gcr.io/data-gpdb-public-images/gpdb6-rocky8-build:latest FROM ${BASE_IMAGE} diff --git a/concourse/pipelines/cloudbuild_pipeline.yml b/concourse/pipelines/cloudbuild_pipeline.yml index 846433bce9..c60243d6e8 100644 --- a/concourse/pipelines/cloudbuild_pipeline.yml +++ b/concourse/pipelines/cloudbuild_pipeline.yml @@ -56,10 +56,12 @@ resources: private_key: ((ud/pxf/secrets/gp-image-baking-repo-key)) paths: - images/docker/gpdb5/gpdb5-centos7-build-test + - images/docker/gpdb6/gpdb6-centos7-build - images/docker/gpdb6/gpdb6-centos7-test - images/docker/gpdb6/gpdb6-ubuntu18.04-test - images/docker/gpdb6/gpdb6-oel7-test - images/docker/gpdb6/gpdb6-rhel8-test + - images/docker/gpdb6/gpdb6-rocky8-build - images/docker/gpdb7/gpdb7-centos7-test - images/docker/gpdb7/gpdb7-rhel8-test - images/docker/gpdb7/gpdb7-rocky8-test From 2543030d20b177751f08309a71b8c06f795b25bd Mon Sep 17 00:00:00 2001 From: Himanshu Pandey Date: Tue, 18 Jul 2023 15:02:42 -0700 Subject: [PATCH 04/24] Disable forks for PR resource (#1001) * PR pipeline security fix --- concourse/pipelines/templates/pr_pipeline-tpl.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/concourse/pipelines/templates/pr_pipeline-tpl.yml b/concourse/pipelines/templates/pr_pipeline-tpl.yml index 2319d2fd36..0a227893f3 100644 --- a/concourse/pipelines/templates/pr_pipeline-tpl.yml +++ b/concourse/pipelines/templates/pr_pipeline-tpl.yml @@ -39,6 +39,7 @@ resources: - name: pxf_src type: pull-request source: + disable_forks: true repository: greenplum-db/pxf base_branch: ((ud/pxf/prod/git-branch)) access_token: ((ud/pxf/secrets/bot-access-token)) From 666ca5e58ba9b11633a94ae0c49c572a22434d06 Mon Sep 17 00:00:00 2001 From: Ashuka Xue Date: Thu, 20 Jul 2023 15:55:48 -0700 Subject: [PATCH 05/24] Update certification pipeline (#1004) * change to debug build for ubuntu18 * we don't need to be certifying every commit. instead we should go back to certifying only released versions of pxf --- concourse/pipelines/certification_pipeline.yml | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/concourse/pipelines/certification_pipeline.yml b/concourse/pipelines/certification_pipeline.yml index bb6571a37d..2a9d030084 100644 --- a/concourse/pipelines/certification_pipeline.yml +++ b/concourse/pipelines/certification_pipeline.yml @@ -57,16 +57,7 @@ resources: type: git icon: git source: - # This pipeline should be using a release tag for running - # the certification tests, but the Ubuntu test is red on the latest release - # of PXF (currently 6.2.0) because of an upstream change in - # greenplum_path.sh. The upstream change only impacts PXF's CI code and - # not the product's functionality. Until the next release of PXF is tagged - # this pipeline should be pointed at the main branch and have the pxf_src - # resource pinned to the commit with the fixed CI code. After the next - # release, this should be reverted back to using a tag_filter. - # TODO: revert this change and unpin resouce in UI. - #tag_filter: release-* + tag_filter: release-* branch: ((pxf-git-branch)) uri: ((ud/pxf/common/git-remote)) @@ -147,7 +138,9 @@ resources: source: bucket: ((ud/pxf/common/gpdb-concourse-resources-prod-bucket-name)) json_key: ((concourse-gcs-resources-service-account-key)) - regexp: server/published/gpdb6/server-rc-(.*)-ubuntu18.04_x86_64.tar.gz + # use debug builds here as upstream gpdb no longer publishes non-debug builds of ubuntu18 release candidates + # TODO: swap out for non-debug builds of ubuntu20 when upstream gpdb start creating release candidates + regexp: server/published/gpdb6/server-rc-(.*)-ubuntu18.04_x86_64.debug.tar.gz ## ---------- PXF Released RPM Artifacts ---------- - name: pxf_gp5_rpm_rhel7 From 0d7c6d3c4c52eb55bfa5cd4aa15ea17a79b0d866 Mon Sep 17 00:00:00 2001 From: Himanshu Pandey Date: Fri, 21 Jul 2023 19:12:32 -0700 Subject: [PATCH 06/24] Few more tests updated to worked with FDW (#988) * Additional Tests for FDW --- .../pxf/automation/components/gpdb/Gpdb.java | 2 +- .../structures/tables/utils/TableFactory.java | 21 +++++++++++++++++++ .../features/jdbc/JdbcHiveTest.java | 7 +++++++ .../features/json/JsonReadTest.java | 21 ++++++++++++++++++- .../automation/features/orc/OrcReadTest.java | 16 +++++++++++++- .../features/parquet/ParquetReadTest.java | 2 ++ 6 files changed, 66 insertions(+), 3 deletions(-) diff --git a/automation/src/main/java/org/greenplum/pxf/automation/components/gpdb/Gpdb.java b/automation/src/main/java/org/greenplum/pxf/automation/components/gpdb/Gpdb.java index b7cf7c66e5..ed65118521 100755 --- a/automation/src/main/java/org/greenplum/pxf/automation/components/gpdb/Gpdb.java +++ b/automation/src/main/java/org/greenplum/pxf/automation/components/gpdb/Gpdb.java @@ -252,7 +252,7 @@ private void createForeignServers(boolean ignoreFail) throws Exception { List servers = Lists.newArrayList( "default_hdfs", "default_hive", - "db_hive_jdbc", // Needed for JdbcHiveTest + "db-hive_jdbc", // Needed for JdbcHiveTest "default_hbase", "default_jdbc", // Needed for JdbcHiveTest and other JdbcTest which refers to the default server. "database_jdbc", diff --git a/automation/src/main/java/org/greenplum/pxf/automation/structures/tables/utils/TableFactory.java b/automation/src/main/java/org/greenplum/pxf/automation/structures/tables/utils/TableFactory.java index fb9155dbbf..a23dc96abf 100755 --- a/automation/src/main/java/org/greenplum/pxf/automation/structures/tables/utils/TableFactory.java +++ b/automation/src/main/java/org/greenplum/pxf/automation/structures/tables/utils/TableFactory.java @@ -228,6 +228,27 @@ public static ReadableExternalTable getPxfReadableTextTable(String name, return exTable; } + /** + * Prepares PXF Readable External or Foreign Table for Json data, using custom format or CSV format and ":json" profile. + * + * @param name name of the table + * @param fields fields of the table + * @param path for external table path + * @param format format used in the external data + * @return PXF Readable External or Foreign table + */ + public static ReadableExternalTable getPxfReadableJsonTable(String name, + String[] fields, + String path, + String format) { + ReadableExternalTable exTable = getReadableExternalOrForeignTable(name, fields, path, format); + if (StringUtils.equals(format, "custom")) { + exTable.setFormatter("pxfwritable_import"); + } + exTable.setProfile(ProtocolUtils.getProtocol().value() + ":json"); + return exTable; + } + /** * Prepares PXF Readable External or Foreign Table for CSV data, using CSV format and ":csv" profile. * diff --git a/automation/src/test/java/org/greenplum/pxf/automation/features/jdbc/JdbcHiveTest.java b/automation/src/test/java/org/greenplum/pxf/automation/features/jdbc/JdbcHiveTest.java index ac2545d45c..ebb555c014 100644 --- a/automation/src/test/java/org/greenplum/pxf/automation/features/jdbc/JdbcHiveTest.java +++ b/automation/src/test/java/org/greenplum/pxf/automation/features/jdbc/JdbcHiveTest.java @@ -1,5 +1,7 @@ package org.greenplum.pxf.automation.features.jdbc; +import annotations.FailsWithFDW; +import annotations.WorksWithFDW; import jsystem.framework.sut.SutFactory; import jsystem.framework.system.SystemManagerImpl; import jsystem.framework.system.SystemObject; @@ -15,6 +17,7 @@ import java.io.File; +@WorksWithFDW public class JdbcHiveTest extends BaseFeature { private static final String HIVE_JDBC_DRIVER_CLASS = "org.apache.hive.jdbc.HiveDriver"; @@ -241,6 +244,8 @@ public void jdbcHiveRead() throws Exception { runTincTest("pxf.features.jdbc.hive.runTest"); } + // Fails with the error: ERROR: PXF server error : java.io.DataInputStream cannot be cast to [B + @FailsWithFDW @Test(groups = {"features", "gpdb", "security"}) public void jdbcHiveWrite() throws Exception { prepareDataForWriteTest(); @@ -266,6 +271,7 @@ public void jdbcHiveReadFromTwoSecuredServers() throws Exception { runTincTest("pxf.features.jdbc.two_secured_hive.runTest"); } + @FailsWithFDW @Test(groups = {"features", "multiClusterSecurity"}) public void jdbcHiveWriteToTwoSecuredServers() throws Exception { prepareDataForWriteTest(); @@ -295,6 +301,7 @@ public void jdbcHiveReadFromSecureServerAndNonSecuredServer() throws Exception { runTincTest("pxf.features.jdbc.secured_and_non_secured_hive.runTest"); } + @FailsWithFDW @Test(groups = {"features", "multiClusterSecurity"}) public void jdbcHiveWriteToSecureServerAndNonSecuredServer() throws Exception { if (hdfsNonSecure == null) return; diff --git a/automation/src/test/java/org/greenplum/pxf/automation/features/json/JsonReadTest.java b/automation/src/test/java/org/greenplum/pxf/automation/features/json/JsonReadTest.java index f1d6dc4539..1c52777664 100755 --- a/automation/src/test/java/org/greenplum/pxf/automation/features/json/JsonReadTest.java +++ b/automation/src/test/java/org/greenplum/pxf/automation/features/json/JsonReadTest.java @@ -1,8 +1,11 @@ package org.greenplum.pxf.automation.features.json; +import annotations.FailsWithFDW; +import annotations.WorksWithFDW; import org.apache.commons.lang.StringUtils; import org.greenplum.pxf.automation.features.BaseFeature; import org.greenplum.pxf.automation.structures.tables.pxf.ReadableExternalTable; +import org.greenplum.pxf.automation.structures.tables.utils.TableFactory; import org.greenplum.pxf.automation.utils.system.ProtocolEnum; import org.greenplum.pxf.automation.utils.system.ProtocolUtils; import org.testng.annotations.Test; @@ -10,6 +13,7 @@ /** * Tests for Json plugin to read HDFS files in JSON format. */ +@WorksWithFDW public class JsonReadTest extends BaseFeature { private String hdfsPath; @@ -249,6 +253,16 @@ public void exceedsMaxSize() throws Exception { * * @throws Exception if test fails to run */ + // Some of these tests failing because the returned error has extra quotes " + // For e.g.: + // ""[truncated 73 chars]; line: 3, column: 22]'. invalid JSON record + // Instead of + // "[truncated 73 chars]; line: 3, column: 22]'. invalid JSON record + // This is getting appended in toCsvField method of GreenplumCSV.java + // And is happening only for the error cases. Adding escape "\" doesn't help here as well. + + // TODO: May be create a separate test suite for all these failing tests + @FailsWithFDW @Test(groups = {"features", "gpdb", "security", "hcfs"}) public void malformedRecord() throws Exception { prepareExternalTable("jsontest_malformed_record", TWEETS_FIELDS, hdfsPath + FILENAME_BROKEN + SUFFIX_JSON, "custom"); @@ -271,6 +285,7 @@ public void malformedRecord() throws Exception { * * @throws Exception if test fails to run */ + @FailsWithFDW @Test(groups = {"features", "gpdb", "security"}) public void malformedRecordWithCsvWireFormat() throws Exception { prepareExternalTable("jsontest_malformed_record", TWEETS_FIELDS, hdfsPath + FILENAME_BROKEN + SUFFIX_JSON, "CSV"); @@ -292,6 +307,7 @@ public void malformedRecordWithCsvWireFormat() throws Exception { * * @throws Exception if test fails to run */ + @FailsWithFDW @Test(groups = {"features", "gpdb", "security", "hcfs"}) public void malformedRecordWithRejectLimit() throws Exception { prepareExternalTable("jsontest_malformed_record_with_reject_limit", TWEETS_FIELDS, hdfsPath + FILENAME_BROKEN + SUFFIX_JSON, "custom"); @@ -317,6 +333,7 @@ public void malformedRecordWithRejectLimit() throws Exception { * * @throws Exception if test fails to run */ + @FailsWithFDW @Test(groups = {"features", "gpdb", "security", "hcfs"}) public void malformedRecordWithRejectLimitWithCsvWireFormat() throws Exception { prepareExternalTable("jsontest_malformed_record_with_reject_limit", TWEETS_FIELDS, hdfsPath + FILENAME_BROKEN + SUFFIX_JSON, "CSV"); @@ -344,6 +361,7 @@ public void malformedRecordWithRejectLimitWithCsvWireFormat() throws Exception { * * @throws Exception if test fails to run */ + @FailsWithFDW @Test(groups = {"features", "gpdb", "security", "hcfs"}) public void mismatchedTypes() throws Exception { prepareExternalTable("jsontest_mismatched_types", SUPPORTED_PRIMITIVE_FIELDS, hdfsPath + FILENAME_MISMATCHED_TYPES + SUFFIX_JSON, "custom"); @@ -361,6 +379,7 @@ public void mismatchedTypes() throws Exception { * * @throws Exception if test fails to run */ + @FailsWithFDW @Test(groups = {"features", "gpdb", "security", "hcfs"}) public void mismatchedTypesWithRejectLimit() throws Exception { prepareExternalTable("jsontest_mismatched_types_with_reject_limit", SUPPORTED_PRIMITIVE_FIELDS, hdfsPath + FILENAME_MISMATCHED_TYPES + SUFFIX_JSON, "custom"); @@ -397,7 +416,7 @@ public void jsonStringArrayAsGpdbText() throws Exception { private void prepareExternalTable(String name, String[] fields, String path, String format) { ProtocolEnum protocol = ProtocolUtils.getProtocol(); - exTable = new ReadableExternalTable(name, fields, + exTable = TableFactory.getPxfReadableJsonTable(name, fields, protocol.getExternalTablePath(hdfs.getBasePath(), path), format); exTable.setHost(pxfHost); exTable.setPort(pxfPort); diff --git a/automation/src/test/java/org/greenplum/pxf/automation/features/orc/OrcReadTest.java b/automation/src/test/java/org/greenplum/pxf/automation/features/orc/OrcReadTest.java index 10d7ffcc16..271e0d3f7c 100644 --- a/automation/src/test/java/org/greenplum/pxf/automation/features/orc/OrcReadTest.java +++ b/automation/src/test/java/org/greenplum/pxf/automation/features/orc/OrcReadTest.java @@ -1,12 +1,14 @@ package org.greenplum.pxf.automation.features.orc; +import annotations.FailsWithFDW; +import annotations.WorksWithFDW; import org.greenplum.pxf.automation.features.BaseFeature; -import org.greenplum.pxf.automation.structures.tables.pxf.ReadableExternalTable; import org.greenplum.pxf.automation.structures.tables.utils.TableFactory; import org.greenplum.pxf.automation.utils.system.ProtocolEnum; import org.greenplum.pxf.automation.utils.system.ProtocolUtils; import org.testng.annotations.Test; +@WorksWithFDW public class OrcReadTest extends BaseFeature { private static final String ORC_PRIMITIVE_TYPES = "orc_types.orc"; @@ -155,6 +157,18 @@ public void orcReadMultiDimensionalLists() throws Exception { runTincTest("pxf.features.orc.read.multidim_list_types.runTest"); } + /* + * FDW fails for the data that contain a NUL-byte (i.e. '\/u000'"). This behaviour is different from external-table but same as GPDB Heap + * FDW Failure: invalid byte sequence for encoding "UTF8": 0x00 + * + * GPDB also throws the same error when copying the data containing a NUL-byte + * + * postgres=# copy test from '/Users/pandeyhi/Documents/bad_data.txt' ; + * ERROR: invalid byte sequence for encoding "UTF8": 0x00 + * TODO Do we need to do some changes to make sure the external-table behaves the same way as GPDB/FDW? + * + */ + @FailsWithFDW @Test(groups = {"features", "gpdb", "security", "hcfs"}) public void orcReadStringsContainingNullByte() throws Exception { prepareReadableExternalTable("pxf_orc_null_in_string", ORC_NULL_IN_STRING_COLUMNS, hdfsPath + ORC_NULL_IN_STRING); diff --git a/automation/src/test/java/org/greenplum/pxf/automation/features/parquet/ParquetReadTest.java b/automation/src/test/java/org/greenplum/pxf/automation/features/parquet/ParquetReadTest.java index cdcaf33b01..9ea5d981b7 100644 --- a/automation/src/test/java/org/greenplum/pxf/automation/features/parquet/ParquetReadTest.java +++ b/automation/src/test/java/org/greenplum/pxf/automation/features/parquet/ParquetReadTest.java @@ -1,5 +1,6 @@ package org.greenplum.pxf.automation.features.parquet; +import annotations.WorksWithFDW; import org.greenplum.pxf.automation.features.BaseFeature; import org.greenplum.pxf.automation.structures.tables.basic.Table; import org.greenplum.pxf.automation.structures.tables.utils.TableFactory; @@ -8,6 +9,7 @@ import org.testng.annotations.Test; import java.io.File; +@WorksWithFDW public class ParquetReadTest extends BaseFeature { private static final String NUMERIC_TABLE = "numeric_precision"; private static final String NUMERIC_UNDEFINED_PRECISION_TABLE = "numeric_undefined_precision"; From a479a8a950ce6e290986b79adb43360e795dcbff Mon Sep 17 00:00:00 2001 From: Himanshu Pandey Date: Mon, 24 Jul 2023 10:14:33 -0700 Subject: [PATCH 07/24] Inclusive Terminology changes (#995) * Changes as per Inclusive Terminology Jira --- .../components/cluster/MultiNodeCluster.java | 12 +-- .../installer/nodes/CoordinatorNode.java | 8 ++ .../cluster/installer/nodes/MasterNode.java | 8 -- .../cluster/installer/nodes/SegmentNode.java | 8 ++ .../cluster/installer/nodes/SlaveNode.java | 8 -- .../testplugin/HBaseAccessorWithFilter.java | 2 +- .../HiveDataFragmenterWithFilter.java | 2 +- .../HiveInputFormatFragmenterWithFilter.java | 2 +- ...ultipleHiveFragmentsPerFileFragmenter.java | 2 +- .../testplugin/UserDataVerifyAccessor.java | 2 +- .../features/general/FailOverTest.java | 6 +- .../sut/MultiHadoopIPAMultiNodesCluster.xml | 6 +- .../sut/MultiHadoopMultiNodesCluster.xml | 6 +- .../test/resources/sut/MultiNodesCluster.xml | 6 +- cli/cmd/cluster.go | 18 ++-- cli/cmd/cluster_test.go | 94 +++++++++---------- cli/cmd/pxf.go | 46 ++++----- concourse/scripts/cli/common.sh | 8 +- concourse/scripts/cli/test_sync.sh | 2 +- external-table/src/pxfuriparser.c | 2 +- .../pxf/service/profile/Profile.java | 6 +- .../pxf/service/HttpRequestParserTest.java | 2 +- 22 files changed, 128 insertions(+), 128 deletions(-) create mode 100755 automation/src/main/java/org/greenplum/pxf/automation/components/cluster/installer/nodes/CoordinatorNode.java delete mode 100755 automation/src/main/java/org/greenplum/pxf/automation/components/cluster/installer/nodes/MasterNode.java create mode 100755 automation/src/main/java/org/greenplum/pxf/automation/components/cluster/installer/nodes/SegmentNode.java delete mode 100755 automation/src/main/java/org/greenplum/pxf/automation/components/cluster/installer/nodes/SlaveNode.java diff --git a/automation/src/main/java/org/greenplum/pxf/automation/components/cluster/MultiNodeCluster.java b/automation/src/main/java/org/greenplum/pxf/automation/components/cluster/MultiNodeCluster.java index 5b3006ab61..4666a2a24b 100755 --- a/automation/src/main/java/org/greenplum/pxf/automation/components/cluster/MultiNodeCluster.java +++ b/automation/src/main/java/org/greenplum/pxf/automation/components/cluster/MultiNodeCluster.java @@ -12,9 +12,9 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; -import org.greenplum.pxf.automation.components.cluster.installer.nodes.MasterNode; +import org.greenplum.pxf.automation.components.cluster.installer.nodes.CoordinatorNode; import org.greenplum.pxf.automation.components.cluster.installer.nodes.Node; -import org.greenplum.pxf.automation.components.cluster.installer.nodes.SlaveNode; +import org.greenplum.pxf.automation.components.cluster.installer.nodes.SegmentNode; import org.greenplum.pxf.automation.components.common.cli.ParallelShellActions; import org.greenplum.pxf.automation.utils.jsystem.report.ReportUtils; @@ -113,11 +113,11 @@ private void handleOperation(String operation, EnumClusterServices service) thro List nodesListByService; switch (service) { case hive: - nodesListByService = getNode(MasterNode.class, service); + nodesListByService = getNode(CoordinatorNode.class, service); break; case pxf: nodesListByService = getNode(service).stream() - .filter(n -> n instanceof SlaveNode) + .filter(n -> n instanceof SegmentNode) .collect(Collectors.toList()); break; default: @@ -150,7 +150,7 @@ public void fetchConfiguration(String targetDirectory) throws Exception { // currently copy only the pxf-conf to tempClusterConfDirectory FileUtils.copyDirectory(new File(getPxfConfLocation()), new File(tempClusterConfDirectory + "/" + getPathToPxfConfInGeneralConf())); // if current node is not pxf node, it requires copying pxf/conf directory from the pxf node - Node pxfNode = getNode(MasterNode.class, EnumClusterServices.pxf).get(0); + Node pxfNode = getNode(CoordinatorNode.class, EnumClusterServices.pxf).get(0); // if pxf node is same as local node, then pxf conf is already there, skip pxf conf copying String localHostName = Inet4Address.getLocalHost().getHostName(); if (!localHostName.equals(pxfNode.getHost())) { @@ -223,7 +223,7 @@ public void runCommandOnNodes(List nodes, String command) throws Exception /** * Gets node List from nodes array according to {@link Node} type and serviceType * - * @param nodeType {@link MasterNode} or {@link SlaveNode} + * @param nodeType {@link CoordinatorNode} or {@link SegmentNode} * @param serviceType required service type to locate in nodes * @return list of nodes of given nodeType and serviceType */ diff --git a/automation/src/main/java/org/greenplum/pxf/automation/components/cluster/installer/nodes/CoordinatorNode.java b/automation/src/main/java/org/greenplum/pxf/automation/components/cluster/installer/nodes/CoordinatorNode.java new file mode 100755 index 0000000000..7980de007c --- /dev/null +++ b/automation/src/main/java/org/greenplum/pxf/automation/components/cluster/installer/nodes/CoordinatorNode.java @@ -0,0 +1,8 @@ +package org.greenplum.pxf.automation.components.cluster.installer.nodes; + +/** + * Represents Coordinator Node for a cluster + */ +public class CoordinatorNode extends Node { + +} diff --git a/automation/src/main/java/org/greenplum/pxf/automation/components/cluster/installer/nodes/MasterNode.java b/automation/src/main/java/org/greenplum/pxf/automation/components/cluster/installer/nodes/MasterNode.java deleted file mode 100755 index e52404d83a..0000000000 --- a/automation/src/main/java/org/greenplum/pxf/automation/components/cluster/installer/nodes/MasterNode.java +++ /dev/null @@ -1,8 +0,0 @@ -package org.greenplum.pxf.automation.components.cluster.installer.nodes; - -/** - * Represents Master Node for a cluster - */ -public class MasterNode extends Node { - -} diff --git a/automation/src/main/java/org/greenplum/pxf/automation/components/cluster/installer/nodes/SegmentNode.java b/automation/src/main/java/org/greenplum/pxf/automation/components/cluster/installer/nodes/SegmentNode.java new file mode 100755 index 0000000000..5b744028d2 --- /dev/null +++ b/automation/src/main/java/org/greenplum/pxf/automation/components/cluster/installer/nodes/SegmentNode.java @@ -0,0 +1,8 @@ +package org.greenplum.pxf.automation.components.cluster.installer.nodes; + +/** + * Represents a single Segment Node in a cluster + */ +public class SegmentNode extends Node { + +} diff --git a/automation/src/main/java/org/greenplum/pxf/automation/components/cluster/installer/nodes/SlaveNode.java b/automation/src/main/java/org/greenplum/pxf/automation/components/cluster/installer/nodes/SlaveNode.java deleted file mode 100755 index 27f9314e6a..0000000000 --- a/automation/src/main/java/org/greenplum/pxf/automation/components/cluster/installer/nodes/SlaveNode.java +++ /dev/null @@ -1,8 +0,0 @@ -package org.greenplum.pxf.automation.components.cluster.installer.nodes; - -/** - * Represents a single Slave Node in a cluster - */ -public class SlaveNode extends Node { - -} diff --git a/automation/src/main/java/org/greenplum/pxf/automation/testplugin/HBaseAccessorWithFilter.java b/automation/src/main/java/org/greenplum/pxf/automation/testplugin/HBaseAccessorWithFilter.java index c167e596fb..92e5ddb833 100755 --- a/automation/src/main/java/org/greenplum/pxf/automation/testplugin/HBaseAccessorWithFilter.java +++ b/automation/src/main/java/org/greenplum/pxf/automation/testplugin/HBaseAccessorWithFilter.java @@ -246,7 +246,7 @@ private void addColumns() { */ private void addFilters() throws Exception { - // TODO whitelist option + // TODO allowlist option String filterStr = context.getOption("TEST-HBASE-FILTER"); LOG.debug("user defined filter: {}", filterStr); if ((filterStr == null) || filterStr.isEmpty() || "null".equals(filterStr)) diff --git a/automation/src/main/java/org/greenplum/pxf/automation/testplugin/HiveDataFragmenterWithFilter.java b/automation/src/main/java/org/greenplum/pxf/automation/testplugin/HiveDataFragmenterWithFilter.java index 5193fea902..f406783569 100755 --- a/automation/src/main/java/org/greenplum/pxf/automation/testplugin/HiveDataFragmenterWithFilter.java +++ b/automation/src/main/java/org/greenplum/pxf/automation/testplugin/HiveDataFragmenterWithFilter.java @@ -20,7 +20,7 @@ public void afterPropertiesSet() { */ private void addFilters() { - //TODO whitelist the option + //TODO allowlist option String filterStr = context.getOption("TEST-HIVE-FILTER"); LOG.debug("user defined filter: " + filterStr); if ((filterStr == null) || filterStr.isEmpty() || "null".equals(filterStr)) diff --git a/automation/src/main/java/org/greenplum/pxf/automation/testplugin/HiveInputFormatFragmenterWithFilter.java b/automation/src/main/java/org/greenplum/pxf/automation/testplugin/HiveInputFormatFragmenterWithFilter.java index 0aa57a8539..cafe87351d 100755 --- a/automation/src/main/java/org/greenplum/pxf/automation/testplugin/HiveInputFormatFragmenterWithFilter.java +++ b/automation/src/main/java/org/greenplum/pxf/automation/testplugin/HiveInputFormatFragmenterWithFilter.java @@ -20,7 +20,7 @@ public void afterPropertiesSet() { */ private void addFilters() { - // TODO: whitelist the option + // TODO: allowlist the option String filterStr = context.getOption("TEST-HIVE-FILTER"); LOG.debug("user defined filter: " + filterStr); if ((filterStr == null) || filterStr.isEmpty() || "null".equals(filterStr)) diff --git a/automation/src/main/java/org/greenplum/pxf/automation/testplugin/MultipleHiveFragmentsPerFileFragmenter.java b/automation/src/main/java/org/greenplum/pxf/automation/testplugin/MultipleHiveFragmentsPerFileFragmenter.java index 43515ac665..56d20301ff 100755 --- a/automation/src/main/java/org/greenplum/pxf/automation/testplugin/MultipleHiveFragmentsPerFileFragmenter.java +++ b/automation/src/main/java/org/greenplum/pxf/automation/testplugin/MultipleHiveFragmentsPerFileFragmenter.java @@ -65,7 +65,7 @@ public void afterPropertiesSet() { @Override public List getFragments() throws Exception { - // TODO whitelist property + // TODO allowlist property int fragmentsNum = Integer.parseInt(context.getOption("TEST-FRAGMENTS-NUM")); Metadata.Item tblDesc = hiveClientWrapper.extractTableFromName(context.getDataSource()); Table tbl; diff --git a/automation/src/main/java/org/greenplum/pxf/automation/testplugin/UserDataVerifyAccessor.java b/automation/src/main/java/org/greenplum/pxf/automation/testplugin/UserDataVerifyAccessor.java index c0708f7950..4544752426 100755 --- a/automation/src/main/java/org/greenplum/pxf/automation/testplugin/UserDataVerifyAccessor.java +++ b/automation/src/main/java/org/greenplum/pxf/automation/testplugin/UserDataVerifyAccessor.java @@ -21,7 +21,7 @@ public class UserDataVerifyAccessor extends BasePlugin implements Accessor { @Override public boolean openForRead() { - // TODO whitelist the option + // TODO allowlist the option FilterVerifyFragmentMetadata metadata = context.getFragmentMetadata(); filter = metadata.getFilter(); userDelimiter = String.valueOf(context.getGreenplumCSV().getDelimiter()); diff --git a/automation/src/test/java/org/greenplum/pxf/automation/features/general/FailOverTest.java b/automation/src/test/java/org/greenplum/pxf/automation/features/general/FailOverTest.java index e52686acec..e85f0c06e4 100644 --- a/automation/src/test/java/org/greenplum/pxf/automation/features/general/FailOverTest.java +++ b/automation/src/test/java/org/greenplum/pxf/automation/features/general/FailOverTest.java @@ -28,19 +28,19 @@ protected void beforeClass() throws Exception { @Override protected void afterClass() throws Exception { super.afterClass(); - // We need to restore the service after it has been killed + // We need to restore the service after it has been stopped if (cluster != null) { cluster.start(PhdCluster.EnumClusterServices.pxf); } } /** - * Should kill the JVM by invoking OutOfMemoryFragmenter + * Should stop the JVM by invoking OutOfMemoryFragmenter * * @throws Exception */ @Test(groups = {"features", "gpdb", "security"}) - public void killTomcatOnOutOfMemory() throws Exception { + public void stopTomcatOnOutOfMemory() throws Exception { // Create PXF external table for out of memory testing ReadableExternalTable pxfExternalTable = new ReadableExternalTable("test_out_of_memory", new String[] { diff --git a/automation/src/test/resources/sut/MultiHadoopIPAMultiNodesCluster.xml b/automation/src/test/resources/sut/MultiHadoopIPAMultiNodesCluster.xml index 9fe2f53598..8dc30fe512 100644 --- a/automation/src/test/resources/sut/MultiHadoopIPAMultiNodesCluster.xml +++ b/automation/src/test/resources/sut/MultiHadoopIPAMultiNodesCluster.xml @@ -8,7 +8,7 @@ /hive/warehouse/ - org.greenplum.pxf.automation.components.cluster.installer.nodes.MasterNode + org.greenplum.pxf.automation.components.cluster.installer.nodes.CoordinatorNode cdw gpadmin @@ -17,7 +17,7 @@ - org.greenplum.pxf.automation.components.cluster.installer.nodes.SlaveNode + org.greenplum.pxf.automation.components.cluster.installer.nodes.SegmentNode sdw1 gpadmin @@ -26,7 +26,7 @@ - org.greenplum.pxf.automation.components.cluster.installer.nodes.SlaveNode + org.greenplum.pxf.automation.components.cluster.installer.nodes.SegmentNode sdw2 gpadmin diff --git a/automation/src/test/resources/sut/MultiHadoopMultiNodesCluster.xml b/automation/src/test/resources/sut/MultiHadoopMultiNodesCluster.xml index 85dbb3032d..c0b782712d 100644 --- a/automation/src/test/resources/sut/MultiHadoopMultiNodesCluster.xml +++ b/automation/src/test/resources/sut/MultiHadoopMultiNodesCluster.xml @@ -8,7 +8,7 @@ /hive/warehouse/ - org.greenplum.pxf.automation.components.cluster.installer.nodes.MasterNode + org.greenplum.pxf.automation.components.cluster.installer.nodes.CoordinatorNode cdw gpadmin @@ -17,7 +17,7 @@ - org.greenplum.pxf.automation.components.cluster.installer.nodes.SlaveNode + org.greenplum.pxf.automation.components.cluster.installer.nodes.SegmentNode sdw1 gpadmin @@ -26,7 +26,7 @@ - org.greenplum.pxf.automation.components.cluster.installer.nodes.SlaveNode + org.greenplum.pxf.automation.components.cluster.installer.nodes.SegmentNode sdw2 gpadmin diff --git a/automation/src/test/resources/sut/MultiNodesCluster.xml b/automation/src/test/resources/sut/MultiNodesCluster.xml index ef3b145f8d..e9ee334192 100644 --- a/automation/src/test/resources/sut/MultiNodesCluster.xml +++ b/automation/src/test/resources/sut/MultiNodesCluster.xml @@ -5,21 +5,21 @@ /usr/local/greenplum-db-devel /hive/warehouse/ - org.greenplum.pxf.automation.components.cluster.installer.nodes.MasterNode + org.greenplum.pxf.automation.components.cluster.installer.nodes.CoordinatorNode cdw gpadmin gpdb,pxf - org.greenplum.pxf.automation.components.cluster.installer.nodes.SlaveNode + org.greenplum.pxf.automation.components.cluster.installer.nodes.SegmentNode sdw1 gpadmin pxf - org.greenplum.pxf.automation.components.cluster.installer.nodes.SlaveNode + org.greenplum.pxf.automation.components.cluster.installer.nodes.SegmentNode sdw2 gpadmin diff --git a/cli/cmd/cluster.go b/cli/cmd/cluster.go index acb4545940..bb56384d07 100644 --- a/cli/cmd/cluster.go +++ b/cli/cmd/cluster.go @@ -39,14 +39,14 @@ func createCobraCommand(use string, short string, cmd *command) *cobra.Command { var ( clusterCmd = createCobraCommand("cluster", "Perform on each segment host in the cluster", nil) - initCmd = createCobraCommand("init", "(deprecated) Install PXF extension under $GPHOME on master, standby master, and all segment hosts", &InitCommand) - startCmd = createCobraCommand("start", "Start the PXF server instances on master, standby master, and all segment hosts", &StartCommand) - stopCmd = createCobraCommand("stop", "Stop the PXF server instances on master, standby master, and all segment hosts", &StopCommand) - statusCmd = createCobraCommand("status", "Get status of PXF servers on master, standby master, and all segment hosts", &StatusCommand) - syncCmd = createCobraCommand("sync", "Sync PXF configs from master to standby master and all segment hosts. Use --delete to delete extraneous remote files", &SyncCommand) + initCmd = createCobraCommand("init", "(deprecated) Install PXF extension under $GPHOME on coordinator, standby coordinator, and all segment hosts", &InitCommand) + startCmd = createCobraCommand("start", "Start the PXF server instances on coordinator, standby coordinator, and all segment hosts", &StartCommand) + stopCmd = createCobraCommand("stop", "Stop the PXF server instances on coordinator, standby coordinator, and all segment hosts", &StopCommand) + statusCmd = createCobraCommand("status", "Get status of PXF servers on coordinator, standby coordinator, and all segment hosts", &StatusCommand) + syncCmd = createCobraCommand("sync", "Sync PXF configs from coordinator to standby coordinator and all segment hosts. Use --delete to delete extraneous remote files", &SyncCommand) resetCmd = createCobraCommand("reset", "(deprecated) No operation", &ResetCommand) - registerCmd = createCobraCommand("register", "Install PXF extension under $GPHOME on master, standby master, and all segment hosts", &RegisterCommand) - restartCmd = createCobraCommand("restart", "Restart the PXF server on master, standby master, and all segment hosts", &RestartCommand) + registerCmd = createCobraCommand("register", "Install PXF extension under $GPHOME on coordinator, standby coordinator, and all segment hosts", &RegisterCommand) + restartCmd = createCobraCommand("restart", "Restart the PXF server on coordinator, standby coordinator, and all segment hosts", &RestartCommand) prepareCmd = createCobraCommand("prepare", "Prepares a new base directory specified by the $PXF_BASE environment variable", &PrepareCommand) migrateCmd = createCobraCommand("migrate", "Migrates configurations from older installations of PXF", &MigrateCommand) // DeleteOnSync is a boolean for determining whether to use rsync with --delete, exported for tests @@ -138,7 +138,7 @@ func doSetup() (*ClusterData, error) { err := connection.Connect(1) if err != nil { gplog.Error(fmt.Sprintf("ERROR: Could not connect to GPDB.\n%s\n"+ - "Please make sure that your Greenplum database is running and you are on the master node.", err.Error())) + "Please make sure that your Greenplum database is running and you are on the coordinator node.", err.Error())) return nil, err } segConfigs, err := cluster.GetSegmentConfiguration(connection, true) @@ -176,7 +176,7 @@ func clusterRun(cmd *command, clusterData *ClusterData) error { func isStandbyAloneOnHost(clusterData *ClusterData) bool { standbyHost := clusterData.Cluster.GetHostForContent(-1, "m") if standbyHost == "" { - return false // there is no standby master + return false // there is no standby coordinator } return len(clusterData.Cluster.GetContentsForHost(standbyHost)) == 1 } diff --git a/cli/cmd/cluster_test.go b/cli/cmd/cluster_test.go index 905fa1b5d5..cb199f8470 100644 --- a/cli/cmd/cluster_test.go +++ b/cli/cmd/cluster_test.go @@ -35,108 +35,108 @@ var ( ) var _ = Describe("GenerateStatusReport()", func() { - Context("When there is no standby master", func() { - It("reports master host and segment hosts are initializing, resetting, registering, syncing, preparing, and migrating", func() { + Context("When there is no standby coordinator", func() { + It("reports coordinator host and segment hosts are initializing, resetting, registering, syncing, preparing, and migrating", func() { cmd.GenerateStatusReport(&cmd.InitCommand, createClusterData(3, clusterWithoutStandby)) - Expect(testStdout).To(gbytes.Say("Initializing PXF on master host and 2 segment hosts")) + Expect(testStdout).To(gbytes.Say("Initializing PXF on coordinator host and 2 segment hosts")) cmd.GenerateStatusReport(&cmd.ResetCommand, createClusterData(3, clusterWithoutStandby)) - Expect(testStdout).To(gbytes.Say("Resetting PXF on master host and 2 segment hosts")) + Expect(testStdout).To(gbytes.Say("Resetting PXF on coordinator host and 2 segment hosts")) cmd.GenerateStatusReport(&cmd.SyncCommand, createClusterData(2, clusterWithoutStandby)) - Expect(testStdout).To(gbytes.Say("Syncing PXF configuration files from master host to 2 segment hosts")) + Expect(testStdout).To(gbytes.Say("Syncing PXF configuration files from coordinator host to 2 segment hosts")) cmd.GenerateStatusReport(&cmd.RegisterCommand, createClusterData(3, clusterWithoutStandby)) - Expect(testStdout).To(gbytes.Say("Installing PXF extension on master host and 2 segment hosts")) + Expect(testStdout).To(gbytes.Say("Installing PXF extension on coordinator host and 2 segment hosts")) cmd.GenerateStatusReport(&cmd.PrepareCommand, createClusterData(3, clusterWithoutStandby)) - Expect(testStdout).To(gbytes.Say("Preparing PXF on master host and 2 segment hosts")) + Expect(testStdout).To(gbytes.Say("Preparing PXF on coordinator host and 2 segment hosts")) cmd.GenerateStatusReport(&cmd.MigrateCommand, createClusterData(3, clusterWithoutStandby)) - Expect(testStdout).To(gbytes.Say("Migrating PXF configuration on master host and 2 segment hosts")) + Expect(testStdout).To(gbytes.Say("Migrating PXF configuration on coordinator host and 2 segment hosts")) }) It("reports segment hosts are starting, stopping, restarting and statusing", func() { cmd.GenerateStatusReport(&cmd.StartCommand, createClusterData(3, clusterWithoutStandby)) - Expect(testStdout).To(gbytes.Say("Starting PXF on master host and 2 segment hosts")) + Expect(testStdout).To(gbytes.Say("Starting PXF on coordinator host and 2 segment hosts")) cmd.GenerateStatusReport(&cmd.StopCommand, createClusterData(3, clusterWithoutStandby)) - Expect(testStdout).To(gbytes.Say("Stopping PXF on master host and 2 segment hosts")) + Expect(testStdout).To(gbytes.Say("Stopping PXF on coordinator host and 2 segment hosts")) cmd.GenerateStatusReport(&cmd.RestartCommand, createClusterData(3, clusterWithoutStandby)) - Expect(testStdout).To(gbytes.Say("Restarting PXF on master host and 2 segment hosts")) + Expect(testStdout).To(gbytes.Say("Restarting PXF on coordinator host and 2 segment hosts")) cmd.GenerateStatusReport(&cmd.StatusCommand, createClusterData(3, clusterWithoutStandby)) - Expect(testStdout).To(gbytes.Say("Checking status of PXF servers on master host and 2 segment hosts")) + Expect(testStdout).To(gbytes.Say("Checking status of PXF servers on coordinator host and 2 segment hosts")) }) }) - Context("When there is a standby master on its own host", func() { - It("reports master host, standby master host and segment hosts are initializing, resetting, registering, syncing, preparing, and migrating", func() { + Context("When there is a standby coordinator on its own host", func() { + It("reports coordinator host, standby coordinator host and segment hosts are initializing, resetting, registering, syncing, preparing, and migrating", func() { cmd.GenerateStatusReport(&cmd.InitCommand, createClusterData(4, clusterWithStandby)) - Expect(testStdout).To(gbytes.Say("Initializing PXF on master host, standby master host, and 2 segment hosts")) + Expect(testStdout).To(gbytes.Say("Initializing PXF on coordinator host, standby coordinator host, and 2 segment hosts")) cmd.GenerateStatusReport(&cmd.ResetCommand, createClusterData(4, clusterWithStandby)) - Expect(testStdout).To(gbytes.Say("Resetting PXF on master host, standby master host, and 2 segment hosts")) + Expect(testStdout).To(gbytes.Say("Resetting PXF on coordinator host, standby coordinator host, and 2 segment hosts")) cmd.GenerateStatusReport(&cmd.SyncCommand, createClusterData(3, clusterWithStandby)) - Expect(testStdout).To(gbytes.Say("Syncing PXF configuration files from master host to standby master host and 2 segment hosts")) + Expect(testStdout).To(gbytes.Say("Syncing PXF configuration files from coordinator host to standby coordinator host and 2 segment hosts")) cmd.GenerateStatusReport(&cmd.RegisterCommand, createClusterData(4, clusterWithStandby)) - Expect(testStdout).To(gbytes.Say("Installing PXF extension on master host, standby master host, and 2 segment hosts")) + Expect(testStdout).To(gbytes.Say("Installing PXF extension on coordinator host, standby coordinator host, and 2 segment hosts")) cmd.GenerateStatusReport(&cmd.PrepareCommand, createClusterData(4, clusterWithStandby)) - Expect(testStdout).To(gbytes.Say("Preparing PXF on master host, standby master host, and 2 segment hosts")) + Expect(testStdout).To(gbytes.Say("Preparing PXF on coordinator host, standby coordinator host, and 2 segment hosts")) cmd.GenerateStatusReport(&cmd.MigrateCommand, createClusterData(4, clusterWithStandby)) - Expect(testStdout).To(gbytes.Say("Migrating PXF configuration on master host, standby master host, and 2 segment hosts")) + Expect(testStdout).To(gbytes.Say("Migrating PXF configuration on coordinator host, standby coordinator host, and 2 segment hosts")) }) It("reports segment hosts are starting, stopping, restarting and statusing", func() { cmd.GenerateStatusReport(&cmd.StartCommand, createClusterData(4, clusterWithStandby)) - Expect(testStdout).To(gbytes.Say("Starting PXF on master host, standby master host, and 2 segment hosts")) + Expect(testStdout).To(gbytes.Say("Starting PXF on coordinator host, standby coordinator host, and 2 segment hosts")) cmd.GenerateStatusReport(&cmd.StopCommand, createClusterData(4, clusterWithStandby)) - Expect(testStdout).To(gbytes.Say("Stopping PXF on master host, standby master host, and 2 segment hosts")) + Expect(testStdout).To(gbytes.Say("Stopping PXF on coordinator host, standby coordinator host, and 2 segment hosts")) cmd.GenerateStatusReport(&cmd.RestartCommand, createClusterData(4, clusterWithStandby)) - Expect(testStdout).To(gbytes.Say("Restarting PXF on master host, standby master host, and 2 segment hosts")) + Expect(testStdout).To(gbytes.Say("Restarting PXF on coordinator host, standby coordinator host, and 2 segment hosts")) cmd.GenerateStatusReport(&cmd.StatusCommand, createClusterData(4, clusterWithStandby)) - Expect(testStdout).To(gbytes.Say("Checking status of PXF servers on master host, standby master host, and 2 segment hosts")) + Expect(testStdout).To(gbytes.Say("Checking status of PXF servers on coordinator host, standby coordinator host, and 2 segment hosts")) }) }) - Context("When there is a standby master on a segment host", func() { - It("reports master host and segment hosts are initializing, resetting, registering, syncing, preparing, and migrating", func() { + Context("When there is a standby coordinator on a segment host", func() { + It("reports coordinator host and segment hosts are initializing, resetting, registering, syncing, preparing, and migrating", func() { cmd.GenerateStatusReport(&cmd.InitCommand, createClusterData(3, clusterWithStandbyOnSegHost)) - Expect(testStdout).To(gbytes.Say("Initializing PXF on master host and 2 segment hosts")) + Expect(testStdout).To(gbytes.Say("Initializing PXF on coordinator host and 2 segment hosts")) cmd.GenerateStatusReport(&cmd.ResetCommand, createClusterData(3, clusterWithStandbyOnSegHost)) - Expect(testStdout).To(gbytes.Say("Resetting PXF on master host and 2 segment hosts")) + Expect(testStdout).To(gbytes.Say("Resetting PXF on coordinator host and 2 segment hosts")) cmd.GenerateStatusReport(&cmd.SyncCommand, createClusterData(2, clusterWithStandbyOnSegHost)) - Expect(testStdout).To(gbytes.Say("Syncing PXF configuration files from master host to 2 segment hosts")) + Expect(testStdout).To(gbytes.Say("Syncing PXF configuration files from coordinator host to 2 segment hosts")) cmd.GenerateStatusReport(&cmd.RegisterCommand, createClusterData(3, clusterWithStandbyOnSegHost)) - Expect(testStdout).To(gbytes.Say("Installing PXF extension on master host and 2 segment hosts")) + Expect(testStdout).To(gbytes.Say("Installing PXF extension on coordinator host and 2 segment hosts")) cmd.GenerateStatusReport(&cmd.PrepareCommand, createClusterData(3, clusterWithStandbyOnSegHost)) - Expect(testStdout).To(gbytes.Say("Preparing PXF on master host and 2 segment hosts")) + Expect(testStdout).To(gbytes.Say("Preparing PXF on coordinator host and 2 segment hosts")) cmd.GenerateStatusReport(&cmd.MigrateCommand, createClusterData(3, clusterWithStandbyOnSegHost)) - Expect(testStdout).To(gbytes.Say("Migrating PXF configuration on master host and 2 segment hosts")) + Expect(testStdout).To(gbytes.Say("Migrating PXF configuration on coordinator host and 2 segment hosts")) }) It("reports segment hosts are starting, stopping, restarting and statusing", func() { cmd.GenerateStatusReport(&cmd.StartCommand, createClusterData(3, clusterWithStandbyOnSegHost)) - Expect(testStdout).To(gbytes.Say("Starting PXF on master host and 2 segment hosts")) + Expect(testStdout).To(gbytes.Say("Starting PXF on coordinator host and 2 segment hosts")) cmd.GenerateStatusReport(&cmd.StopCommand, createClusterData(3, clusterWithStandbyOnSegHost)) - Expect(testStdout).To(gbytes.Say("Stopping PXF on master host and 2 segment hosts")) + Expect(testStdout).To(gbytes.Say("Stopping PXF on coordinator host and 2 segment hosts")) cmd.GenerateStatusReport(&cmd.RestartCommand, createClusterData(3, clusterWithStandbyOnSegHost)) - Expect(testStdout).To(gbytes.Say("Restarting PXF on master host and 2 segment hosts")) + Expect(testStdout).To(gbytes.Say("Restarting PXF on coordinator host and 2 segment hosts")) cmd.GenerateStatusReport(&cmd.StatusCommand, createClusterData(3, clusterWithStandbyOnSegHost)) - Expect(testStdout).To(gbytes.Say("Checking status of PXF servers on master host and 2 segment hosts")) + Expect(testStdout).To(gbytes.Say("Checking status of PXF servers on coordinator host and 2 segment hosts")) }) }) Context("When there is only one segment host", func() { - It("reports master host and segment host are initializing, resetting, registering, syncing, preparing, and migrating", func() { + It("reports coordinator host and segment host are initializing, resetting, registering, syncing, preparing, and migrating", func() { cmd.GenerateStatusReport(&cmd.InitCommand, createClusterData(2, clusterWithOneSeg)) - Expect(testStdout).To(gbytes.Say("Initializing PXF on master host and 1 segment host")) + Expect(testStdout).To(gbytes.Say("Initializing PXF on coordinator host and 1 segment host")) cmd.GenerateStatusReport(&cmd.ResetCommand, createClusterData(2, clusterWithOneSeg)) - Expect(testStdout).To(gbytes.Say("Resetting PXF on master host and 1 segment host")) + Expect(testStdout).To(gbytes.Say("Resetting PXF on coordinator host and 1 segment host")) cmd.GenerateStatusReport(&cmd.SyncCommand, createClusterData(1, clusterWithOneSeg)) - Expect(testStdout).To(gbytes.Say("Syncing PXF configuration files from master host to 1 segment host")) + Expect(testStdout).To(gbytes.Say("Syncing PXF configuration files from coordinator host to 1 segment host")) cmd.GenerateStatusReport(&cmd.RegisterCommand, createClusterData(2, clusterWithOneSeg)) - Expect(testStdout).To(gbytes.Say("Installing PXF extension on master host and 1 segment host")) + Expect(testStdout).To(gbytes.Say("Installing PXF extension on coordinator host and 1 segment host")) cmd.GenerateStatusReport(&cmd.PrepareCommand, createClusterData(2, clusterWithOneSeg)) - Expect(testStdout).To(gbytes.Say("Preparing PXF on master host and 1 segment host")) + Expect(testStdout).To(gbytes.Say("Preparing PXF on coordinator host and 1 segment host")) cmd.GenerateStatusReport(&cmd.MigrateCommand, createClusterData(2, clusterWithOneSeg)) - Expect(testStdout).To(gbytes.Say("Migrating PXF configuration on master host and 1 segment host")) + Expect(testStdout).To(gbytes.Say("Migrating PXF configuration on coordinator host and 1 segment host")) }) It("reports segment hosts are starting, stopping, restarting and statusing", func() { cmd.GenerateStatusReport(&cmd.StartCommand, createClusterData(2, clusterWithOneSeg)) - Expect(testStdout).To(gbytes.Say("Starting PXF on master host and 1 segment host")) + Expect(testStdout).To(gbytes.Say("Starting PXF on coordinator host and 1 segment host")) cmd.GenerateStatusReport(&cmd.StopCommand, createClusterData(2, clusterWithOneSeg)) - Expect(testStdout).To(gbytes.Say("Stopping PXF on master host and 1 segment host")) + Expect(testStdout).To(gbytes.Say("Stopping PXF on coordinator host and 1 segment host")) cmd.GenerateStatusReport(&cmd.RestartCommand, createClusterData(2, clusterWithOneSeg)) - Expect(testStdout).To(gbytes.Say("Restarting PXF on master host and 1 segment host")) + Expect(testStdout).To(gbytes.Say("Restarting PXF on coordinator host and 1 segment host")) cmd.GenerateStatusReport(&cmd.StatusCommand, createClusterData(2, clusterWithOneSeg)) - Expect(testStdout).To(gbytes.Say("Checking status of PXF servers on master host and 1 segment host")) + Expect(testStdout).To(gbytes.Say("Checking status of PXF servers on coordinator host and 1 segment host")) }) }) }) diff --git a/cli/cmd/pxf.go b/cli/cmd/pxf.go index e0fa0da773..e7850ed7a0 100644 --- a/cli/cmd/pxf.go +++ b/cli/cmd/pxf.go @@ -143,8 +143,8 @@ var ( "* Use the \"pxf cluster register\" command instead.\n" + "*\n" + "*****************************************************************************\n\n" + - "Initializing PXF on master host%s and %d segment host%s...\n", - standby: ", standby master host,", + "Initializing PXF on coordinator host%s and %d segment host%s...\n", + standby: ", standby coordinator host,", err: "PXF failed to initialize on %d out of %d host%s\n", }, warn: false, @@ -155,8 +155,8 @@ var ( name: start, messages: map[messageType]string{ success: "PXF started successfully on %d out of %d host%s\n", - status: "Starting PXF on master host%s and %d segment host%s...\n", - standby: ", standby master host,", + status: "Starting PXF on coordinator host%s and %d segment host%s...\n", + standby: ", standby coordinator host,", err: "PXF failed to start on %d out of %d host%s\n", }, warn: false, @@ -167,8 +167,8 @@ var ( name: stop, messages: map[messageType]string{ success: "PXF stopped successfully on %d out of %d host%s\n", - status: "Stopping PXF on master host%s and %d segment host%s...\n", - standby: ", standby master host,", + status: "Stopping PXF on coordinator host%s and %d segment host%s...\n", + standby: ", standby coordinator host,", err: "PXF failed to stop on %d out of %d host%s\n", }, warn: false, @@ -179,23 +179,23 @@ var ( name: sync, messages: map[messageType]string{ success: "PXF configs synced successfully on %d out of %d host%s\n", - status: "Syncing PXF configuration files from master host to%s %d segment host%s...\n", - standby: " standby master host and", + status: "Syncing PXF configuration files from coordinator host to%s %d segment host%s...\n", + standby: " standby coordinator host and", err: "PXF configs failed to sync on %d out of %d host%s\n", }, warn: false, envVars: []envVar{pxfBase}, - // cluster.ON_LOCAL | cluster.ON_HOSTS: the command will target host%s, but be run from master - // this is ideal for copying files from master to segment host(s) using rsync. - // since the files are already on master, we exclude master but include standby master + // cluster.ON_LOCAL | cluster.ON_HOSTS: the command will target host%s, but be run from coordinator + // this is ideal for copying files from coordinator to segment host(s) using rsync. + // since the files are already on coordinator, we exclude coordinator but include standby coordinator whereToRun: cluster.ON_LOCAL | cluster.ON_HOSTS | cluster.EXCLUDE_MASTER | cluster.INCLUDE_MIRRORS, } StatusCommand = command{ name: statuses, messages: map[messageType]string{ success: "PXF is running on %d out of %d host%s\n", - status: "Checking status of PXF servers on master host%s and %d segment host%s...\n", - standby: ", standby master host,", + status: "Checking status of PXF servers on coordinator host%s and %d segment host%s...\n", + standby: ", standby coordinator host,", err: "PXF is not running on %d out of %d host%s\n", }, warn: false, @@ -206,8 +206,8 @@ var ( name: register, messages: map[messageType]string{ success: "PXF extension has been installed on %d out of %d host%s\n", - status: "Installing PXF extension on master host%s and %d segment host%s...\n", - standby: ", standby master host,", + status: "Installing PXF extension on coordinator host%s and %d segment host%s...\n", + standby: ", standby coordinator host,", err: "Failed to install PXF extension on %d out of %d host%s\n", }, warn: false, @@ -223,8 +223,8 @@ var ( "* The \"pxf cluster reset\" command is deprecated and will be removed\n" + "* in a future release of PXF.\n" + "*****************************************************************************\n\n" + - "Resetting PXF on master host%s and %d segment host%s...\n", - standby: ", standby master host,", + "Resetting PXF on coordinator host%s and %d segment host%s...\n", + standby: ", standby coordinator host,", err: "Failed to reset PXF on %d out of %d host%s\n", }, warn: false, @@ -235,8 +235,8 @@ var ( name: restart, messages: map[messageType]string{ success: "PXF restarted successfully on %d out of %d host%s\n", - status: "Restarting PXF on master host%s and %d segment host%s...\n", - standby: ", standby master host,", + status: "Restarting PXF on coordinator host%s and %d segment host%s...\n", + standby: ", standby coordinator host,", err: "PXF failed to restart on %d out of %d host%s\n", }, warn: false, @@ -247,8 +247,8 @@ var ( name: prepare, messages: map[messageType]string{ success: "PXF prepared successfully on %d out of %d host%s\n", - status: "Preparing PXF on master host%s and %d segment host%s...\n", - standby: ", standby master host,", + status: "Preparing PXF on coordinator host%s and %d segment host%s...\n", + standby: ", standby coordinator host,", err: "PXF failed to prepare on %d out of %d host%s\n", }, warn: false, @@ -259,8 +259,8 @@ var ( name: migrate, messages: map[messageType]string{ success: "PXF configuration migrated successfully on %d out of %d host%s\n", - status: "Migrating PXF configuration on master host%s and %d segment host%s...\n", - standby: ", standby master host,", + status: "Migrating PXF configuration on coordinator host%s and %d segment host%s...\n", + standby: ", standby coordinator host,", err: "PXF failed to migrate configuration on %d out of %d host%s\n", }, warn: false, diff --git a/concourse/scripts/cli/common.sh b/concourse/scripts/cli/common.sh index e8422c2b4b..fa3cd7c9fe 100644 --- a/concourse/scripts/cli/common.sh +++ b/concourse/scripts/cli/common.sh @@ -143,9 +143,9 @@ has_standby_coordinator() { } get_cluster_description() { - local cluster_description="master host" + local cluster_description="coordinator host" if has_standby_coordinator; then - cluster_description+=", standby master host," + cluster_description+=", standby coordinator host," fi local num_segment_hosts num_segment_hosts="$(grep -c -P 'sdw\d+' hostfile_all)" @@ -155,9 +155,9 @@ get_cluster_description() { } get_cluster_sync_description() { - local cluster_sync_description="master host to " + local cluster_sync_description="coordinator host to " if has_standby_coordinator; then - cluster_sync_description+="standby master host and " + cluster_sync_description+="standby coordinator host and " fi local num_segment_hosts num_segment_hosts="$(grep -c -P 'sdw\d+' hostfile_all)" diff --git a/concourse/scripts/cli/test_sync.sh b/concourse/scripts/cli/test_sync.sh index cdfa723c18..93a5c28c43 100755 --- a/concourse/scripts/cli/test_sync.sh +++ b/concourse/scripts/cli/test_sync.sh @@ -264,7 +264,7 @@ run_test test_sync_succeeds_delete_server "pxf cluster sync (delete server) shou # === Test "pxf cluster sync (no standby)" ================================================================ expected_sync_message=\ -"Syncing PXF configuration files from master host to 2 segment hosts... +"Syncing PXF configuration files from coordinator host to 2 segment hosts... PXF configs synced successfully on 2 out of 2 hosts" expected_cluster_configs=\ "cdw: diff --git a/external-table/src/pxfuriparser.c b/external-table/src/pxfuriparser.c index a27a986853..47fce006ce 100644 --- a/external-table/src/pxfuriparser.c +++ b/external-table/src/pxfuriparser.c @@ -179,7 +179,7 @@ GPHDUri_parse_options(GPHDUri *uri, char **cursor) /* skip '?' */ start++; - /* sanity check */ + /* confidence check */ if (strlen(start) < 2) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), diff --git a/server/pxf-service/src/main/java/org/greenplum/pxf/service/profile/Profile.java b/server/pxf-service/src/main/java/org/greenplum/pxf/service/profile/Profile.java index f45ca437e6..c565fa080d 100644 --- a/server/pxf-service/src/main/java/org/greenplum/pxf/service/profile/Profile.java +++ b/server/pxf-service/src/main/java/org/greenplum/pxf/service/profile/Profile.java @@ -206,7 +206,7 @@ String getOutputFormat() { } /** - * A mapping defines a whitelisted option that is allowed + * A mapping defines an option that is allowed * for the given profile. The option maps to a property that * can be interpreted by the Profile Plugins. */ @@ -221,9 +221,9 @@ static class Mapping { private String property; /** - * Returns the whitelisted option + * Returns the allowlisted option * - * @return the whitelisted option + * @return the allowlisted option */ String getOption() { return option; diff --git a/server/pxf-service/src/test/java/org/greenplum/pxf/service/HttpRequestParserTest.java b/server/pxf-service/src/test/java/org/greenplum/pxf/service/HttpRequestParserTest.java index 2000a04e0a..037950e573 100644 --- a/server/pxf-service/src/test/java/org/greenplum/pxf/service/HttpRequestParserTest.java +++ b/server/pxf-service/src/test/java/org/greenplum/pxf/service/HttpRequestParserTest.java @@ -466,7 +466,7 @@ public void profileIsSetInLowerCase() { } @Test - public void whitelistedOptionsAreAddedAsProperties() { + public void allowlistOptionsAreAddedAsProperties() { parameters.set("X-GP-OPTIONS-PROFILE", "test-profile"); parameters.set("X-GP-OPTIONS-CONFIGPROP1", "config-prop-value1"); parameters.set("X-GP-OPTIONS-CONFIGPROP3", "config-prop-value3"); From 09c3f50120c666da5896145be078e6b35164b75c Mon Sep 17 00:00:00 2001 From: Ashuka Xue Date: Tue, 25 Jul 2023 15:07:05 -0700 Subject: [PATCH 08/24] Refactor cloudbuild (#1008) This commit creates a script that downloads the specified version of maven-3. You can also specify `latest` to grab the latest maven-3 version. This commit also refactors the cloudbuild to pull the maven tarball from GCS instead of from upstream Apache. This is to prevent future potential pipeline issues. Additionally, we do not need the latest build of maven to run automation tests. --- concourse/README.md | 25 ++++++++++ concourse/docker/pxf-dev-base/README.md | 4 +- concourse/docker/pxf-dev-base/cloudbuild.yaml | 18 +++++--- .../pxf-dev-base/gpdb6/centos7/Dockerfile | 10 ++-- .../docker/pxf-dev-base/gpdb6/oel7/Dockerfile | 10 ++-- .../pxf-dev-base/gpdb6/rocky8/Dockerfile | 10 ++-- .../pxf-dev-base/gpdb6/ubuntu18.04/Dockerfile | 11 ++--- .../pxf-dev-base/gpdb7/centos7/Dockerfile | 10 ++-- .../pxf-dev-base/gpdb7/rhel8/Dockerfile | 10 ++-- .../pxf-dev-base/gpdb7/rocky8/Dockerfile | 10 ++-- .../pxf-dev-base/gpdb7/ubuntu18.04/Dockerfile | 12 ++--- .../download-maven-from-apache-mirror.sh | 46 +++++++++++++++++++ 12 files changed, 111 insertions(+), 65 deletions(-) create mode 100755 concourse/scripts/download-maven-from-apache-mirror.sh diff --git a/concourse/README.md b/concourse/README.md index abf8923ff5..1143052d1b 100644 --- a/concourse/README.md +++ b/concourse/README.md @@ -113,3 +113,28 @@ it needs to be cleaned manually and so do the dataproc clusters. ```shell YOUR_TAG= make -C "${HOME}/workspace/pxf/concourse" longevity ``` + +## Uploading a new Apache Maven 3 version + +The CI pipelines for PXF run automation tests using Apache Maven 3.x. Instead of downloading this directly from the Apache +mirrors or Apache archive, we store a copy in Google Cloud Storage to use when we create our images in Cloudbuild. +Typically, we will not be updating these values very often. However, if we need to upload a new version of Maven, you +can use a snippet like this one to download and then upload to GCS. + +```bash +./scripts/download-maven-from-apache-mirror.sh +gcloud storage cp ../downloads/apache-maven--bin.tar.gz gs://data-gpdb-ud-pxf-build-resources/apache-maven + +# Example for Apache Maven 3.9.2 +./scripts/download-spark-from-apache-mirror.sh 3.9.2 +gcloud storage cp ../downloads/apache-maven-3.9.2-bin.tar.gz gs://data-gpdb-ud-pxf-build-resources/apache-maven + +# Example for Apache Maven 3 Latest +$ ./scripts/download-spark-from-apache-mirror.sh latest +> Looking for latest maven-3 version... +> Latest maven version determined to be: 3.9.3 +> Would you like to proceed (y/n)? y + +gcloud storage cp ../downloads/apache-maven-3.9.3-bin.tar.gz gs://data-gpdb-ud-pxf-build-resources/apache-maven + +``` diff --git a/concourse/docker/pxf-dev-base/README.md b/concourse/docker/pxf-dev-base/README.md index c064f34a06..f797657679 100644 --- a/concourse/docker/pxf-dev-base/README.md +++ b/concourse/docker/pxf-dev-base/README.md @@ -16,12 +16,12 @@ You can run the entire cloudbuild using Google Cloud Build by doing the followin cd ~/workspace/pxf gcloud builds submit . --config=concourse/docker/pxf-dev-base/cloudbuild.yaml \ - --substitutions=_BASE_IMAGE_REPOSITORY=gcr.io/data-gpdb-public-images,_PRIVATE_BASE_IMAGE_REPOSITORY=gcr.io/data-gpdb-private-images,COMMIT_SHA=dev-build- + --substitutions=_BASE_IMAGE_REPOSITORY=gcr.io/data-gpdb-public-images,COMMIT_SHA=dev-build- -- or if you would like to modify the go and ginkgo versions, you can do so by doing the following -- gcloud builds submit . --config=concourse/docker/pxf-dev-base/cloudbuild.yaml \ ---substitutions=_BASE_IMAGE_REPOSITORY=gcr.io/data-gpdb-public-images,_PRIVATE_BASE_IMAGE_REPOSITORY=gcr.io/data-gpdb-private-images,_GO_VERSION=,COMMIT_SHA=dev-build- +--substitutions=_BASE_IMAGE_REPOSITORY=gcr.io/data-gpdb-public-images,_GO_VERSION=,_MAVEN_VERSION=,COMMIT_SHA=dev-build- ``` This guide assumes the PXF repository lives under the `~/workspace/pxf` diff --git a/concourse/docker/pxf-dev-base/cloudbuild.yaml b/concourse/docker/pxf-dev-base/cloudbuild.yaml index a63b12fb80..74af725570 100644 --- a/concourse/docker/pxf-dev-base/cloudbuild.yaml +++ b/concourse/docker/pxf-dev-base/cloudbuild.yaml @@ -10,6 +10,11 @@ options: steps: +- name: 'gcr.io/cloud-builders/gsutil' + id: apache-maven-binaries + args: ['cp', 'gs://${_PXF_BUILD_BUCKET}/apache-maven/apache-maven-${_MAVEN_VERSION}-bin.tar.gz', 'build/apache-maven.tar.gz'] + waitFor: ['-'] + ############################################################################## # GPDB 5 Images ############################################################################## @@ -38,6 +43,7 @@ steps: - 'concourse/docker/pxf-dev-base/gpdb5/centos7/Dockerfile' - '/workspace/build/' waitFor: + - apache-maven-binaries - gpdb5-centos7-test-pxf-image-cache ############################################################################## @@ -60,7 +66,6 @@ steps: - 'build' - '--build-arg=BASE_IMAGE=${_BASE_IMAGE_REPOSITORY}/gpdb6-centos7-test:latest' - '--build-arg=GO_VERSION=${_GO_VERSION}' - - '--build-arg=MAVEN_VERSION=${_MAVEN_VERSION}' - '--tag=gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb6-centos7-test-pxf:$COMMIT_SHA' - '--cache-from' - 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb6-centos7-test-pxf:latest' @@ -69,6 +74,7 @@ steps: - '/workspace/build/' waitFor: - gpdb6-centos7-test-pxf-image-cache + - apache-maven-binaries - name: 'gcr.io/cloud-builders/docker' id: gpdb6-rocky8-test-pxf-image-cache @@ -86,7 +92,6 @@ steps: - 'build' - '--build-arg=BASE_IMAGE=${_BASE_IMAGE_REPOSITORY}/gpdb6-rocky8-test:latest' - '--build-arg=GO_VERSION=${_GO_VERSION}' - - '--build-arg=MAVEN_VERSION=${_MAVEN_VERSION}' - '--tag=gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb6-rocky8-test-pxf:$COMMIT_SHA' - '--cache-from' - 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb6-rocky8-test-pxf:latest' @@ -94,6 +99,7 @@ steps: - 'concourse/docker/pxf-dev-base/gpdb6/rocky8/Dockerfile' - '/workspace/build/' waitFor: + - apache-maven-binaries - gpdb6-rocky8-test-pxf-image-cache - name: 'gcr.io/cloud-builders/docker' @@ -112,7 +118,6 @@ steps: - 'build' - '--build-arg=BASE_IMAGE=${_BASE_IMAGE_REPOSITORY}/gpdb6-ubuntu18.04-test:latest' - '--build-arg=GO_VERSION=${_GO_VERSION}' - - '--build-arg=MAVEN_VERSION=${_MAVEN_VERSION}' - '--tag=gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb6-ubuntu18.04-test-pxf:$COMMIT_SHA' - '--cache-from' - 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb6-ubuntu18.04-test-pxf:latest' @@ -120,6 +125,7 @@ steps: - 'concourse/docker/pxf-dev-base/gpdb6/ubuntu18.04/Dockerfile' - '/workspace/build/' waitFor: + - apache-maven-binaries - gpdb6-ubuntu18.04-test-pxf-image-cache # An image for gpdb6 running on OEL7 @@ -139,7 +145,6 @@ steps: - 'build' - '--build-arg=BASE_IMAGE=${_BASE_IMAGE_REPOSITORY}/gpdb6-oel7-test:latest' - '--build-arg=GO_VERSION=${_GO_VERSION}' - - '--build-arg=MAVEN_VERSION=${_MAVEN_VERSION}' - '--tag=gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb6-oel7-test-pxf:$COMMIT_SHA' - '--cache-from' - 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb6-oel7-test-pxf:latest' @@ -147,6 +152,7 @@ steps: - 'concourse/docker/pxf-dev-base/gpdb6/oel7/Dockerfile' - '/workspace/build/' waitFor: + - apache-maven-binaries - gpdb6-oel7-test-pxf-image-cache ############################################################################## @@ -170,7 +176,6 @@ steps: - 'build' - '--build-arg=BASE_IMAGE=${_BASE_IMAGE_REPOSITORY}/gpdb7-rocky8-test:latest' - '--build-arg=GO_VERSION=${_GO_VERSION}' - - '--build-arg=MAVEN_VERSION=${_MAVEN_VERSION}' - '--tag=gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-rocky8-test-pxf:$COMMIT_SHA' - '--cache-from' - 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-rocky8-test-pxf:latest' @@ -178,11 +183,12 @@ steps: - 'concourse/docker/pxf-dev-base/gpdb7/rocky8/Dockerfile' - '/workspace/build/' waitFor: + - apache-maven-binaries - gpdb7-rocky8-test-pxf-image-cache substitutions: _GO_VERSION: '1.19.6' # default values - _MAVEN_VERSION: '3.9.2' # default values + _MAVEN_VERSION: '3.9.3' # default values # Push images to Cloud Build to Container Registry images: diff --git a/concourse/docker/pxf-dev-base/gpdb6/centos7/Dockerfile b/concourse/docker/pxf-dev-base/gpdb6/centos7/Dockerfile index a485b47f5d..449dc8fac0 100644 --- a/concourse/docker/pxf-dev-base/gpdb6/centos7/Dockerfile +++ b/concourse/docker/pxf-dev-base/gpdb6/centos7/Dockerfile @@ -4,6 +4,8 @@ FROM ${BASE_IMAGE} ARG GO_VERSION +ADD apache-maven.tar.gz /usr/share + # install Go utilities RUN mkdir -p /tmp/pxf_src/ && cd /tmp \ && wget -O go.tgz -q https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz \ @@ -24,16 +26,10 @@ RUN useradd -s /sbin/nologin -d /opt/minio minio \ && chmod +x /opt/minio/bin/minio \ && chown -R minio:minio /opt/minio -ARG MAVEN_VERSION ARG USER_HOME_DIR="/root" -ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries # install dependencies that are missing on the base images -RUN curl -fsSL -o /tmp/apache-maven.tar.gz ${BASE_URL}/apache-maven-${MAVEN_VERSION}-bin.tar.gz \ - && mkdir -p /usr/share/maven \ - && tar -xzf /tmp/apache-maven.tar.gz -C /usr/share/maven --strip-components=1 \ - && rm -f /tmp/apache-maven.tar.gz \ - && ln -s /usr/share/maven/bin/mvn /usr/bin/mvn \ +RUN ln -s /usr/share/apache-maven-*/bin/mvn /usr/bin/mvn \ && yum install -y rpm-build python-devel jq sudo && yum clean all \ && cd /tmp && /usr/bin/pip install --upgrade pip==20.3.3 \ && /usr/bin/pip install psi paramiko --no-cache-dir diff --git a/concourse/docker/pxf-dev-base/gpdb6/oel7/Dockerfile b/concourse/docker/pxf-dev-base/gpdb6/oel7/Dockerfile index a27a00e8c3..edbdde7981 100644 --- a/concourse/docker/pxf-dev-base/gpdb6/oel7/Dockerfile +++ b/concourse/docker/pxf-dev-base/gpdb6/oel7/Dockerfile @@ -4,22 +4,18 @@ FROM ${BASE_IMAGE} ARG GO_VERSION +ADD apache-maven.tar.gz /usr/share + # install Go utilities RUN mkdir -p /tmp/pxf_src/ && cd /tmp \ && wget -O go.tgz -q https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz \ && rm -rf /usr/local/go && tar -C /usr/local -xzf go.tgz && rm go.tgz -ARG MAVEN_VERSION ARG USER_HOME_DIR="/root" -ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries # install dependencies that are missing on the base images # glibc-common is for adding locales -RUN curl -fsSL -o /tmp/apache-maven.tar.gz ${BASE_URL}/apache-maven-${MAVEN_VERSION}-bin.tar.gz \ - && mkdir -p /usr/share/maven \ - && tar -xzf /tmp/apache-maven.tar.gz -C /usr/share/maven --strip-components=1 \ - && rm -f /tmp/apache-maven.tar.gz \ - && ln -s /usr/share/maven/bin/mvn /usr/bin/mvn \ +RUN ln -s /usr/share/apache-maven-*/bin/mvn /usr/bin/mvn \ # explicitly reinstall glibc-common to add back deleted charmaps && yum reinstall -y glibc-common \ && yum install -y rpm-build python-devel jq sudo && yum clean all \ diff --git a/concourse/docker/pxf-dev-base/gpdb6/rocky8/Dockerfile b/concourse/docker/pxf-dev-base/gpdb6/rocky8/Dockerfile index aa42adcf48..72a020088c 100644 --- a/concourse/docker/pxf-dev-base/gpdb6/rocky8/Dockerfile +++ b/concourse/docker/pxf-dev-base/gpdb6/rocky8/Dockerfile @@ -4,6 +4,8 @@ FROM ${BASE_IMAGE} ARG GO_VERSION +ADD apache-maven.tar.gz /usr/share + # install Go utilities RUN mkdir -p /tmp/pxf_src/ && cd /tmp \ && wget -O go.tgz -q https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz \ @@ -24,16 +26,10 @@ RUN useradd -s /sbin/nologin -d /opt/minio minio \ && chmod +x /opt/minio/bin/minio \ && chown -R minio:minio /opt/minio -ARG MAVEN_VERSION ARG USER_HOME_DIR="/root" -ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries # install dependencies that are missing on the base images -RUN curl -fsSL -o /tmp/apache-maven.tar.gz ${BASE_URL}/apache-maven-${MAVEN_VERSION}-bin.tar.gz \ - && mkdir -p /usr/share/maven \ - && tar -xzf /tmp/apache-maven.tar.gz -C /usr/share/maven --strip-components=1 \ - && rm -f /tmp/apache-maven.tar.gz \ - && ln -s /usr/share/maven/bin/mvn /usr/bin/mvn \ +RUN ln -s /usr/share/apache-maven-*/bin/mvn /usr/bin/mvn \ && /usr/bin/pip2 install paramiko --no-cache-dir # create rsa key for the root user diff --git a/concourse/docker/pxf-dev-base/gpdb6/ubuntu18.04/Dockerfile b/concourse/docker/pxf-dev-base/gpdb6/ubuntu18.04/Dockerfile index 289d6dec58..fc55e3e574 100644 --- a/concourse/docker/pxf-dev-base/gpdb6/ubuntu18.04/Dockerfile +++ b/concourse/docker/pxf-dev-base/gpdb6/ubuntu18.04/Dockerfile @@ -4,6 +4,8 @@ FROM ${BASE_IMAGE} ARG GO_VERSION +ADD apache-maven.tar.gz /usr/share + # install Go utilities RUN mkdir -p /tmp/pxf_src/ && cd /tmp \ && wget -O go.tgz -q https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz \ @@ -12,19 +14,14 @@ RUN mkdir -p /tmp/pxf_src/ && cd /tmp \ # install dependencies that are missing on the base images # install a specific version of perl for tinc -ARG MAVEN_VERSION ARG USER_HOME_DIR="/root" -ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries RUN apt-get update -y \ && DEBIAN_FRONTEND=noninteractive apt-get install -y python-dev curl sudo jq \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ - && mkdir -p /usr/share/maven /usr/share/maven/ref \ - && curl -fsSL -o /tmp/apache-maven.tar.gz ${BASE_URL}/apache-maven-${MAVEN_VERSION}-bin.tar.gz \ - && tar -xzf /tmp/apache-maven.tar.gz -C /usr/share/maven --strip-components=1 \ - && rm -f /tmp/apache-maven.tar.gz \ - && ln -s /usr/share/maven/bin/mvn /usr/bin/mvn + && mkdir -p /usr/share/apache-maven-*/ref \ + && ln -s /usr/share/apache-maven-*/bin/mvn /usr/bin/mvn # create user gpadmin since GPDB cannot run under root RUN locale-gen en_US.UTF-8 \ diff --git a/concourse/docker/pxf-dev-base/gpdb7/centos7/Dockerfile b/concourse/docker/pxf-dev-base/gpdb7/centos7/Dockerfile index ca869cdd58..73c35b20fa 100644 --- a/concourse/docker/pxf-dev-base/gpdb7/centos7/Dockerfile +++ b/concourse/docker/pxf-dev-base/gpdb7/centos7/Dockerfile @@ -4,6 +4,8 @@ FROM ${BASE_IMAGE} ARG GO_VERSION +ADD apache-maven.tar.gz /usr/share + # install Go utilities RUN mkdir -p /tmp/pxf_src/ && cd /tmp \ && wget -O go.tgz -q https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz \ @@ -18,16 +20,10 @@ RUN useradd -s /sbin/nologin -d /opt/minio minio \ && chmod +x /opt/minio/bin/minio \ && chown -R minio:minio /opt/minio -ARG MAVEN_VERSION ARG USER_HOME_DIR="/root" -ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries # install dependencies that are missing on the base images -RUN curl -fsSL -o /tmp/apache-maven.tar.gz ${BASE_URL}/apache-maven-${MAVEN_VERSION}-bin.tar.gz \ - && mkdir -p /usr/share/maven \ - && tar -xzf /tmp/apache-maven.tar.gz -C /usr/share/maven --strip-components=1 \ - && rm -f /tmp/apache-maven.tar.gz \ - && ln -s /usr/share/maven/bin/mvn /usr/bin/mvn \ +RUN ln -s /usr/share/apache-maven-*/bin/mvn /usr/bin/mvn \ && yum install -y rpm-build python-devel jq sudo java-1.8.0-openjdk-devel java-11-openjdk-devel && yum clean all \ && cd /tmp && /usr/bin/pip install --upgrade pip==20.3.3 \ && /usr/bin/pip install paramiko --no-cache-dir diff --git a/concourse/docker/pxf-dev-base/gpdb7/rhel8/Dockerfile b/concourse/docker/pxf-dev-base/gpdb7/rhel8/Dockerfile index 619d69a180..8fae857631 100644 --- a/concourse/docker/pxf-dev-base/gpdb7/rhel8/Dockerfile +++ b/concourse/docker/pxf-dev-base/gpdb7/rhel8/Dockerfile @@ -4,6 +4,8 @@ FROM ${BASE_IMAGE} ARG GO_VERSION +ADD apache-maven.tar.gz /usr/share + # install Go utilities RUN mkdir -p /tmp/pxf_src/ && cd /tmp \ && wget -O go.tgz -q https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz \ @@ -18,16 +20,10 @@ RUN useradd -s /sbin/nologin -d /opt/minio minio \ && chmod +x /opt/minio/bin/minio \ && chown -R minio:minio /opt/minio -ARG MAVEN_VERSION ARG USER_HOME_DIR="/root" -ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries # install dependencies that are missing on the base images -RUN curl -fsSL -o /tmp/apache-maven.tar.gz ${BASE_URL}/apache-maven-${MAVEN_VERSION}-bin.tar.gz \ - && mkdir -p /usr/share/maven \ - && tar -xzf /tmp/apache-maven.tar.gz -C /usr/share/maven --strip-components=1 \ - && rm -f /tmp/apache-maven.tar.gz \ - && ln -s /usr/share/maven/bin/mvn /usr/bin/mvn \ +RUN ln -s /usr/share/apache-maven-*/bin/mvn /usr/bin/mvn \ && dnf install -y -d1 python2 python2-pip \ && /usr/bin/pip2 install paramiko --no-cache-dir diff --git a/concourse/docker/pxf-dev-base/gpdb7/rocky8/Dockerfile b/concourse/docker/pxf-dev-base/gpdb7/rocky8/Dockerfile index 6f732bba43..27f3c66e20 100644 --- a/concourse/docker/pxf-dev-base/gpdb7/rocky8/Dockerfile +++ b/concourse/docker/pxf-dev-base/gpdb7/rocky8/Dockerfile @@ -4,6 +4,8 @@ FROM ${BASE_IMAGE} ARG GO_VERSION +ADD apache-maven.tar.gz /usr/share + # install Go utilities RUN mkdir -p /tmp/pxf_src/ && cd /tmp \ && wget -O go.tgz -q https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz \ @@ -18,16 +20,10 @@ RUN useradd -s /sbin/nologin -d /opt/minio minio \ && chmod +x /opt/minio/bin/minio \ && chown -R minio:minio /opt/minio -ARG MAVEN_VERSION ARG USER_HOME_DIR="/root" -ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries # install dependencies that are missing on the base images -RUN curl -fsSL -o /tmp/apache-maven.tar.gz ${BASE_URL}/apache-maven-${MAVEN_VERSION}-bin.tar.gz \ - && mkdir -p /usr/share/maven \ - && tar -xzf /tmp/apache-maven.tar.gz -C /usr/share/maven --strip-components=1 \ - && rm -f /tmp/apache-maven.tar.gz \ - && ln -s /usr/share/maven/bin/mvn /usr/bin/mvn \ +RUN ln -s /usr/share/apache-maven-*/bin/mvn /usr/bin/mvn \ && dnf install -y -d1 python2 python2-pip \ && /usr/bin/pip2 install paramiko --no-cache-dir diff --git a/concourse/docker/pxf-dev-base/gpdb7/ubuntu18.04/Dockerfile b/concourse/docker/pxf-dev-base/gpdb7/ubuntu18.04/Dockerfile index 2ef6b480a6..7b470231b2 100644 --- a/concourse/docker/pxf-dev-base/gpdb7/ubuntu18.04/Dockerfile +++ b/concourse/docker/pxf-dev-base/gpdb7/ubuntu18.04/Dockerfile @@ -4,6 +4,8 @@ FROM ${BASE_IMAGE} ARG GO_VERSION +ADD apache-maven.tar.gz /usr/share + # install Go utilities RUN mkdir -p /tmp/pxf_src/ && cd /tmp \ && wget -O go.tgz -q https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz \ @@ -11,20 +13,14 @@ RUN mkdir -p /tmp/pxf_src/ && cd /tmp \ # install dependencies that are missing on the base images # install a specific version of perl for tinc - -ARG MAVEN_VERSION ARG USER_HOME_DIR="/root" -ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries RUN apt-get update -y \ && DEBIAN_FRONTEND=noninteractive apt-get install -y python-dev curl sudo jq openjdk-8-jdk openjdk-11-jdk \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ - && mkdir -p /usr/share/maven /usr/share/maven/ref \ - && curl -fsSL -o /tmp/apache-maven.tar.gz ${BASE_URL}/apache-maven-${MAVEN_VERSION}-bin.tar.gz \ - && tar -xzf /tmp/apache-maven.tar.gz -C /usr/share/maven --strip-components=1 \ - && rm -f /tmp/apache-maven.tar.gz \ - && ln -s /usr/share/maven/bin/mvn /usr/bin/mvn + && mkdir -p /usr/share/apache-maven-*/ref \ + && ln -s /usr/share/apache-maven-*/bin/mvn /usr/bin/mvn # create user gpadmin since GPDB cannot run under root RUN locale-gen en_US.UTF-8 \ diff --git a/concourse/scripts/download-maven-from-apache-mirror.sh b/concourse/scripts/download-maven-from-apache-mirror.sh new file mode 100755 index 0000000000..d95f643f1a --- /dev/null +++ b/concourse/scripts/download-maven-from-apache-mirror.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +set -e +set -u + +downloads_dir=${HOME}/workspace/pxf/downloads/ + +MAVEN_VERSION="${1:?a Maven version must be provided}" + +if [[ "${MAVEN_VERSION}" == "latest" ]]; then + echo "Looking for latest maven-3 version..." + MAVEN_VERSION=$(curl -fsSL https://archive.apache.org/dist/maven/maven-3/ | perl -lne 'print for /href="([0-9.]+)\/"/' | sort --version-sort | tail -1) + + echo "Latest maven-3 version determined to be: ${MAVEN_VERSION}" + while true; do + read -r -p "Would you like to proceed (y/n)? " yn + case $yn in + [Yy]*) break ;; + [Nn]*) exit ;; + *) echo "Please answer yes or no." ;; + esac + done +fi + +maven_dist="apache-maven-${MAVEN_VERSION}-bin.tar.gz" +maven_full_path="maven/maven-3/${MAVEN_VERSION}/binaries/${maven_dist}" + +response_json="$(curl -fsSL "https://www.apache.org/dyn/closer.lua/${maven_full_path}?as_json")" +in_dist="$(jq -r '.in_dist // false' <<<"$response_json")" + +if [[ "$in_dist" == "false" ]]; then + echo >&2 "${maven_dist} was not found in dist; attempting to download from archive.apache.org" + download_url="https://archive.apache.org/dist/${maven_full_path}" +else + preferred="$(jq -r ".preferred" <<<"$response_json")" + path_info="$(jq -r ".path_info" <<<"$response_json")" + if [[ -z "$preferred" || -z "$path_info" ]]; then + echo >&2 "unable to get download URL from response" + echo >&2 "$response_json" + exit 1 + fi + download_url="${preferred%/}/$path_info" +fi + +echo "Downloading $download_url to ${PXF_HOME}/${maven_dist}..." +curl -Lo "${downloads_dir}/${maven_dist}" "$download_url" From 40a34a0f01ec3ba6bf2d95cbd2f4d8cfb6500788 Mon Sep 17 00:00:00 2001 From: Himanshu Pandey Date: Tue, 8 Aug 2023 14:44:45 -0700 Subject: [PATCH 09/24] Updated certification pipeline to include GPDB 7 certification job (#1011) * Add GP7 certification * Changed the Slack Alerts for the certification pipeline. This commit updates the Slack alert configuration in the certification pipeline to improve team communication and visibility during CI/CD processes. Previously, Slack alerts were sent only when the pipeline ran successfully. With this change, Slack alerts will now be triggered on individual job failures, providing timely notifications and enabling faster issue resolution. EMs will be mentioned in the corresponding slack alerts. * The changes in certification_pipeline.yml passes the yamllint check. * Added 3 TODOs are added in this commit so that once pg_regress replaces TINC, we will remove the gp6_python_libs resource and its reference. --- .../pipelines/certification_pipeline.yml | 104 ++++++++++++++++-- concourse/scripts/test.bash | 5 +- concourse/tasks/test_certification.yml | 3 + 3 files changed, 102 insertions(+), 10 deletions(-) diff --git a/concourse/pipelines/certification_pipeline.yml b/concourse/pipelines/certification_pipeline.yml index 2a9d030084..cb060c6323 100644 --- a/concourse/pipelines/certification_pipeline.yml +++ b/concourse/pipelines/certification_pipeline.yml @@ -7,13 +7,8 @@ anchors: on_failure: put: slack-alert params: - text: | - <((ud/pxf/secrets/ud-concourse-url))/builds/$BUILD_ID|$BUILD_PIPELINE_NAME/$BUILD_JOB_NAME> went red :blob_slightly_frowning_face: - on_success: - put: slack-alert - params: - text: | - <((ud/pxf/secrets/ud-concourse-url))/builds/$BUILD_ID|$BUILD_PIPELINE_NAME/$BUILD_JOB_NAME> went green! :smile: + # yamllint disable-line rule:line-length + text: ":ci-fail: <${ATC_EXTERNAL_URL}/builds/$BUILD_ID|$BUILD_PIPELINE_NAME/$BUILD_JOB_NAME> has failed (((ud/common/slack-em-ids)))" ## ====================================================================== ## RESOURCE TYPES @@ -44,6 +39,15 @@ resources: json_key: ((ud/pxf/secrets/pxf-storage-service-account-key)) versioned_file: automation-dependencies/pxf-automation-dependencies.tar.gz +## TODO Remove this resource block once Tinc is replaced with pg_regress. +- name: gp6-python-libs + type: gcs + icon: google-drive + source: + bucket: ((ud/pxf/common/build-resources-bucket-name)) + json_key: ((ud/pxf/secrets/pxf-storage-service-account-key)) + versioned_file: automation-dependencies/gp6-python-libs.tar.gz + - name: singlecluster-hdp2 type: gcs icon: google-drive @@ -98,6 +102,15 @@ resources: username: _json_key password: ((ud/pxf/secrets/pxf-cloudbuild-service-account-key)) +- name: gpdb7-pxf-dev-rocky8-image + type: registry-image + icon: docker + source: + repository: gcr.io/data-gpdb-ud/gpdb-pxf-dev/gpdb7-rocky8-test-pxf + tag: latest + username: _json_key + password: ((ud/pxf/secrets/pxf-cloudbuild-service-account-key)) + - name: ccp-7-image type: registry-image icon: docker @@ -132,6 +145,14 @@ resources: json_key: ((concourse-gcs-resources-service-account-key)) regexp: server/published/gpdb6/server-rc-(.*)-rhel8_x86_64.tar.gz +- name: gpdb7_tarball_rhel8 + type: gcs + icon: google-drive + source: + bucket: ((ud/pxf/common/gpdb-concourse-resources-prod-bucket-name)) + json_key: ((concourse-gcs-resources-service-account-key)) + regexp: server/published/main/server-rc-7.(.*)-el8_x86_64.tar.gz + - name: gpdb6_ubuntu18_tarball type: gcs icon: google-drive @@ -175,6 +196,14 @@ resources: json_key: ((ud/pxf/secrets/pxf-storage-service-account-key)) regexp: prod/releases/gp6/pxf-gp6-(.*)-1-ubuntu18.04-amd64.deb +- name: pxf_gp7_rpm_rhel8 + type: gcs + icon: google-drive + source: + bucket: ((ud/pxf/common/releases-bucket-name)) + json_key: ((ud/pxf/secrets/pxf-storage-service-account-key)) + regexp: prod/releases/gp7/pxf-gp7-(.*)-1.el8.x86_64.rpm + - name: slack-alert type: slack-notification source: @@ -202,6 +231,7 @@ jobs: - get: singlecluster resource: singlecluster-hdp2 - task: Test GPDB-5 with PXF-GP5-HDP2 on RHEL7 + <<: *slack_alert file: pxf_src/concourse/tasks/test_certification.yml image: gpdb5-pxf-dev-centos7-image params: @@ -233,6 +263,7 @@ jobs: - get: singlecluster resource: singlecluster-hdp2 - task: Test GPDB-6 with PXF-GP6-HDP2 on RHEL7 + <<: *slack_alert file: pxf_src/concourse/tasks/test_certification.yml image: gpdb6-pxf-dev-centos7-image params: @@ -265,6 +296,7 @@ jobs: - get: singlecluster resource: singlecluster-hdp2 - task: Test GPDB-6 with PXF-GP6-HDP2 on RHEL8 + <<: *slack_alert file: pxf_src/concourse/tasks/test_certification.yml image: gpdb6-pxf-dev-rocky8-image params: @@ -280,6 +312,39 @@ jobs: GP_VER: 6 PXF_CERTIFICATION_FOLDER: data-gpdb-ud-pxf-build/prod/certifications +- name: Certify GPDB-7 with PXF-GP7 on RHEL8 + plan: + - in_parallel: + - get: pxf_src + - get: bin_gpdb + resource: gpdb7_tarball_rhel8 + trigger: true + - get: pxf_package + resource: pxf_gp7_rpm_rhel8 + trigger: true + - get: gpdb7-pxf-dev-rocky8-image + - get: ccp-7-image + - get: pxf-automation-dependencies + - get: gp6-python-libs + - get: singlecluster + resource: singlecluster-hdp2 + - task: Test GPDB-7 with PXF-GP7-HDP2 on RHEL8 + <<: *slack_alert + file: pxf_src/concourse/tasks/test_certification.yml + image: gpdb7-pxf-dev-rocky8-image + params: + ACCESS_KEY_ID: ((tf-machine-access-key-id)) + GP_VER: 7 + GROUP: gpdb,proxy,profile + SECRET_ACCESS_KEY: ((tf-machine-secret-access-key)) + - task: Upload certification for GPDB-7 with PXF-GP7-HDP2 on RHEL8 + file: pxf_src/concourse/tasks/certification_upload.yml + image: ccp-7-image + params: + GOOGLE_CREDENTIALS: ((ud/pxf/secrets/pxf-storage-service-account-key)) + GP_VER: 7 + PXF_CERTIFICATION_FOLDER: data-gpdb-ud-pxf-build/prod/certifications + ## ---------- Ubuntu 18 Swimlane ---------- - name: Certify GPDB-6 with PXF-GP6 on Ubuntu 18.04 plan: @@ -297,6 +362,7 @@ jobs: - get: singlecluster resource: singlecluster-hdp2 - task: Test GPDB-6 with PXF-GP6-HDP2 on Ubuntu 18.04 + <<: *slack_alert file: pxf_src/concourse/tasks/test_certification.yml image: gpdb6-pxf-dev-ubuntu18-image params: @@ -328,7 +394,6 @@ jobs: trigger: true - get: ccp-7-image - task: Print Report for GPDB-5 with PXF-GP5 Artifacts - <<: *slack_alert file: pxf_src/concourse/tasks/certification_list.yml image: ccp-7-image params: @@ -357,10 +422,31 @@ jobs: - Certify GPDB-6 with PXF-GP6 on Ubuntu 18.04 - get: ccp-7-image - task: Print Report for GPDB-6 with PXF-GP6 Artifacts - <<: *slack_alert file: pxf_src/concourse/tasks/certification_list.yml image: ccp-7-image params: GOOGLE_CREDENTIALS: ((ud/pxf/secrets/pxf-storage-service-account-key)) GP_VER: 6 PXF_CERTIFICATION_FOLDER: data-gpdb-ud-pxf-build/prod/certifications + +- name: Reporting Gate for PXF-GP7 + plan: + - in_parallel: + - get: pxf_src + # gpdb release candidate tarballs and PXF RPMs used in testing jobs + - get: gpdb7_tarball_rhel8 + passed: + - Certify GPDB-7 with PXF-GP7 on RHEL8 + trigger: true + - get: pxf_gp7_rpm_rhel8 + passed: + - Certify GPDB-7 with PXF-GP7 on RHEL8 + trigger: true + - get: ccp-7-image + - task: Print Report for GPDB-7 with PXF-GP7 Artifacts + file: pxf_src/concourse/tasks/certification_list.yml + image: ccp-7-image + params: + GOOGLE_CREDENTIALS: ((ud/pxf/secrets/pxf-storage-service-account-key)) + GP_VER: 7 + PXF_CERTIFICATION_FOLDER: data-gpdb-ud-pxf-build/prod/certifications diff --git a/concourse/scripts/test.bash b/concourse/scripts/test.bash index 227a893298..f179dfe12e 100755 --- a/concourse/scripts/test.bash +++ b/concourse/scripts/test.bash @@ -253,11 +253,14 @@ function _main() { inflate_dependencies + #TODO Remove this "if" block once Tinc is replaced with pg_regress. # To run Tinc against GP7 we need to modify PYTHONPATH in $GPHOME/greenplum_path.sh since Tinc calls that script # we will set PYTHONPATH to point to the set of python libs compiled with Python2 for GP6 if [[ ${GP_VER} == 7 ]]; then local gp6_python_libs=~gpadmin/python - echo "export PYTHONPATH=${gp6_python_libs}" >> /usr/local/greenplum-db/greenplum_path.sh + echo "# Added by test.bash - Overriding PYTHONPATH to run the Tinc Tests in GP7" >> ${GPHOME}/greenplum_path.sh + echo "# Comment the following line out if you need to run GP utilities" >> ${GPHOME}/greenplum_path.sh + echo "export PYTHONPATH=${gp6_python_libs}" >> ${GPHOME}/greenplum_path.sh fi ln -s "${PWD}/pxf_src" ~gpadmin/pxf_src diff --git a/concourse/tasks/test_certification.yml b/concourse/tasks/test_certification.yml index 41c285ca35..e4d540c5ec 100644 --- a/concourse/tasks/test_certification.yml +++ b/concourse/tasks/test_certification.yml @@ -9,6 +9,9 @@ inputs: - name: pxf_package - name: singlecluster - name: pxf-automation-dependencies +# TODO Remove gp6-python-libs input once Tinc is replaced with pg_regress. +- name: gp6-python-libs + optional: true outputs: - name: certification From b5d43349c75645bbee1575df9a6ef61caeb6609f Mon Sep 17 00:00:00 2001 From: Alexander Denissov Date: Wed, 16 Aug 2023 09:58:36 -0700 Subject: [PATCH 10/24] Bumped Azure Storage dependency to 5.5.0 (#1013) --- automation/pom.xml | 2 +- server/build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/automation/pom.xml b/automation/pom.xml index 26dae47130..39e2224f41 100644 --- a/automation/pom.xml +++ b/automation/pom.xml @@ -287,7 +287,7 @@ com.microsoft.azure azure-storage - 5.4.0 + 5.5.0 diff --git a/server/build.gradle b/server/build.gradle index b110e59a81..770901203b 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -75,7 +75,7 @@ configure(javaProjects) { dependency("com.google.guava:guava:20.0") dependency("com.google.protobuf:protobuf-java:2.5.0") dependency("com.google.cloud.bigdataoss:gcs-connector:hadoop2-1.9.17") - dependency("com.microsoft.azure:azure-storage:5.4.0") + dependency("com.microsoft.azure:azure-storage:5.5.0") dependency("com.microsoft.azure:azure-data-lake-store-sdk:2.3.9") dependency("com.univocity:univocity-parsers:2.9.1") dependency("com.yammer.metrics:metrics-core:2.2.0") From ac202f98d9cc8cdb81d3e09d1b6077cc1ef6b11d Mon Sep 17 00:00:00 2001 From: Himanshu Pandey Date: Thu, 17 Aug 2023 11:57:40 -0700 Subject: [PATCH 11/24] Adding CLI Test Job for GP7 for dev pipeline (#1012) * Adding CLI Test Job for GP7 - In GP7, gpscp is replaced by gpsync. Introduced a variable (GP_SCP_CMD) to specify the appropriate command based on GP version. - Replaced the hardcoded pxf home (/usr/local/pxf-gp6) with the variable PXF_HOME. - Added variables to the Makefile (GP7_CLI) and dev pipeline template (gp7_cli), when set to true, this will introduce PXF_CLI tests for GP7. - Ran yamllint and shellcheck to confirm there are no issues with the new changes. - For ccp, use PLATFORM value (rocky8-gpdb7) as the corresponding VM Image comes preinstalled with all GPDB7 runtime dependencies. --------- Co-authored-by: Ed Espino --- concourse/Makefile | 2 + .../templates/dev_build_pipeline-tpl.yml | 84 +++++++++++++++++-- concourse/scripts/cli/test_reset_init.sh | 4 +- concourse/scripts/install_pxf.bash | 18 ++-- 4 files changed, 96 insertions(+), 12 deletions(-) diff --git a/concourse/Makefile b/concourse/Makefile index 5fda6dd540..5177890594 100644 --- a/concourse/Makefile +++ b/concourse/Makefile @@ -37,6 +37,7 @@ GS ?= false MINIO ?= false OEL7 ?= false FILE ?= false +GP7_CLI ?= false SET_PIPELINE := set-pipeline ifeq ($(CHECK_CREDS), true) @@ -116,6 +117,7 @@ set-dev-build-pipeline: gs=$(GS) \ minio=$(MINIO) \ oel7=$(OEL7) \ + gp7_cli=$(GP7_CLI) \ dev_pipeline=true \ user=$(USER) \ branch=$(BRANCH) \ diff --git a/concourse/pipelines/templates/dev_build_pipeline-tpl.yml b/concourse/pipelines/templates/dev_build_pipeline-tpl.yml index 21863fb011..e302fa26fb 100644 --- a/concourse/pipelines/templates/dev_build_pipeline-tpl.yml +++ b/concourse/pipelines/templates/dev_build_pipeline-tpl.yml @@ -73,7 +73,7 @@ anchors: ## ====================================================================== resource_types: -{% if multinode or multinode_no_impersonation or file %} +{% if multinode or multinode_no_impersonation or file or gp7_cli %} - name: terraform type: registry-image source: @@ -169,7 +169,7 @@ resources: branch: ((pxf-git-branch)) uri: ((ud/pxf/common/git-remote)) -{% if multinode or multinode_no_impersonation or file %} +{% if multinode or multinode_no_impersonation or file or gp7_cli %} - name: ccp_src type: git source: @@ -202,7 +202,6 @@ resources: username: _json_key password: ((ud/pxf/secrets/pxf-cloudbuild-service-account-key)) {% endfor %} -{% set gp_ver = None %} {% if oel7 %} - name: gpdb6-pxf-dev-oel7-image @@ -237,7 +236,7 @@ resources: {% endif %} {% set gp_ver = None %} -{% if multinode or multinode_no_impersonation or ambari or file %} +{% if multinode or multinode_no_impersonation or ambari or file or gp7_cli %} - name: ccp-7-image type: registry-image icon: docker @@ -351,7 +350,7 @@ resources: {% endfor %} ## ---------- Auxiliary Resources ---------- -{% if multinode or multinode_no_impersonation or file %} +{% if multinode or multinode_no_impersonation or file or gp7_cli %} - name: terraform type: terraform source: @@ -618,6 +617,81 @@ jobs: {% endfor %} {% set gp_ver = None %} +## ---------- GP7 PXF CLI tests ---------- + +{% if gp7_cli %} +{% set gp_ver = 7 %} +- name: Test PXF-GP[[gp_ver]]-CLI on RHEL8 + max_in_flight: 2 + plan: + - in_parallel: + - get: pxf_src + passed: [Build PXF-GP[[gp_ver]] on RHEL8] + trigger: true + - get: pxf_tarball + resource: pxf_gp[[gp_ver]]_tarball_rhel8 + passed: [Build PXF-GP[[gp_ver]] on RHEL8] + - get: gpdb_package + resource: gpdb[[gp_ver]]_rhel8_rpm_latest-0 + passed: [Build PXF-GP[[gp_ver]] on RHEL8] + - get: gpdb[[gp_ver]]-pxf-dev-rocky8-image + - get: ccp_src + - get: ccp-7-image + - in_parallel: + - do: + - put: terraform_gpdb + resource: terraform + params: + action: create + delete_on_failure: true + generate_random_name: true + terraform_source: ccp_src/google/ + vars: + PLATFORM: rocky8-gpdb[[gp_ver]] + number_of_nodes: ((number_of_gpdb_nodes)) + extra_nodes: 1 + segments_per_host: 4 + instance_type: n1-standard-4 + ccp_reap_minutes: 120 + standby_master: true + - task: Generate Greenplum Cluster + input_mapping: + gpdb_rpm: gpdb_package + terraform: terraform_gpdb + file: ccp_src/ci/tasks/gen_cluster.yml + image: ccp-7-image + params: + AWS_ACCESS_KEY_ID: ((tf-machine-access-key-id)) + AWS_SECRET_ACCESS_KEY: ((tf-machine-secret-access-key)) + AWS_DEFAULT_REGION: ((ud/common/aws-region)) + BUCKET_PATH: ((tf-bucket-path)) + BUCKET_NAME: ((ud/pxf/common/tf-bucket-name)) + PLATFORM: rocky8-gpdb[[gp_ver]] + CLOUD_PROVIDER: google + GPDB_RPM: true + - task: Initialize Greenplum + file: ccp_src/ci/tasks/gpinitsystem.yml + - task: Setup PXF + input_mapping: + terraform: terraform_gpdb + file: pxf_src/concourse/tasks/install_pxf_on_ccp.yml + image: ccp-7-image + params: + IMPERSONATION: false + INSTALL_GPHDFS: false + GP_VER: [[gp_ver]] + PXF_JVM_OPTS: ((pxf-jvm-opts)) + - task: Test PXF CLI + on_success: + <<: *destroy_gpdb_cluster + image: gpdb[[gp_ver]]-pxf-dev-rocky8-image + file: pxf_src/concourse/tasks/test_pxf_cli.yml +{% if slack_notification %} + <<: *slack_alert +{% endif %} +{% endif %} # if gp7_cli ends here +{% set gp_ver = None %} + ## ---------- Extended Tests ---------- {% set gp_ver = 6 %} {% if hdp2 %} diff --git a/concourse/scripts/cli/test_reset_init.sh b/concourse/scripts/cli/test_reset_init.sh index 1ba0531999..18ca90c229 100755 --- a/concourse/scripts/cli/test_reset_init.sh +++ b/concourse/scripts/cli/test_reset_init.sh @@ -93,10 +93,10 @@ Initializing PXF on ${cluster_description} PXF initialized successfully on ${num_hosts} out of ${num_hosts} hosts" control_file_content=\ -"directory = '/usr/local/pxf-gp6/gpextable/' +"directory = '${PXF_HOME}/gpextable/' default_version = '2.1' comment = 'Extension which allows to access unmanaged data' -module_pathname = '/usr/local/pxf-gp6/gpextable/pxf' +module_pathname = '${PXF_HOME}/gpextable/pxf' superuser = true relocatable = false schema = public" diff --git a/concourse/scripts/install_pxf.bash b/concourse/scripts/install_pxf.bash index a8bf5f976d..ff763ac2f2 100755 --- a/concourse/scripts/install_pxf.bash +++ b/concourse/scripts/install_pxf.bash @@ -12,6 +12,14 @@ if [[ ${PXF_COMPONENT} == "true" ]]; then else PXF_HOME=${GPHOME}/pxf fi + +# gpscp command is replaced with gpsync in GP7 +if [ "${GP_VER}" -lt 7 ]; then + GP_SCP_CMD=gpscp +else + GP_SCP_CMD=gpsync +fi + PXF_VERSION=${PXF_VERSION:=6} # read default user from terraform metadata file DEFAULT_OS_USER=$(jq --raw-output ".ami_default_user" terraform/metadata) @@ -126,16 +134,16 @@ function create_pxf_installer_scripts() { if [[ ${PXF_VERSION} == 5 ]]; then echo 'export PXF_KEYTAB="\${PXF_CONF}/keytabs/pxf.service.keytab"' >> "\${PXF_CONF}/conf/pxf-env.sh" echo 'export PXF_PRINCIPAL="gpadmin@${REALM}"' >> "\${PXF_CONF}/conf/pxf-env.sh" - gpscp -f ~gpadmin/hostfile_all -v -r -u gpadmin ~/dataproc_env_files/pxf.service.keytab =:/home/gpadmin/pxf/keytabs/ + ${GP_SCP_CMD} -f ~gpadmin/hostfile_all -v -r -u gpadmin ~/dataproc_env_files/pxf.service.keytab =:/home/gpadmin/pxf/keytabs/ else if [[ ! -f \${PXF_BASE}/servers/default/pxf-site.xml ]]; then cp \${PXF_HOME}/templates/pxf-site.xml \${PXF_BASE}/servers/default/pxf-site.xml fi sed -i -e "s|gpadmin/_HOST@EXAMPLE.COM|gpadmin@${REALM}|g" ${BASE_DIR}/servers/default/pxf-site.xml - gpscp -f ~gpadmin/hostfile_all -v -r -u gpadmin ~/dataproc_env_files/pxf.service.keytab =:${BASE_DIR}/keytabs/ + ${GP_SCP_CMD} -f ~gpadmin/hostfile_all -v -r -u gpadmin ~/dataproc_env_files/pxf.service.keytab =:${BASE_DIR}/keytabs/ fi - gpscp -f ~gpadmin/hostfile_all -v -r -u ${DEFAULT_OS_USER} /tmp/krb5.conf =:/tmp/krb5.conf + ${GP_SCP_CMD} -f ~gpadmin/hostfile_all -v -r -u ${DEFAULT_OS_USER} /tmp/krb5.conf =:/tmp/krb5.conf gpssh -f ~gpadmin/hostfile_all -v -u ${DEFAULT_OS_USER} -s -e 'sudo mv /tmp/krb5.conf /etc/krb5.conf' fi } @@ -239,7 +247,7 @@ function run_pxf_installer_scripts() { gpstop -u fi && sed -i '/edw/d' hostfile_all && - gpscp -f ~gpadmin/hostfile_all -v -u ${DEFAULT_OS_USER} -r ~/pxf_installer ~gpadmin/install_pxf_dependencies.sh ${DEFAULT_OS_USER}@=: && + ${GP_SCP_CMD} -f ~gpadmin/hostfile_all -v -u ${DEFAULT_OS_USER} -r ~/pxf_installer ~gpadmin/install_pxf_dependencies.sh ${DEFAULT_OS_USER}@=: && gpssh -f ~gpadmin/hostfile_all -v -u ${DEFAULT_OS_USER} -s -e 'sudo ~${DEFAULT_OS_USER}/install_pxf_dependencies.sh' && gpssh -f ~gpadmin/hostfile_all -v -u ${DEFAULT_OS_USER} -s -e 'sudo GPHOME=${GPHOME} ~${DEFAULT_OS_USER}/pxf_installer/install_component' gpssh -f ~gpadmin/hostfile_all -v -u ${DEFAULT_OS_USER} -s -e 'sudo chown -R gpadmin:gpadmin ${PXF_HOME}' @@ -249,7 +257,7 @@ function run_pxf_installer_scripts() { ${PXF_HOME}/bin/pxf cluster register fi && if [[ -d ~/dataproc_env_files ]]; then - gpscp -f ~gpadmin/hostfile_init -v -r -u gpadmin ~/dataproc_env_files =: + ${GP_SCP_CMD} -f ~gpadmin/hostfile_init -v -r -u gpadmin ~/dataproc_env_files =: fi && ~gpadmin/configure_pxf.sh && source ~gpadmin/.bashrc && From 78613bad99786e7a59f1dbf81a8cdafe768378ef Mon Sep 17 00:00:00 2001 From: Himanshu Pandey Date: Thu, 24 Aug 2023 14:01:09 -0700 Subject: [PATCH 12/24] Ensured Consistent Ordering in GP6 and GP7 for JdbcHiveTest (#1017) * Ensured Consistent Ordering in GP6 and GP7 for JdbcHiveTest Addressed query result ordering inconsistencies observed between Greenplum 6 (GP6)and Greenplum 7 (GP7) for JdbcHiveTest. Modified test queries and answer files to enhance result ordering consistency. Added an extra column "s1" to the ORDER BY clause in test queries, ensuring predictable and stable ordering across both GP6 and GP7 environments. --- .../expected/query02.ans | 26 +++--- .../expected/query03.ans | 60 +++++++------- .../expected/query04.ans | 80 +++++++++---------- .../sql/query02.sql | 2 +- .../sql/query03.sql | 6 +- .../sql/query04.sql | 6 +- .../two_secured_hive/expected/query02.ans | 16 ++-- .../two_secured_hive/expected/query03.ans | 30 +++---- .../two_secured_hive/expected/query04.ans | 64 +++++++-------- .../jdbc/two_secured_hive/sql/query02.sql | 2 +- .../jdbc/two_secured_hive/sql/query03.sql | 6 +- .../jdbc/two_secured_hive/sql/query04.sql | 6 +- 12 files changed, 152 insertions(+), 152 deletions(-) diff --git a/automation/tincrepo/main/pxf/features/jdbc/secured_and_non_secured_hive/expected/query02.ans b/automation/tincrepo/main/pxf/features/jdbc/secured_and_non_secured_hive/expected/query02.ans index 34b34157e6..ded6c21acb 100644 --- a/automation/tincrepo/main/pxf/features/jdbc/secured_and_non_secured_hive/expected/query02.ans +++ b/automation/tincrepo/main/pxf/features/jdbc/secured_and_non_secured_hive/expected/query02.ans @@ -33,28 +33,28 @@ SELECT s1, n1 FROM pxf_jdbc_hive_non_secure_types_table WHERE tn < 11 ORDER BY n SELECT s1, n1 FROM pxf_jdbc_hive_types_table WHERE tn < 11 UNION ALL SELECT s1, n1 FROM pxf_jdbc_hive_non_secure_types_table WHERE tn < 11 -ORDER BY n1; - s1 | n1 --------------------+---- +ORDER BY n1, s1; + s1 | n1 +------------------+---- + row1 | 1 third_hive_row1 | 1 - row1 | 1 + row2 | 2 third_hive_row2 | 2 - row2 | 2 + row3 | 3 third_hive_row3 | 3 - row3 | 3 - row4 | 4 + row4 | 4 third_hive_row4 | 4 + row5 | 5 third_hive_row5 | 5 - row5 | 5 - row6 | 6 + row6 | 6 third_hive_row6 | 6 + row7 | 7 third_hive_row7 | 7 - row7 | 7 + row8 | 8 third_hive_row8 | 8 - row8 | 8 - row9 | 9 + row9 | 9 third_hive_row9 | 9 + row10 | 10 third_hive_row10 | 10 - row10 | 10 (20 rows) diff --git a/automation/tincrepo/main/pxf/features/jdbc/secured_and_non_secured_hive/expected/query03.ans b/automation/tincrepo/main/pxf/features/jdbc/secured_and_non_secured_hive/expected/query03.ans index bebb689906..d84428f7fc 100644 --- a/automation/tincrepo/main/pxf/features/jdbc/secured_and_non_secured_hive/expected/query03.ans +++ b/automation/tincrepo/main/pxf/features/jdbc/secured_and_non_secured_hive/expected/query03.ans @@ -1,12 +1,11 @@ -- start_ignore -- end_ignore -- @description query03 for Multiple JDBC Hive Server queries with timestamp filter -SELECT s1, n1, tm FROM pxf_jdbc_hive_types_table WHERE tm > '2013-07-23 21:00:01' ORDER BY n1; +SELECT s1, n1, tm FROM pxf_jdbc_hive_types_table WHERE tm > '2013-07-23 21:00:01' ORDER BY n1, s1; s1 | n1 | tm ---------------------+----+--------------------- + row11 | 11 | 2013-07-23 21:00:05 row12_text_null | 11 | 2013-07-23 21:00:05 - row24_char_null | 11 | 2013-07-23 21:00:05 - row23_varchar_null | 11 | 2013-07-23 21:00:05 row14_double_null | 11 | 2013-07-23 21:00:05 row17_real_null | 11 | 2013-07-23 21:00:05 row18_bigint_null | 11 | 2013-07-23 21:00:05 @@ -14,17 +13,17 @@ SELECT s1, n1, tm FROM pxf_jdbc_hive_types_table WHERE tm > '2013-07-23 21:00:01 row20_tinyint_null | 11 | 2013-07-23 21:00:05 row21_smallint_null | 11 | 2013-07-23 21:00:05 row22_date_null | 11 | 2013-07-23 21:00:05 - row11 | 11 | 2013-07-23 21:00:05 + row23_varchar_null | 11 | 2013-07-23 21:00:05 + row24_char_null | 11 | 2013-07-23 21:00:05 row15_decimal_null | 12 | 2013-07-24 21:00:05 row13_int_null | | 2013-07-23 21:00:05 (13 rows) -SELECT s1, n1, tm FROM pxf_jdbc_hive_non_secure_types_table WHERE tm > '2013-07-23 21:00:01' ORDER BY n1; - s1 | n1 | tm ----------------------------------+----+--------------------- +SELECT s1, n1, tm FROM pxf_jdbc_hive_non_secure_types_table WHERE tm > '2013-07-23 21:00:01' ORDER BY n1, s1; + s1 | n1 | tm +--------------------------------+----+--------------------- + third_hive_row11 | 11 | 2013-07-23 21:00:05 third_hive_row12_text_null | 11 | 2013-07-23 21:00:05 - third_hive_row24_char_null | 11 | 2013-07-23 21:00:05 - third_hive_row23_varchar_null | 11 | 2013-07-23 21:00:05 third_hive_row14_double_null | 11 | 2013-07-23 21:00:05 third_hive_row17_real_null | 11 | 2013-07-23 21:00:05 third_hive_row18_bigint_null | 11 | 2013-07-23 21:00:05 @@ -32,30 +31,28 @@ SELECT s1, n1, tm FROM pxf_jdbc_hive_non_secure_types_table WHERE tm > '2013-07- third_hive_row20_tinyint_null | 11 | 2013-07-23 21:00:05 third_hive_row21_smallint_null | 11 | 2013-07-23 21:00:05 third_hive_row22_date_null | 11 | 2013-07-23 21:00:05 - third_hive_row11 | 11 | 2013-07-23 21:00:05 + third_hive_row23_varchar_null | 11 | 2013-07-23 21:00:05 + third_hive_row24_char_null | 11 | 2013-07-23 21:00:05 third_hive_row15_decimal_null | 12 | 2013-07-24 21:00:05 third_hive_row13_int_null | | 2013-07-23 21:00:05 (13 rows) SELECT s1, n1, tm FROM pxf_jdbc_hive_types_table WHERE tm > '2013-07-23 21:00:01' UNION ALL SELECT s1, n1, tm FROM pxf_jdbc_hive_non_secure_types_table WHERE tm > '2013-07-23 21:00:01' -ORDER BY n1; - s1 | n1 | tm ----------------------------------+----+--------------------- - row12_text_null | 11 | 2013-07-23 21:00:05 - third_hive_row24_char_null | 11 | 2013-07-23 21:00:05 - third_hive_row23_varchar_null | 11 | 2013-07-23 21:00:05 - third_hive_row22_date_null | 11 | 2013-07-23 21:00:05 - third_hive_row21_smallint_null | 11 | 2013-07-23 21:00:05 - row14_double_null | 11 | 2013-07-23 21:00:05 - row17_real_null | 11 | 2013-07-23 21:00:05 - row18_bigint_null | 11 | 2013-07-23 21:00:05 - row19_bool_null | 11 | 2013-07-23 21:00:05 - row20_tinyint_null | 11 | 2013-07-23 21:00:05 - row21_smallint_null | 11 | 2013-07-23 21:00:05 - row22_date_null | 11 | 2013-07-23 21:00:05 - row23_varchar_null | 11 | 2013-07-23 21:00:05 - row24_char_null | 11 | 2013-07-23 21:00:05 +ORDER BY n1, s1; + s1 | n1 | tm +--------------------------------+----+--------------------- + row11 | 11 | 2013-07-23 21:00:05 + row12_text_null | 11 | 2013-07-23 21:00:05 + row14_double_null | 11 | 2013-07-23 21:00:05 + row17_real_null | 11 | 2013-07-23 21:00:05 + row18_bigint_null | 11 | 2013-07-23 21:00:05 + row19_bool_null | 11 | 2013-07-23 21:00:05 + row20_tinyint_null | 11 | 2013-07-23 21:00:05 + row21_smallint_null | 11 | 2013-07-23 21:00:05 + row22_date_null | 11 | 2013-07-23 21:00:05 + row23_varchar_null | 11 | 2013-07-23 21:00:05 + row24_char_null | 11 | 2013-07-23 21:00:05 third_hive_row11 | 11 | 2013-07-23 21:00:05 third_hive_row12_text_null | 11 | 2013-07-23 21:00:05 third_hive_row14_double_null | 11 | 2013-07-23 21:00:05 @@ -63,10 +60,13 @@ ORDER BY n1; third_hive_row18_bigint_null | 11 | 2013-07-23 21:00:05 third_hive_row19_bool_null | 11 | 2013-07-23 21:00:05 third_hive_row20_tinyint_null | 11 | 2013-07-23 21:00:05 - row11 | 11 | 2013-07-23 21:00:05 - row15_decimal_null | 12 | 2013-07-24 21:00:05 + third_hive_row21_smallint_null | 11 | 2013-07-23 21:00:05 + third_hive_row22_date_null | 11 | 2013-07-23 21:00:05 + third_hive_row23_varchar_null | 11 | 2013-07-23 21:00:05 + third_hive_row24_char_null | 11 | 2013-07-23 21:00:05 + row15_decimal_null | 12 | 2013-07-24 21:00:05 third_hive_row15_decimal_null | 12 | 2013-07-24 21:00:05 + row13_int_null | | 2013-07-23 21:00:05 third_hive_row13_int_null | | 2013-07-23 21:00:05 - row13_int_null | | 2013-07-23 21:00:05 (26 rows) diff --git a/automation/tincrepo/main/pxf/features/jdbc/secured_and_non_secured_hive/expected/query04.ans b/automation/tincrepo/main/pxf/features/jdbc/secured_and_non_secured_hive/expected/query04.ans index 8e9f543cb6..fb68935c04 100644 --- a/automation/tincrepo/main/pxf/features/jdbc/secured_and_non_secured_hive/expected/query04.ans +++ b/automation/tincrepo/main/pxf/features/jdbc/secured_and_non_secured_hive/expected/query04.ans @@ -1,7 +1,7 @@ -- start_ignore -- end_ignore -- @description query04 for Multiple JDBC Hive Server queries with date filter -SELECT s1, n1, dt FROM pxf_jdbc_hive_types_table WHERE dt = '2015-03-06' ORDER BY n1; +SELECT s1, n1, dt FROM pxf_jdbc_hive_types_table WHERE dt = '2015-03-06' ORDER BY n1, s1; s1 | n1 | dt ----------------------+----+------------ row1 | 1 | 2015-03-06 @@ -15,8 +15,7 @@ SELECT s1, n1, dt FROM pxf_jdbc_hive_types_table WHERE dt = '2015-03-06' ORDER B row9 | 9 | 2015-03-06 row10 | 10 | 2015-03-06 row11 | 11 | 2015-03-06 - row24_char_null | 11 | 2015-03-06 - row23_varchar_null | 11 | 2015-03-06 + row12_text_null | 11 | 2015-03-06 row14_double_null | 11 | 2015-03-06 row16_timestamp_null | 11 | 2015-03-06 row17_real_null | 11 | 2015-03-06 @@ -24,14 +23,15 @@ SELECT s1, n1, dt FROM pxf_jdbc_hive_types_table WHERE dt = '2015-03-06' ORDER B row19_bool_null | 11 | 2015-03-06 row20_tinyint_null | 11 | 2015-03-06 row21_smallint_null | 11 | 2015-03-06 - row12_text_null | 11 | 2015-03-06 + row23_varchar_null | 11 | 2015-03-06 + row24_char_null | 11 | 2015-03-06 row15_decimal_null | 12 | 2015-03-06 row13_int_null | | 2015-03-06 (23 rows) -SELECT s1, n1, dt FROM pxf_jdbc_hive_non_secure_types_table WHERE dt = '2015-03-06' ORDER BY n1; - s1 | n1 | dt -----------------------------------+----+------------ +SELECT s1, n1, dt FROM pxf_jdbc_hive_non_secure_types_table WHERE dt = '2015-03-06' ORDER BY n1, s1; + s1 | n1 | dt +---------------------------------+----+------------ third_hive_row1 | 1 | 2015-03-06 third_hive_row2 | 2 | 2015-03-06 third_hive_row3 | 3 | 2015-03-06 @@ -43,8 +43,7 @@ SELECT s1, n1, dt FROM pxf_jdbc_hive_non_secure_types_table WHERE dt = '2015-03- third_hive_row9 | 9 | 2015-03-06 third_hive_row10 | 10 | 2015-03-06 third_hive_row11 | 11 | 2015-03-06 - third_hive_row24_char_null | 11 | 2015-03-06 - third_hive_row23_varchar_null | 11 | 2015-03-06 + third_hive_row12_text_null | 11 | 2015-03-06 third_hive_row14_double_null | 11 | 2015-03-06 third_hive_row16_timestamp_null | 11 | 2015-03-06 third_hive_row17_real_null | 11 | 2015-03-06 @@ -52,50 +51,48 @@ SELECT s1, n1, dt FROM pxf_jdbc_hive_non_secure_types_table WHERE dt = '2015-03- third_hive_row19_bool_null | 11 | 2015-03-06 third_hive_row20_tinyint_null | 11 | 2015-03-06 third_hive_row21_smallint_null | 11 | 2015-03-06 - third_hive_row12_text_null | 11 | 2015-03-06 + third_hive_row23_varchar_null | 11 | 2015-03-06 + third_hive_row24_char_null | 11 | 2015-03-06 third_hive_row15_decimal_null | 12 | 2015-03-06 third_hive_row13_int_null | | 2015-03-06 (23 rows) SELECT s1, n1, dt FROM pxf_jdbc_hive_types_table WHERE dt = '2015-03-06' UNION ALL SELECT s1, n1, dt FROM pxf_jdbc_hive_non_secure_types_table WHERE dt = '2015-03-06' -ORDER BY n1; - s1 | n1 | dt -----------------------------------+----+------------ - row1 | 1 | 2015-03-06 +ORDER BY n1, s1; + s1 | n1 | dt +---------------------------------+----+------------ + row1 | 1 | 2015-03-06 third_hive_row1 | 1 | 2015-03-06 - row2 | 2 | 2015-03-06 + row2 | 2 | 2015-03-06 third_hive_row2 | 2 | 2015-03-06 + row3 | 3 | 2015-03-06 third_hive_row3 | 3 | 2015-03-06 - row3 | 3 | 2015-03-06 + row4 | 4 | 2015-03-06 third_hive_row4 | 4 | 2015-03-06 - row4 | 4 | 2015-03-06 + row5 | 5 | 2015-03-06 third_hive_row5 | 5 | 2015-03-06 - row5 | 5 | 2015-03-06 + row6 | 6 | 2015-03-06 third_hive_row6 | 6 | 2015-03-06 - row6 | 6 | 2015-03-06 + row7 | 7 | 2015-03-06 third_hive_row7 | 7 | 2015-03-06 - row7 | 7 | 2015-03-06 - row8 | 8 | 2015-03-06 + row8 | 8 | 2015-03-06 third_hive_row8 | 8 | 2015-03-06 - row9 | 9 | 2015-03-06 + row9 | 9 | 2015-03-06 third_hive_row9 | 9 | 2015-03-06 + row10 | 10 | 2015-03-06 third_hive_row10 | 10 | 2015-03-06 - row10 | 10 | 2015-03-06 - row11 | 11 | 2015-03-06 - third_hive_row24_char_null | 11 | 2015-03-06 - row23_varchar_null | 11 | 2015-03-06 - row21_smallint_null | 11 | 2015-03-06 - row20_tinyint_null | 11 | 2015-03-06 - row19_bool_null | 11 | 2015-03-06 - row18_bigint_null | 11 | 2015-03-06 - row17_real_null | 11 | 2015-03-06 - row16_timestamp_null | 11 | 2015-03-06 - row14_double_null | 11 | 2015-03-06 - row12_text_null | 11 | 2015-03-06 - third_hive_row23_varchar_null | 11 | 2015-03-06 - third_hive_row21_smallint_null | 11 | 2015-03-06 - third_hive_row20_tinyint_null | 11 | 2015-03-06 + row11 | 11 | 2015-03-06 + row12_text_null | 11 | 2015-03-06 + row14_double_null | 11 | 2015-03-06 + row16_timestamp_null | 11 | 2015-03-06 + row17_real_null | 11 | 2015-03-06 + row18_bigint_null | 11 | 2015-03-06 + row19_bool_null | 11 | 2015-03-06 + row20_tinyint_null | 11 | 2015-03-06 + row21_smallint_null | 11 | 2015-03-06 + row23_varchar_null | 11 | 2015-03-06 + row24_char_null | 11 | 2015-03-06 third_hive_row11 | 11 | 2015-03-06 third_hive_row12_text_null | 11 | 2015-03-06 third_hive_row14_double_null | 11 | 2015-03-06 @@ -103,10 +100,13 @@ ORDER BY n1; third_hive_row17_real_null | 11 | 2015-03-06 third_hive_row18_bigint_null | 11 | 2015-03-06 third_hive_row19_bool_null | 11 | 2015-03-06 - row24_char_null | 11 | 2015-03-06 + third_hive_row20_tinyint_null | 11 | 2015-03-06 + third_hive_row21_smallint_null | 11 | 2015-03-06 + third_hive_row23_varchar_null | 11 | 2015-03-06 + third_hive_row24_char_null | 11 | 2015-03-06 + row15_decimal_null | 12 | 2015-03-06 third_hive_row15_decimal_null | 12 | 2015-03-06 - row15_decimal_null | 12 | 2015-03-06 + row13_int_null | | 2015-03-06 third_hive_row13_int_null | | 2015-03-06 - row13_int_null | | 2015-03-06 (46 rows) diff --git a/automation/tincrepo/main/pxf/features/jdbc/secured_and_non_secured_hive/sql/query02.sql b/automation/tincrepo/main/pxf/features/jdbc/secured_and_non_secured_hive/sql/query02.sql index 939c7d63f5..61370dcf13 100644 --- a/automation/tincrepo/main/pxf/features/jdbc/secured_and_non_secured_hive/sql/query02.sql +++ b/automation/tincrepo/main/pxf/features/jdbc/secured_and_non_secured_hive/sql/query02.sql @@ -5,4 +5,4 @@ SELECT s1, n1 FROM pxf_jdbc_hive_non_secure_types_table WHERE tn < 11 ORDER BY n SELECT s1, n1 FROM pxf_jdbc_hive_types_table WHERE tn < 11 UNION ALL SELECT s1, n1 FROM pxf_jdbc_hive_non_secure_types_table WHERE tn < 11 -ORDER BY n1; +ORDER BY n1, s1; diff --git a/automation/tincrepo/main/pxf/features/jdbc/secured_and_non_secured_hive/sql/query03.sql b/automation/tincrepo/main/pxf/features/jdbc/secured_and_non_secured_hive/sql/query03.sql index 1882ebfc6d..732f8e449b 100644 --- a/automation/tincrepo/main/pxf/features/jdbc/secured_and_non_secured_hive/sql/query03.sql +++ b/automation/tincrepo/main/pxf/features/jdbc/secured_and_non_secured_hive/sql/query03.sql @@ -1,8 +1,8 @@ -- @description query03 for Multiple JDBC Hive Server queries with timestamp filter -SELECT s1, n1, tm FROM pxf_jdbc_hive_types_table WHERE tm > '2013-07-23 21:00:01' ORDER BY n1; +SELECT s1, n1, tm FROM pxf_jdbc_hive_types_table WHERE tm > '2013-07-23 21:00:01' ORDER BY n1, s1; -SELECT s1, n1, tm FROM pxf_jdbc_hive_non_secure_types_table WHERE tm > '2013-07-23 21:00:01' ORDER BY n1; +SELECT s1, n1, tm FROM pxf_jdbc_hive_non_secure_types_table WHERE tm > '2013-07-23 21:00:01' ORDER BY n1, s1; SELECT s1, n1, tm FROM pxf_jdbc_hive_types_table WHERE tm > '2013-07-23 21:00:01' UNION ALL SELECT s1, n1, tm FROM pxf_jdbc_hive_non_secure_types_table WHERE tm > '2013-07-23 21:00:01' -ORDER BY n1; +ORDER BY n1, s1; diff --git a/automation/tincrepo/main/pxf/features/jdbc/secured_and_non_secured_hive/sql/query04.sql b/automation/tincrepo/main/pxf/features/jdbc/secured_and_non_secured_hive/sql/query04.sql index 802e991d76..c774976c1c 100644 --- a/automation/tincrepo/main/pxf/features/jdbc/secured_and_non_secured_hive/sql/query04.sql +++ b/automation/tincrepo/main/pxf/features/jdbc/secured_and_non_secured_hive/sql/query04.sql @@ -1,8 +1,8 @@ -- @description query04 for Multiple JDBC Hive Server queries with date filter -SELECT s1, n1, dt FROM pxf_jdbc_hive_types_table WHERE dt = '2015-03-06' ORDER BY n1; +SELECT s1, n1, dt FROM pxf_jdbc_hive_types_table WHERE dt = '2015-03-06' ORDER BY n1, s1; -SELECT s1, n1, dt FROM pxf_jdbc_hive_non_secure_types_table WHERE dt = '2015-03-06' ORDER BY n1; +SELECT s1, n1, dt FROM pxf_jdbc_hive_non_secure_types_table WHERE dt = '2015-03-06' ORDER BY n1, s1; SELECT s1, n1, dt FROM pxf_jdbc_hive_types_table WHERE dt = '2015-03-06' UNION ALL SELECT s1, n1, dt FROM pxf_jdbc_hive_non_secure_types_table WHERE dt = '2015-03-06' -ORDER BY n1; +ORDER BY n1, s1; diff --git a/automation/tincrepo/main/pxf/features/jdbc/two_secured_hive/expected/query02.ans b/automation/tincrepo/main/pxf/features/jdbc/two_secured_hive/expected/query02.ans index 525772377b..b84bce9204 100644 --- a/automation/tincrepo/main/pxf/features/jdbc/two_secured_hive/expected/query02.ans +++ b/automation/tincrepo/main/pxf/features/jdbc/two_secured_hive/expected/query02.ans @@ -33,28 +33,28 @@ SELECT s1, n1 FROM pxf_jdbc_hive_2_types_table WHERE tn < 11 ORDER BY n1; SELECT s1, n1 FROM pxf_jdbc_hive_types_table WHERE tn < 11 UNION ALL SELECT s1, n1 FROM pxf_jdbc_hive_2_types_table WHERE tn < 11 -ORDER BY n1; +ORDER BY n1, s1; s1 | n1 -------------------+---- - second_hive_row1 | 1 row1 | 1 - second_hive_row2 | 2 + second_hive_row1 | 1 row2 | 2 - second_hive_row3 | 3 + second_hive_row2 | 2 row3 | 3 + second_hive_row3 | 3 row4 | 4 second_hive_row4 | 4 - second_hive_row5 | 5 row5 | 5 + second_hive_row5 | 5 row6 | 6 second_hive_row6 | 6 - second_hive_row7 | 7 row7 | 7 - second_hive_row8 | 8 + second_hive_row7 | 7 row8 | 8 + second_hive_row8 | 8 row9 | 9 second_hive_row9 | 9 - second_hive_row10 | 10 row10 | 10 + second_hive_row10 | 10 (20 rows) diff --git a/automation/tincrepo/main/pxf/features/jdbc/two_secured_hive/expected/query03.ans b/automation/tincrepo/main/pxf/features/jdbc/two_secured_hive/expected/query03.ans index a89cdb2299..6139dff054 100644 --- a/automation/tincrepo/main/pxf/features/jdbc/two_secured_hive/expected/query03.ans +++ b/automation/tincrepo/main/pxf/features/jdbc/two_secured_hive/expected/query03.ans @@ -1,12 +1,11 @@ -- start_ignore -- end_ignore -- @description query03 for Multiple JDBC Hive Server queries with timestamp filter -SELECT s1, n1, tm FROM pxf_jdbc_hive_types_table WHERE tm > '2013-07-23 21:00:01' ORDER BY n1; +SELECT s1, n1, tm FROM pxf_jdbc_hive_types_table WHERE tm > '2013-07-23 21:00:01' ORDER BY n1, s1; s1 | n1 | tm ---------------------+----+--------------------- + row11 | 11 | 2013-07-23 21:00:05 row12_text_null | 11 | 2013-07-23 21:00:05 - row24_char_null | 11 | 2013-07-23 21:00:05 - row23_varchar_null | 11 | 2013-07-23 21:00:05 row14_double_null | 11 | 2013-07-23 21:00:05 row17_real_null | 11 | 2013-07-23 21:00:05 row18_bigint_null | 11 | 2013-07-23 21:00:05 @@ -14,17 +13,17 @@ SELECT s1, n1, tm FROM pxf_jdbc_hive_types_table WHERE tm > '2013-07-23 21:00:01 row20_tinyint_null | 11 | 2013-07-23 21:00:05 row21_smallint_null | 11 | 2013-07-23 21:00:05 row22_date_null | 11 | 2013-07-23 21:00:05 - row11 | 11 | 2013-07-23 21:00:05 + row23_varchar_null | 11 | 2013-07-23 21:00:05 + row24_char_null | 11 | 2013-07-23 21:00:05 row15_decimal_null | 12 | 2013-07-24 21:00:05 row13_int_null | | 2013-07-23 21:00:05 (13 rows) -SELECT s1, n1, tm FROM pxf_jdbc_hive_2_types_table WHERE tm > '2013-07-23 21:00:01' ORDER BY n1; +SELECT s1, n1, tm FROM pxf_jdbc_hive_2_types_table WHERE tm > '2013-07-23 21:00:01' ORDER BY n1, s1; s1 | n1 | tm ---------------------------------+----+--------------------- + second_hive_row11 | 11 | 2013-07-23 21:00:05 second_hive_row12_text_null | 11 | 2013-07-23 21:00:05 - second_hive_row24_char_null | 11 | 2013-07-23 21:00:05 - second_hive_row23_varchar_null | 11 | 2013-07-23 21:00:05 second_hive_row14_double_null | 11 | 2013-07-23 21:00:05 second_hive_row17_real_null | 11 | 2013-07-23 21:00:05 second_hive_row18_bigint_null | 11 | 2013-07-23 21:00:05 @@ -32,21 +31,19 @@ SELECT s1, n1, tm FROM pxf_jdbc_hive_2_types_table WHERE tm > '2013-07-23 21:00: second_hive_row20_tinyint_null | 11 | 2013-07-23 21:00:05 second_hive_row21_smallint_null | 11 | 2013-07-23 21:00:05 second_hive_row22_date_null | 11 | 2013-07-23 21:00:05 - second_hive_row11 | 11 | 2013-07-23 21:00:05 + second_hive_row23_varchar_null | 11 | 2013-07-23 21:00:05 + second_hive_row24_char_null | 11 | 2013-07-23 21:00:05 second_hive_row15_decimal_null | 12 | 2013-07-24 21:00:05 second_hive_row13_int_null | | 2013-07-23 21:00:05 (13 rows) SELECT s1, n1, tm FROM pxf_jdbc_hive_types_table WHERE tm > '2013-07-23 21:00:01' UNION ALL SELECT s1, n1, tm FROM pxf_jdbc_hive_2_types_table WHERE tm > '2013-07-23 21:00:01' -ORDER BY n1; +ORDER BY n1, s1; s1 | n1 | tm ---------------------------------+----+--------------------- + row11 | 11 | 2013-07-23 21:00:05 row12_text_null | 11 | 2013-07-23 21:00:05 - second_hive_row24_char_null | 11 | 2013-07-23 21:00:05 - second_hive_row23_varchar_null | 11 | 2013-07-23 21:00:05 - second_hive_row22_date_null | 11 | 2013-07-23 21:00:05 - second_hive_row21_smallint_null | 11 | 2013-07-23 21:00:05 row14_double_null | 11 | 2013-07-23 21:00:05 row17_real_null | 11 | 2013-07-23 21:00:05 row18_bigint_null | 11 | 2013-07-23 21:00:05 @@ -63,10 +60,13 @@ ORDER BY n1; second_hive_row18_bigint_null | 11 | 2013-07-23 21:00:05 second_hive_row19_bool_null | 11 | 2013-07-23 21:00:05 second_hive_row20_tinyint_null | 11 | 2013-07-23 21:00:05 - row11 | 11 | 2013-07-23 21:00:05 + second_hive_row21_smallint_null | 11 | 2013-07-23 21:00:05 + second_hive_row22_date_null | 11 | 2013-07-23 21:00:05 + second_hive_row23_varchar_null | 11 | 2013-07-23 21:00:05 + second_hive_row24_char_null | 11 | 2013-07-23 21:00:05 row15_decimal_null | 12 | 2013-07-24 21:00:05 second_hive_row15_decimal_null | 12 | 2013-07-24 21:00:05 - second_hive_row13_int_null | | 2013-07-23 21:00:05 row13_int_null | | 2013-07-23 21:00:05 + second_hive_row13_int_null | | 2013-07-23 21:00:05 (26 rows) diff --git a/automation/tincrepo/main/pxf/features/jdbc/two_secured_hive/expected/query04.ans b/automation/tincrepo/main/pxf/features/jdbc/two_secured_hive/expected/query04.ans index 616c829e59..bff696ff0a 100644 --- a/automation/tincrepo/main/pxf/features/jdbc/two_secured_hive/expected/query04.ans +++ b/automation/tincrepo/main/pxf/features/jdbc/two_secured_hive/expected/query04.ans @@ -1,7 +1,7 @@ -- start_ignore -- end_ignore -- @description query04 for Multiple JDBC Hive Server queries with date filter -SELECT s1, n1, dt FROM pxf_jdbc_hive_types_table WHERE dt = '2015-03-06' ORDER BY n1; +SELECT s1, n1, dt FROM pxf_jdbc_hive_types_table WHERE dt = '2015-03-06' ORDER BY n1, s1; s1 | n1 | dt ----------------------+----+------------ row1 | 1 | 2015-03-06 @@ -15,8 +15,7 @@ SELECT s1, n1, dt FROM pxf_jdbc_hive_types_table WHERE dt = '2015-03-06' ORDER B row9 | 9 | 2015-03-06 row10 | 10 | 2015-03-06 row11 | 11 | 2015-03-06 - row24_char_null | 11 | 2015-03-06 - row23_varchar_null | 11 | 2015-03-06 + row12_text_null | 11 | 2015-03-06 row14_double_null | 11 | 2015-03-06 row16_timestamp_null | 11 | 2015-03-06 row17_real_null | 11 | 2015-03-06 @@ -24,13 +23,14 @@ SELECT s1, n1, dt FROM pxf_jdbc_hive_types_table WHERE dt = '2015-03-06' ORDER B row19_bool_null | 11 | 2015-03-06 row20_tinyint_null | 11 | 2015-03-06 row21_smallint_null | 11 | 2015-03-06 - row12_text_null | 11 | 2015-03-06 + row23_varchar_null | 11 | 2015-03-06 + row24_char_null | 11 | 2015-03-06 row15_decimal_null | 12 | 2015-03-06 row13_int_null | | 2015-03-06 (23 rows) -SELECT s1, n1, dt FROM pxf_jdbc_hive_2_types_table WHERE dt = '2015-03-06' ORDER BY n1; - s1 | n1 | dt +SELECT s1, n1, dt FROM pxf_jdbc_hive_2_types_table WHERE dt = '2015-03-06' ORDER BY n1, s1; + s1 | n1 | dt ----------------------------------+----+------------ second_hive_row1 | 1 | 2015-03-06 second_hive_row2 | 2 | 2015-03-06 @@ -43,8 +43,7 @@ SELECT s1, n1, dt FROM pxf_jdbc_hive_2_types_table WHERE dt = '2015-03-06' ORDER second_hive_row9 | 9 | 2015-03-06 second_hive_row10 | 10 | 2015-03-06 second_hive_row11 | 11 | 2015-03-06 - second_hive_row24_char_null | 11 | 2015-03-06 - second_hive_row23_varchar_null | 11 | 2015-03-06 + second_hive_row12_text_null | 11 | 2015-03-06 second_hive_row14_double_null | 11 | 2015-03-06 second_hive_row16_timestamp_null | 11 | 2015-03-06 second_hive_row17_real_null | 11 | 2015-03-06 @@ -52,50 +51,48 @@ SELECT s1, n1, dt FROM pxf_jdbc_hive_2_types_table WHERE dt = '2015-03-06' ORDER second_hive_row19_bool_null | 11 | 2015-03-06 second_hive_row20_tinyint_null | 11 | 2015-03-06 second_hive_row21_smallint_null | 11 | 2015-03-06 - second_hive_row12_text_null | 11 | 2015-03-06 + second_hive_row23_varchar_null | 11 | 2015-03-06 + second_hive_row24_char_null | 11 | 2015-03-06 second_hive_row15_decimal_null | 12 | 2015-03-06 second_hive_row13_int_null | | 2015-03-06 (23 rows) SELECT s1, n1, dt FROM pxf_jdbc_hive_types_table WHERE dt = '2015-03-06' UNION ALL SELECT s1, n1, dt FROM pxf_jdbc_hive_2_types_table WHERE dt = '2015-03-06' -ORDER BY n1; - s1 | n1 | dt +ORDER BY n1, s1; + s1 | n1 | dt ----------------------------------+----+------------ row1 | 1 | 2015-03-06 second_hive_row1 | 1 | 2015-03-06 row2 | 2 | 2015-03-06 second_hive_row2 | 2 | 2015-03-06 - second_hive_row3 | 3 | 2015-03-06 row3 | 3 | 2015-03-06 - second_hive_row4 | 4 | 2015-03-06 + second_hive_row3 | 3 | 2015-03-06 row4 | 4 | 2015-03-06 - second_hive_row5 | 5 | 2015-03-06 + second_hive_row4 | 4 | 2015-03-06 row5 | 5 | 2015-03-06 - second_hive_row6 | 6 | 2015-03-06 + second_hive_row5 | 5 | 2015-03-06 row6 | 6 | 2015-03-06 - second_hive_row7 | 7 | 2015-03-06 + second_hive_row6 | 6 | 2015-03-06 row7 | 7 | 2015-03-06 + second_hive_row7 | 7 | 2015-03-06 row8 | 8 | 2015-03-06 second_hive_row8 | 8 | 2015-03-06 row9 | 9 | 2015-03-06 second_hive_row9 | 9 | 2015-03-06 - second_hive_row10 | 10 | 2015-03-06 row10 | 10 | 2015-03-06 + second_hive_row10 | 10 | 2015-03-06 row11 | 11 | 2015-03-06 - second_hive_row24_char_null | 11 | 2015-03-06 - row23_varchar_null | 11 | 2015-03-06 - row21_smallint_null | 11 | 2015-03-06 - row20_tinyint_null | 11 | 2015-03-06 - row19_bool_null | 11 | 2015-03-06 - row18_bigint_null | 11 | 2015-03-06 - row17_real_null | 11 | 2015-03-06 - row16_timestamp_null | 11 | 2015-03-06 - row14_double_null | 11 | 2015-03-06 row12_text_null | 11 | 2015-03-06 - second_hive_row23_varchar_null | 11 | 2015-03-06 - second_hive_row21_smallint_null | 11 | 2015-03-06 - second_hive_row20_tinyint_null | 11 | 2015-03-06 + row14_double_null | 11 | 2015-03-06 + row16_timestamp_null | 11 | 2015-03-06 + row17_real_null | 11 | 2015-03-06 + row18_bigint_null | 11 | 2015-03-06 + row19_bool_null | 11 | 2015-03-06 + row20_tinyint_null | 11 | 2015-03-06 + row21_smallint_null | 11 | 2015-03-06 + row23_varchar_null | 11 | 2015-03-06 + row24_char_null | 11 | 2015-03-06 second_hive_row11 | 11 | 2015-03-06 second_hive_row12_text_null | 11 | 2015-03-06 second_hive_row14_double_null | 11 | 2015-03-06 @@ -103,10 +100,13 @@ ORDER BY n1; second_hive_row17_real_null | 11 | 2015-03-06 second_hive_row18_bigint_null | 11 | 2015-03-06 second_hive_row19_bool_null | 11 | 2015-03-06 - row24_char_null | 11 | 2015-03-06 - second_hive_row15_decimal_null | 12 | 2015-03-06 + second_hive_row20_tinyint_null | 11 | 2015-03-06 + second_hive_row21_smallint_null | 11 | 2015-03-06 + second_hive_row23_varchar_null | 11 | 2015-03-06 + second_hive_row24_char_null | 11 | 2015-03-06 row15_decimal_null | 12 | 2015-03-06 - second_hive_row13_int_null | | 2015-03-06 + second_hive_row15_decimal_null | 12 | 2015-03-06 row13_int_null | | 2015-03-06 + second_hive_row13_int_null | | 2015-03-06 (46 rows) diff --git a/automation/tincrepo/main/pxf/features/jdbc/two_secured_hive/sql/query02.sql b/automation/tincrepo/main/pxf/features/jdbc/two_secured_hive/sql/query02.sql index e0d36d3855..8a3573ac04 100644 --- a/automation/tincrepo/main/pxf/features/jdbc/two_secured_hive/sql/query02.sql +++ b/automation/tincrepo/main/pxf/features/jdbc/two_secured_hive/sql/query02.sql @@ -5,4 +5,4 @@ SELECT s1, n1 FROM pxf_jdbc_hive_2_types_table WHERE tn < 11 ORDER BY n1; SELECT s1, n1 FROM pxf_jdbc_hive_types_table WHERE tn < 11 UNION ALL SELECT s1, n1 FROM pxf_jdbc_hive_2_types_table WHERE tn < 11 -ORDER BY n1; +ORDER BY n1, s1; diff --git a/automation/tincrepo/main/pxf/features/jdbc/two_secured_hive/sql/query03.sql b/automation/tincrepo/main/pxf/features/jdbc/two_secured_hive/sql/query03.sql index 38b7779c02..a1a23c311b 100644 --- a/automation/tincrepo/main/pxf/features/jdbc/two_secured_hive/sql/query03.sql +++ b/automation/tincrepo/main/pxf/features/jdbc/two_secured_hive/sql/query03.sql @@ -1,8 +1,8 @@ -- @description query03 for Multiple JDBC Hive Server queries with timestamp filter -SELECT s1, n1, tm FROM pxf_jdbc_hive_types_table WHERE tm > '2013-07-23 21:00:01' ORDER BY n1; +SELECT s1, n1, tm FROM pxf_jdbc_hive_types_table WHERE tm > '2013-07-23 21:00:01' ORDER BY n1, s1; -SELECT s1, n1, tm FROM pxf_jdbc_hive_2_types_table WHERE tm > '2013-07-23 21:00:01' ORDER BY n1; +SELECT s1, n1, tm FROM pxf_jdbc_hive_2_types_table WHERE tm > '2013-07-23 21:00:01' ORDER BY n1, s1; SELECT s1, n1, tm FROM pxf_jdbc_hive_types_table WHERE tm > '2013-07-23 21:00:01' UNION ALL SELECT s1, n1, tm FROM pxf_jdbc_hive_2_types_table WHERE tm > '2013-07-23 21:00:01' -ORDER BY n1; +ORDER BY n1, s1; diff --git a/automation/tincrepo/main/pxf/features/jdbc/two_secured_hive/sql/query04.sql b/automation/tincrepo/main/pxf/features/jdbc/two_secured_hive/sql/query04.sql index 43a7c70e41..b8931889dc 100644 --- a/automation/tincrepo/main/pxf/features/jdbc/two_secured_hive/sql/query04.sql +++ b/automation/tincrepo/main/pxf/features/jdbc/two_secured_hive/sql/query04.sql @@ -1,8 +1,8 @@ -- @description query04 for Multiple JDBC Hive Server queries with date filter -SELECT s1, n1, dt FROM pxf_jdbc_hive_types_table WHERE dt = '2015-03-06' ORDER BY n1; +SELECT s1, n1, dt FROM pxf_jdbc_hive_types_table WHERE dt = '2015-03-06' ORDER BY n1, s1; -SELECT s1, n1, dt FROM pxf_jdbc_hive_2_types_table WHERE dt = '2015-03-06' ORDER BY n1; +SELECT s1, n1, dt FROM pxf_jdbc_hive_2_types_table WHERE dt = '2015-03-06' ORDER BY n1, s1; SELECT s1, n1, dt FROM pxf_jdbc_hive_types_table WHERE dt = '2015-03-06' UNION ALL SELECT s1, n1, dt FROM pxf_jdbc_hive_2_types_table WHERE dt = '2015-03-06' -ORDER BY n1; +ORDER BY n1, s1; From 656ba7f7a23c68b043878c83b076da56ee28b6cd Mon Sep 17 00:00:00 2001 From: lucas bonner <66486962+lucasbonner@users.noreply.github.com> Date: Mon, 28 Aug 2023 11:53:43 -0500 Subject: [PATCH 13/24] CLA changes in CONTRIBUTING.md (#1016) Co-authored-by: Lucas Bonner --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 66e8b7526a..3c406b6f3f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,7 +4,7 @@ We warmly welcome and appreciate contributions from the community! By participating you agree to the [code of conduct](https://github.com/greenplum-db/pxf/blob/main/CODE-OF-CONDUCT.md). To contribute: -- Sign our [Contributor License Agreement](https://cla.pivotal.io/sign/greenplum). +- Sign our [Contributor License Agreement](https://cla.vmware.com/cla/1/preview). - Fork the PXF repository on GitHub. From 41713568f3a71f7223344ad851070bedf9e294c4 Mon Sep 17 00:00:00 2001 From: Himanshu Pandey Date: Mon, 28 Aug 2023 18:13:28 -0700 Subject: [PATCH 14/24] Add MultiNode Test Job for GP7 for dev pipeline (#1020) * Add MultiNode Test Job for GP7 for dev pipeline - Added variables to the Makefile (GP7_MULTINODE) and dev pipeline template (gp7_multinode), when set to true, this will introduce MultiNode tests for GP7. - In pxf_common.bash, `grep "JAVA_HOME"` command returs two occurrences of JAVA_HOME (for Rocky8) and the script to fails. To address this issue, I have modified the script to grep for a single occurrence of JAVA_HOME: `grep "export JAVA_HOME"` - Replaced the hardcoded `centos` user with the variable (DEFAULT_CCP_USER) which contains the user based on the OS. - `ssh-keyscan` doesn't work in Rocky8. Replaced it with awk command to set the localhost in `/home/gpadmin/.ssh/known_hosts`. - In GP7, `gpscp` is replaced by `gpsync`. Introduced a variable (GP_SCP_CMD) to specify the appropriate command based on GP version. - Added `gp6_python_libs` to set the PYTHONPATH in test_pxf_multinode.bash (same as test.bash). This is needed to run the Tinc Test. - The command used in the script to set the locale `sudo localedef -c -i ru_RU -f CP1251 ru_RU.CP1251` fails for Rocky8. The package `glibc-locale-source` is needed to successfully set the locale and was missing from ccp image for Rocky8. Added the `yum install` command in the script to install this package if missing from the vms. - Ran `yamllint` and `shellcheck` to confirm there are no issues with the new changes. Co-authored-by: Ed Espino --- concourse/Makefile | 2 + .../templates/dev_build_pipeline-tpl.yml | 160 +++++++++++++++++- concourse/scripts/install_hadoop.bash | 9 +- concourse/scripts/pxf_common.bash | 28 ++- concourse/scripts/test_pxf_multinode.bash | 47 +++-- concourse/tasks/test_pxf_on_ccp.yml | 3 + 6 files changed, 223 insertions(+), 26 deletions(-) diff --git a/concourse/Makefile b/concourse/Makefile index 5177890594..3e4cff35e0 100644 --- a/concourse/Makefile +++ b/concourse/Makefile @@ -38,6 +38,7 @@ MINIO ?= false OEL7 ?= false FILE ?= false GP7_CLI ?= false +GP7_MULTINODE ?= false SET_PIPELINE := set-pipeline ifeq ($(CHECK_CREDS), true) @@ -104,6 +105,7 @@ set-dev-build-pipeline: $(TEMPLATE_CMD) --template dev_build_pipeline-tpl.yml --vars \ slack_notification=$(SLACK) \ multinode=$(MULTINODE) \ + gp7_multinode=$(GP7_MULTINODE) \ multinode_no_impersonation=$(MULTINODE_NO_IMPERSONATION) \ jdk11=$(JDK11) \ cdh=$(CDH) \ diff --git a/concourse/pipelines/templates/dev_build_pipeline-tpl.yml b/concourse/pipelines/templates/dev_build_pipeline-tpl.yml index e302fa26fb..7fc04cf12d 100644 --- a/concourse/pipelines/templates/dev_build_pipeline-tpl.yml +++ b/concourse/pipelines/templates/dev_build_pipeline-tpl.yml @@ -73,7 +73,7 @@ anchors: ## ====================================================================== resource_types: -{% if multinode or multinode_no_impersonation or file or gp7_cli %} +{% if multinode or multinode_no_impersonation or file or gp7_cli or gp7_multinode %} - name: terraform type: registry-image source: @@ -83,7 +83,7 @@ resource_types: password: ((ud/pxf/secrets/pxf-cloudbuild-service-account-key)) {% endif %} -{% if multinode %} +{% if multinode or gp7_multinode %} - name: terraform-0.14.10 type: registry-image source: @@ -169,7 +169,7 @@ resources: branch: ((pxf-git-branch)) uri: ((ud/pxf/common/git-remote)) -{% if multinode or multinode_no_impersonation or file or gp7_cli %} +{% if multinode or multinode_no_impersonation or file or gp7_cli or gp7_multinode %} - name: ccp_src type: git source: @@ -236,7 +236,7 @@ resources: {% endif %} {% set gp_ver = None %} -{% if multinode or multinode_no_impersonation or ambari or file or gp7_cli %} +{% if multinode or multinode_no_impersonation or ambari or file or gp7_cli or gp7_multinode %} - name: ccp-7-image type: registry-image icon: docker @@ -350,7 +350,7 @@ resources: {% endfor %} ## ---------- Auxiliary Resources ---------- -{% if multinode or multinode_no_impersonation or file or gp7_cli %} +{% if multinode or multinode_no_impersonation or file or gp7_cli or gp7_multinode %} - name: terraform type: terraform source: @@ -368,7 +368,7 @@ resources: bucket_path: ((tf-bucket-path)) {% endif %} -{% if multinode %} +{% if multinode or gp7_multinode %} - name: terraform_ipa_hadoop type: terraform-0.14.10 source: @@ -1431,3 +1431,151 @@ jobs: <<: *slack_alert {% endif %} {% endif %} + +## ---------- Multi-node test for GP7 ---------- +{% if gp7_multinode %} +{% set gp_ver = 7 %} +- name: Test PXF-GP[[gp_ver]]-HDP2-SECURE-MULTI-IMPERS on RHEL8 + max_in_flight: 2 + plan: + - in_parallel: + - get: pxf_src + passed: [Build PXF-GP[[gp_ver]] on RHEL8] + trigger: true + - get: pxf_tarball + resource: pxf_gp[[gp_ver]]_tarball_rhel8 + passed: [Build PXF-GP[[gp_ver]] on RHEL8] + - get: gpdb_package + resource: gpdb[[gp_ver]]_rhel8_rpm_latest-0 + passed: [Build PXF-GP[[gp_ver]] on RHEL8] + - get: gpdb[[gp_ver]]-pxf-dev-rocky8-image + - get: ccp_src + - get: ccp-7-image + - get: pxf-automation-dependencies + - get: singlecluster + resource: singlecluster-hdp2 + - get: gp6-python-libs + - in_parallel: + - do: + - put: terraform_gpdb + resource: terraform + params: + action: create + delete_on_failure: true + generate_random_name: true + terraform_source: ccp_src/google/ + vars: + PLATFORM: rocky8-gpdb[[gp_ver]] + number_of_nodes: ((number_of_gpdb_nodes)) + extra_nodes: 1 + segments_per_host: 4 + instance_type: n1-standard-4 + ccp_reap_minutes: 120 + standby_master: true + - task: Generate Greenplum Cluster + input_mapping: + gpdb_rpm: gpdb_package + terraform: terraform_gpdb + file: ccp_src/ci/tasks/gen_cluster.yml + image: ccp-7-image + params: + AWS_ACCESS_KEY_ID: ((tf-machine-access-key-id)) + AWS_SECRET_ACCESS_KEY: ((tf-machine-secret-access-key)) + AWS_DEFAULT_REGION: ((ud/common/aws-region)) + BUCKET_PATH: ((tf-bucket-path)) + BUCKET_NAME: ((ud/pxf/common/tf-bucket-name)) + PLATFORM: rocky8-gpdb[[gp_ver]] + CLOUD_PROVIDER: google + GPDB_RPM: true + - in_parallel: + - task: Initialize Greenplum + file: ccp_src/ci/tasks/gpinitsystem.yml + - task: Install Hadoop + file: pxf_src/concourse/tasks/install_hadoop.yml + image: gpdb[[gp_ver]]-pxf-dev-rocky8-image + params: + ACCESS_KEY_ID: ((tf-machine-access-key-id)) + SECRET_ACCESS_KEY: ((tf-machine-secret-access-key)) + IMPERSONATION: ((enable-impersonation-multinode)) + - task: Generate Hadoop Cluster 1 + file: pxf_src/concourse/tasks/install_dataproc.yml + params: + GOOGLE_CREDENTIALS: ((ud/pxf/secrets/ccp-ci-service-account-key)) + GOOGLE_PROJECT_ID: ((ud/pxf/common/google-project-id)) + GOOGLE_ZONE: ((ud/pxf/common/google-zone)) + IMAGE_VERSION: ((dataproc-image-version)) + KERBEROS: ((kerberos-enabled)) + ccp_reap_minutes: 120 + - task: Generate Hadoop Cluster 2 + file: pxf_src/concourse/tasks/install_dataproc.yml + output_mapping: + dataproc_env_files: dataproc_2_env_files + params: + GOOGLE_CREDENTIALS: ((ud/pxf/secrets/kerberos-ccp-ci-service-account-key)) + GOOGLE_PROJECT_ID: ((ud/pxf/common/kerberos-google-project-id)) + GOOGLE_ZONE: ((ud/pxf/common/kerberos-google-zone)) + HADOOP_USER: gpuser + IMAGE_VERSION: ((dataproc-image-version)) + INITIALIZATION_SCRIPT: gs://data-gpdb-ud-kerberos-scripts/scripts/initialization-for-kerberos.sh + INSTANCE_TAGS: bosh-network,data-gpdb-ud-access + KERBEROS: ((kerberos-enabled)) + KEY: dataproc-kerberos-key + KEYRING: dataproc-kerberos + ccp_reap_minutes: 120 + NO_ADDRESS: false + PROXY_USER: gpuser + SECRETS_BUCKET: ((ud/pxf/secrets/kerberos-pxf-secrets-bucket-name)) + - do: # Generate IPA Hadoop cluster + - put: terraform_ipa_hadoop + params: + action: create + generate_random_name: true + terraform_source: pxf_src/concourse/terraform/ipa-multinode-hadoop + vars: + gcp_project: ((ud/pxf/common/ipa-google-project-id)) + - task: Generate Multinode Hadoop Cluster + file: pxf_src/concourse/tasks/install_multinode_hadoop.yml + image: gpdb[[gp_ver]]-pxf-dev-rocky8-image + params: + ANSIBLE_VAR_gcp_storage_bucket: ((ud/pxf/common/build-resources-bucket-name)) + ANSIBLE_VAR_ipa_password: ((ud/pxf/secrets/ipa-password)) + ANSIBLE_VAR_ssl_store_password: ((ud/pxf/secrets/ssl-store-password)) + - task: Setup PXF + input_mapping: + terraform: terraform_gpdb + file: pxf_src/concourse/tasks/install_pxf_on_ccp.yml + image: ccp-7-image + params: + IMPERSONATION: true + INSTALL_GPHDFS: false + GP_VER: [[gp_ver]] + KERBEROS: ((kerberos-enabled)) + PXF_JVM_OPTS: ((pxf-jvm-opts)) + PXF_BASE_DIR: /home/gpadmin/pxf-boot + - task: Test PXF-GP[[gp_ver]]-HDP2-SECURE-MULTI-IMPERS on RHEL8 + image: gpdb[[gp_ver]]-pxf-dev-rocky8-image + file: pxf_src/concourse/tasks/test_pxf_on_ccp.yml + attempts: 2 + params: + ACCESS_KEY_ID: ((tf-machine-access-key-id)) + SECRET_ACCESS_KEY: ((tf-machine-secret-access-key)) + HIVE_VERSION: 2 + IMPERSONATION: true + KERBEROS: ((kerberos-enabled)) + GOOGLE_PROJECT_ID: ((ud/pxf/common/google-project-id)) + GP_VER: [[gp_ver]] + GROUP: security,proxySecurity,proxySecurityIpa,multiClusterSecurity + PXF_JVM_OPTS: ((pxf-jvm-opts)) + PXF_BASE_DIR: /home/gpadmin/pxf-boot + on_success: + in_parallel: + steps: + - *destroy_dataproc_1 + - *destroy_dataproc_2 + - *destroy_gpdb_cluster + - *destroy_hadoop_cluster +{% if slack_notification %} + <<: *slack_alert +{% endif %} +{% endif %} # if gp7_multinode ends here +{% set gp_ver = None %} diff --git a/concourse/scripts/install_hadoop.bash b/concourse/scripts/install_hadoop.bash index 1f787f1def..24ae60e920 100755 --- a/concourse/scripts/install_hadoop.bash +++ b/concourse/scripts/install_hadoop.bash @@ -5,10 +5,11 @@ set -exuo pipefail CWDIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) source "${CWDIR}/pxf_common.bash" +set_ccp_os_user SSH_OPTS=(-i cluster_env_files/private_key.pem) LOCAL_GPHD_ROOT=/singlecluster inflate_singlecluster -REMOTE_GPHD_ROOT=~centos/singlecluster +REMOTE_GPHD_ROOT=~"${CCP_OS_USER}"/singlecluster function install_hadoop_single_cluster() { local hadoop_ip=${1} @@ -38,16 +39,16 @@ function install_hadoop_single_cluster() { groupadd supergroup && usermod -aG supergroup gpadmin && - source ~centos/pxf_common.bash && + source ~${CCP_OS_USER}/pxf_common.bash && export IMPERSONATION=${IMPERSONATION} && setup_impersonation ${REMOTE_GPHD_ROOT} && start_hadoop_services ${REMOTE_GPHD_ROOT} EOF local FILES_TO_SCP=(~/setup_hadoop.sh pxf_src/concourse/scripts/pxf_common.bash "${LOCAL_GPHD_ROOT}") - scp "${SSH_OPTS[@]}" -rq "${FILES_TO_SCP[@]}" centos@edw0: + scp "${SSH_OPTS[@]}" -rq "${FILES_TO_SCP[@]}" "${CCP_OS_USER}"@edw0: - ssh "${SSH_OPTS[@]}" centos@edw0 ' + ssh "${SSH_OPTS[@]}" "${CCP_OS_USER}"@edw0 ' sudo chmod +x ~/setup_hadoop.sh && sudo ~/setup_hadoop.sh ' diff --git a/concourse/scripts/pxf_common.bash b/concourse/scripts/pxf_common.bash index 1db48a4624..df9853deb5 100755 --- a/concourse/scripts/pxf_common.bash +++ b/concourse/scripts/pxf_common.bash @@ -9,6 +9,7 @@ PROXY_USER=${PROXY_USER:-pxfuser} PROTOCOL=${PROTOCOL:-} GOOGLE_PROJECT_ID=${GOOGLE_PROJECT_ID:-data-gpdb-ud} PXF_SRC=$(find /tmp/build -name pxf_src -type d) +CCP_OS_USER="" # on purpose do not call this PXF_CONF|PXF_BASE so that it is not set during pxf operations if [[ ${PXF_VERSION} == 5 ]]; then @@ -22,7 +23,8 @@ else fi if [[ -f ~/.pxfrc ]]; then - source <(grep JAVA_HOME ~/.pxfrc) + # shellcheck disable=SC1090 + source <(grep "export JAVA_HOME" ~/.pxfrc) echo "JAVA_HOME found in ${HOME}/.pxfrc, set to ${JAVA_HOME}..." else JAVA_HOME=$(find /usr/lib/jvm -name 'java-1.8.0-openjdk*' | head -1) @@ -267,7 +269,7 @@ function remote_access_to_gpdb() { cp cluster_env_files/.ssh/* /home/gpadmin/.ssh cp cluster_env_files/.ssh/*.pem /home/gpadmin/.ssh/id_rsa cp cluster_env_files/public_key.openssh /home/gpadmin/.ssh/authorized_keys - { ssh-keyscan localhost; ssh-keyscan 0.0.0.0; } >> /home/gpadmin/.ssh/known_hosts + awk '{print "localhost", $1, $2; print "0.0.0.0", $1, $2}' /etc/ssh/ssh_host_rsa_key.pub >> /home/gpadmin/.ssh/known_hosts ssh "${SSH_OPTS[@]}" gpadmin@cdw " source ${GPHOME}/greenplum_path.sh && export MASTER_DATA_DIRECTORY=${CDD_VALUE} && @@ -312,7 +314,7 @@ function setup_gpadmin_user() { ssh-keygen -t rsa -N "" -f ~gpadmin/.ssh/id_rsa cat /home/gpadmin/.ssh/id_rsa.pub >> ~gpadmin/.ssh/authorized_keys chmod 0600 /home/gpadmin/.ssh/authorized_keys - { ssh-keyscan localhost; ssh-keyscan 0.0.0.0; } >> ~gpadmin/.ssh/known_hosts + awk '{print "localhost", $1, $2; print "0.0.0.0", $1, $2}' /etc/ssh/ssh_host_rsa_key.pub >> ~gpadmin/.ssh/known_hosts chown -R gpadmin:gpadmin ${GPHOME} ~gpadmin/.ssh # don't chown cached dirs ~/.m2, etc. echo -e "password\npassword" | passwd gpadmin 2> /dev/null fi @@ -864,3 +866,23 @@ function setup_minio() { # export minio credentials as access environment variables export ACCESS_KEY_ID=${MINIO_ACCESS_KEY} SECRET_ACCESS_KEY=${MINIO_SECRET_KEY} } + +function set_ccp_os_user() { + metadata_file="cluster_env_files/terraform/metadata" + + # TODO: Remove the jq installation from here once available in the base image. + # Check if jq is installed + if ! rpm -q jq &> /dev/null; then + echo "jq is not installed. Installing jq..." + sudo yum install -y jq + fi + + # Check if the metadata file exists + if [ ! -e "$metadata_file" ]; then + echo "The $metadata_file file does not exist." + exit 2 + fi + + # shellcheck disable=SC2034 + CCP_OS_USER=$(jq -r '.ami_default_user' "$metadata_file") +} diff --git a/concourse/scripts/test_pxf_multinode.bash b/concourse/scripts/test_pxf_multinode.bash index 7bdfd579ad..485d31f6f1 100755 --- a/concourse/scripts/test_pxf_multinode.bash +++ b/concourse/scripts/test_pxf_multinode.bash @@ -14,8 +14,16 @@ else fi export PXF_HOME +# gpscp command is replaced with gpsync in GP7 +if [ "${GP_VER}" -lt 7 ]; then + GP_SCP_CMD=gpscp +else + GP_SCP_CMD=gpsync +fi + # shellcheck source=/dev/null source "${CWDIR}/pxf_common.bash" +set_ccp_os_user SSH_OPTS=(-i cluster_env_files/private_key.pem -o 'StrictHostKeyChecking=no') HADOOP_SSH_OPTS=(-o 'StrictHostKeyChecking=no') @@ -78,9 +86,10 @@ function update_pghba_conf() { function add_testing_encoding() { # install new encoding and restart Greenplum so that the new encoding is picked up by Greenplum + # TODO: Remove the glibc-locale-source installation from here once available in CCP ssh "${SSH_OPTS[@]}" gpadmin@cdw " source ${GPHOME}/greenplum_path.sh && - gpssh -f ~gpadmin/hostfile_all -v -u centos -s -e 'sudo localedef -c -i ru_RU -f CP1251 ru_RU.CP1251' && + gpssh -f ~gpadmin/hostfile_all -v -u ${CCP_OS_USER} -s -e 'if ! rpm -q glibc-locale-source >/dev/null 2>&1; then sudo yum install -y glibc-locale-source; fi; sudo localedef -c -i ru_RU -f CP1251 ru_RU.CP1251' && export MASTER_DATA_DIRECTORY=/data/gpdata/coordinator/gpseg-1 && gpstop -air " @@ -209,7 +218,7 @@ function setup_pxf_kerberos_on_cluster() { sudo mkdir -p /etc/security/keytabs sudo cp "${DATAPROC_DIR}/pxf.service.keytab" /etc/security/keytabs/gpadmin.headless.keytab sudo chown gpadmin:gpadmin /etc/security/keytabs/gpadmin.headless.keytab - scp centos@cdw:/etc/krb5.conf /tmp/krb5.conf + scp "${CCP_OS_USER}"@cdw:/etc/krb5.conf /tmp/krb5.conf sudo cp /tmp/krb5.conf /etc/krb5.conf # Add foreign dataproc hostfile to /etc/hosts @@ -284,11 +293,11 @@ function setup_pxf_kerberos_on_cluster() { # Add foreign dataproc hostfile to /etc/hosts on all nodes and copy keytab ssh gpadmin@cdw " source ${GPHOME}/greenplum_path.sh && - gpscp -f ~gpadmin/hostfile_all -v -r -u centos ~/dataproc_2_env_files/etc_hostfile =:/tmp/etc_hostfile && - gpssh -f ~gpadmin/hostfile_all -v -u centos -s -e 'sudo tee --append /etc/hosts < /tmp/etc_hostfile' && - gpscp -h cdw -v -r -u gpadmin ~/dataproc_2_env_files/pxf.service-cdw.keytab =:${BASE_DIR}/keytabs/pxf.service.2.keytab && - gpscp -h sdw1 -v -r -u gpadmin ~/dataproc_2_env_files/pxf.service-sdw1.keytab =:${BASE_DIR}/keytabs/pxf.service.2.keytab && - gpscp -h sdw2 -v -r -u gpadmin ~/dataproc_2_env_files/pxf.service-sdw2.keytab =:${BASE_DIR}/keytabs/pxf.service.2.keytab + ${GP_SCP_CMD} -f ~gpadmin/hostfile_all -v -r -u ${CCP_OS_USER} ~/dataproc_2_env_files/etc_hostfile =:/tmp/etc_hostfile && + gpssh -f ~gpadmin/hostfile_all -v -u ${CCP_OS_USER} -s -e 'sudo tee --append /etc/hosts < /tmp/etc_hostfile' && + ${GP_SCP_CMD} -h cdw -v -r -u gpadmin ~/dataproc_2_env_files/pxf.service-cdw.keytab =:${BASE_DIR}/keytabs/pxf.service.2.keytab && + ${GP_SCP_CMD} -h sdw1 -v -r -u gpadmin ~/dataproc_2_env_files/pxf.service-sdw1.keytab =:${BASE_DIR}/keytabs/pxf.service.2.keytab && + ${GP_SCP_CMD} -h sdw2 -v -r -u gpadmin ~/dataproc_2_env_files/pxf.service-sdw2.keytab =:${BASE_DIR}/keytabs/pxf.service.2.keytab " sudo cp "${DATAPROC_2_DIR}/pxf.service.keytab" /etc/security/keytabs/gpuser.headless.keytab sudo chown gpadmin:gpadmin /etc/security/keytabs/gpuser.headless.keytab @@ -355,10 +364,10 @@ function setup_pxf_kerberos_on_cluster() { # add foreign Hadoop and IPA KDC hostfile to /etc/hosts on all nodes ssh gpadmin@cdw " source ${GPHOME}/greenplum_path.sh && - gpscp -f ~gpadmin/hostfile_all -v -r -u centos ~/ipa_env_files/etc_hostfile =:/tmp/etc_hostfile && - gpssh -f ~gpadmin/hostfile_all -v -u centos -s -e 'sudo tee --append /etc/hosts < /tmp/etc_hostfile' && - gpscp -f ~gpadmin/hostfile_all -v -r -u gpadmin ~/ipa_env_files/pxf.service.keytab =:${BASE_DIR}/keytabs/pxf.service.3.keytab - gpscp -f ~gpadmin/hostfile_all -v -r -u gpadmin ~/ipa_env_files/hadoop.user.keytab =:${BASE_DIR}/keytabs/hadoop.user.3.keytab + ${GP_SCP_CMD} -f ~gpadmin/hostfile_all -v -r -u ${CCP_OS_USER} ~/ipa_env_files/etc_hostfile =:/tmp/etc_hostfile && + gpssh -f ~gpadmin/hostfile_all -v -u ${CCP_OS_USER} -s -e 'sudo tee --append /etc/hosts < /tmp/etc_hostfile' && + ${GP_SCP_CMD} -f ~gpadmin/hostfile_all -v -r -u gpadmin ~/ipa_env_files/pxf.service.keytab =:${BASE_DIR}/keytabs/pxf.service.3.keytab + ${GP_SCP_CMD} -f ~gpadmin/hostfile_all -v -r -u gpadmin ~/ipa_env_files/hadoop.user.keytab =:${BASE_DIR}/keytabs/hadoop.user.3.keytab " sudo cp "${HADOOP_3_DIR}/hadoop.user.keytab" "/etc/security/keytabs/${HADOOP_3_USER}.headless.keytab" @@ -531,8 +540,8 @@ function _main() { HDFS_BIN=/usr/bin elif grep "edw0" cluster_env_files/etc_hostfile; then HADOOP_HOSTNAME=hadoop - HADOOP_USER=centos - HDFS_BIN=~centos/singlecluster/bin + HADOOP_USER="${CCP_OS_USER}" + HDFS_BIN=~"${CCP_OS_USER}"/singlecluster/bin hadoop_ip=$(grep < cluster_env_files/etc_hostfile edw0 | awk '{print $1}') # tell hbase where to find zookeeper sed -i "/hbase.zookeeper.quorum<\/name>/ {n; s/127.0.0.1/${hadoop_ip}/}" \ @@ -572,6 +581,18 @@ function _main() { run_multinode_smoke_test 1000 fi + #TODO Remove this "if" block once Tinc is replaced with pg_regress. + # To run Tinc against GP7 we need to modify PYTHONPATH in $GPHOME/greenplum_path.sh since Tinc calls that script + # we will set PYTHONPATH to point to the set of python libs compiled with Python2 for GP6 + if [[ ${GP_VER} == 7 ]]; then + local gp6_python_libs=~gpadmin/python + { + echo "# Added by test.bash - Overriding PYTHONPATH to run the Tinc Tests in GP7" >> ${GPHOME}/greenplum_path.sh + echo "# Comment the following line out if you need to run GP utilities" >> ${GPHOME}/greenplum_path.sh + echo "export PYTHONPATH=${gp6_python_libs}" + } >> ${GPHOME}/greenplum_path.sh + fi + inflate_dependencies run_pxf_automation } diff --git a/concourse/tasks/test_pxf_on_ccp.yml b/concourse/tasks/test_pxf_on_ccp.yml index 5366887cce..235cea9908 100644 --- a/concourse/tasks/test_pxf_on_ccp.yml +++ b/concourse/tasks/test_pxf_on_ccp.yml @@ -16,6 +16,9 @@ inputs: optional: true - name: singlecluster optional: true +# TODO Remove gp6-python-libs input once Tinc is replaced with pg_regress. +- name: gp6-python-libs + optional: true params: ACCEPTANCE: From a8209850918132854ec20fe4f83992aa6f882633 Mon Sep 17 00:00:00 2001 From: "Bradford D. Boyle" Date: Tue, 29 Aug 2023 15:13:53 -0700 Subject: [PATCH 15/24] ci: fix issue in automation Makefile (#1021) For all of the protocols other than file, PXF requires $(PROTOCOL)-site.xml to configure the particular protocol (e.g., AWS credentials for s3 protocol). Rather then repeating the copying of the required template from $PXF_HOME, the automation Makefile has the copy command at the start of the shell script for the sync_cloud_config target.The configuration for the file protocol is done via pxf-site.xml instead of file-site.xml. This causes the copy command to attempt to copy a non-existent file-site.xml. This commit adds a check for the protocl before attempting to copy. The reason the failed copy doesn't fail the build in CI is that a multiline shell command is used in the recipe for sync_cloud_configs and this shell command does not include errexit. The shell keeps processing the multiline command despite the failing copy command and returns the last exist code which is 0 so make considers the command a success. This commit enables errexit to cause the shell to immediately exit if any command exits with a non-zero status. Authored-by: Bradford D. Boyle --- automation/Makefile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/automation/Makefile b/automation/Makefile index d2a0303acb..0db9571062 100755 --- a/automation/Makefile +++ b/automation/Makefile @@ -166,8 +166,11 @@ sync_jdbc_config: sync_cloud_configs: ifneq "$(PROTOCOL)" "" @mkdir -p $(PROTOCOL_HOME) - @if [ ! -f "$(PROTOCOL_HOME)/$(PROTOCOL)-site.xml" ]; then \ - cp $(TEMPLATES_DIR)/$(PROTOCOL)-site.xml $(PROTOCOL_HOME)/; \ + @ set -e; \ + if [ ! -f "$(PROTOCOL_HOME)/$(PROTOCOL)-site.xml" ]; then \ + if [ $(PROTOCOL) != file ]; then \ + cp $(TEMPLATES_DIR)/$(PROTOCOL)-site.xml $(PROTOCOL_HOME)/; \ + fi; \ cp $(TEMPLATES_DIR)/mapred-site.xml $(PROTOCOL_HOME)/; \ if [ $(PROTOCOL) = file ]; then \ if [ ! -d "$(BASE_PATH)" ]; then \ @@ -175,7 +178,7 @@ ifneq "$(PROTOCOL)" "" rm -rf $(PROTOCOL_HOME); \ exit 1; \ fi; \ - echo "Make sure your $PXF_BASE/conf/pxf-profiles.xml file configures the file:AvroSequenceFile and file:SequenceFile profiles"; \ + echo "Make sure your $$PXF_BASE/conf/pxf-profiles.xml file configures the file:AvroSequenceFile and file:SequenceFile profiles"; \ cp $(TEMPLATES_DIR)/pxf-site.xml $(PROTOCOL_HOME)/; \ sed $(SED_OPTS) 's||pxf.fs.basePath$(BASE_PATH)|g' $(PROTOCOL_HOME)/pxf-site.xml; \ fi; \ From b92e3297065a6ba244bd097614b106425d5ba670 Mon Sep 17 00:00:00 2001 From: Ashuka Xue Date: Wed, 30 Aug 2023 11:37:05 -0700 Subject: [PATCH 16/24] Add pxf.service.kerberos.ticket-renew-window option to pxf-site.xml (#1018) PXF currently renews the kerberos ticket only when at least 80% of the ticket lifespan has passed. This PR introduces a new property `pxf.service.kerberos.ticket-renew-window` that allows users to change the threshold at which we renew the ticket. The default value is 0.8f to maintain backwards compatibility. This value can be changed by adding the property into the pxf-site.xml file and can be different for each server config. For example, to renew the ticket after 30% of the ticket lifespan has passed, set the following: ``` pxf.service.kerberos.ticket-renew-window 0.3 ``` --- .../apache/hadoop/security/LoginSession.java | 30 +++-- .../security/PxfUserGroupInformation.java | 27 +++- .../pxf/api/security/SecureLogin.java | 4 +- .../hadoop/security/LoginSessionTest.java | 32 +++-- .../security/PxfUserGroupInformationTest.java | 126 ++++++++++++++---- .../pxf/api/security/SecureLoginTest.java | 22 +-- .../src/templates/templates/pxf-site.xml | 10 ++ 7 files changed, 189 insertions(+), 62 deletions(-) diff --git a/server/pxf-api/src/main/java/org/apache/hadoop/security/LoginSession.java b/server/pxf-api/src/main/java/org/apache/hadoop/security/LoginSession.java index 5f63f89140..7f55888006 100644 --- a/server/pxf-api/src/main/java/org/apache/hadoop/security/LoginSession.java +++ b/server/pxf-api/src/main/java/org/apache/hadoop/security/LoginSession.java @@ -21,6 +21,7 @@ public class LoginSession { private String principalName; private String keytabPath; private long kerberosMinMillisBeforeRelogin; + private float kerberosTicketRenewWindow; // derived fields stored to be re-used for subsequent requests private UserGroupInformation loginUser; @@ -33,7 +34,7 @@ public class LoginSession { * @param configDirectory server configuration directory */ public LoginSession(String configDirectory) { - this(configDirectory, null, null, null, null, 0L); + this(configDirectory, null, null, null, null, 0L, 0f); } /** @@ -43,18 +44,19 @@ public LoginSession(String configDirectory) { * @param loginUser UserGroupInformation for the given principal after login to Kerberos was performed */ public LoginSession(String configDirectory, UserGroupInformation loginUser) { - this(configDirectory, null, null, loginUser, null, 0L); + this(configDirectory, null, null, loginUser, null, 0L, 0f); } /** * Creates a new session object. * - * @param configDirectory server configuration directory - * @param principalName Kerberos principal name to use to obtain tokens - * @param keytabPath full path to a keytab file for the principal + * @param configDirectory server configuration directory + * @param principalName Kerberos principal name to use to obtain tokens + * @param keytabPath full path to a keytab file for the principal + * @param kerberosTicketRenewWindow the percentage of the ticket lifespan */ - public LoginSession(String configDirectory, String principalName, String keytabPath, long kerberosMinMillisBeforeRelogin) { - this(configDirectory, principalName, keytabPath, null, null, kerberosMinMillisBeforeRelogin); + public LoginSession(String configDirectory, String principalName, String keytabPath, long kerberosMinMillisBeforeRelogin, float kerberosTicketRenewWindow) { + this(configDirectory, principalName, keytabPath, null, null, kerberosMinMillisBeforeRelogin, kerberosTicketRenewWindow); } /** @@ -66,9 +68,10 @@ public LoginSession(String configDirectory, String principalName, String keytabP * @param loginUser UserGroupInformation for the given principal after login to Kerberos was performed * @param subject the subject * @param kerberosMinMillisBeforeRelogin the number of milliseconds before re-login + * @param kerberosTicketRenewWindow the percentage of the ticket lifespan */ public LoginSession(String configDirectory, String principalName, String keytabPath, UserGroupInformation loginUser, - Subject subject, long kerberosMinMillisBeforeRelogin) { + Subject subject, long kerberosMinMillisBeforeRelogin, float kerberosTicketRenewWindow) { this.configDirectory = configDirectory; this.principalName = principalName; this.keytabPath = keytabPath; @@ -78,6 +81,7 @@ public LoginSession(String configDirectory, String principalName, String keytabP this.user = subject.getPrincipals(User.class).iterator().next(); } this.kerberosMinMillisBeforeRelogin = kerberosMinMillisBeforeRelogin; + this.kerberosTicketRenewWindow = kerberosTicketRenewWindow; } /** @@ -134,6 +138,10 @@ public String getPrincipalName() { return principalName; } + public float getKerberosTicketRenewWindow() { + return kerberosTicketRenewWindow; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -143,12 +151,13 @@ public boolean equals(Object o) { return Objects.equals(configDirectory, that.configDirectory) && Objects.equals(principalName, that.principalName) && Objects.equals(keytabPath, that.keytabPath) && - kerberosMinMillisBeforeRelogin == that.kerberosMinMillisBeforeRelogin; + kerberosMinMillisBeforeRelogin == that.kerberosMinMillisBeforeRelogin && + kerberosTicketRenewWindow == that.kerberosTicketRenewWindow; } @Override public int hashCode() { - return Objects.hash(configDirectory, principalName, keytabPath, kerberosMinMillisBeforeRelogin); + return Objects.hash(configDirectory, principalName, keytabPath, kerberosMinMillisBeforeRelogin, kerberosTicketRenewWindow); } @Override @@ -158,6 +167,7 @@ public String toString() { .append("principal", principalName) .append("keytab", keytabPath) .append("kerberosMinMillisBeforeRelogin", kerberosMinMillisBeforeRelogin) + .append("kerberosTicketRenewWindow", kerberosTicketRenewWindow) .toString(); } } diff --git a/server/pxf-api/src/main/java/org/apache/hadoop/security/PxfUserGroupInformation.java b/server/pxf-api/src/main/java/org/apache/hadoop/security/PxfUserGroupInformation.java index 1802608480..c0688642f2 100644 --- a/server/pxf-api/src/main/java/org/apache/hadoop/security/PxfUserGroupInformation.java +++ b/server/pxf-api/src/main/java/org/apache/hadoop/security/PxfUserGroupInformation.java @@ -25,6 +25,7 @@ import java.util.function.Supplier; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN_DEFAULT; /** * PXF-specific analog of UserGroupInformation class to support multiple concurrent Kerberos sessions @@ -43,7 +44,8 @@ public class PxfUserGroupInformation { /** * Percentage of the ticket window to use before we renew ticket. */ - private static final float TICKET_RENEW_WINDOW = 0.80f; + public static final String CONFIG_KEY_TICKET_RENEW_WINDOW = "pxf.service.kerberos.ticket-renew-window"; + public static final float DEFAULT_TICKET_RENEW_WINDOW = 0.8f; private static final boolean windows = System.getProperty("os.name").startsWith("Windows"); private static final boolean is64Bit = System.getProperty("os.arch").contains("64") || @@ -106,7 +108,8 @@ public synchronized LoginSession loginUserFromKeytab(Configuration configuration // store all the relevant information in the login session and return it return new LoginSession(configDirectory, principal, keytabFilename, loginUser, subject, - getKerberosMinMillisBeforeRelogin(serverName, configuration)); + getKerberosMinMillisBeforeRelogin(serverName, configuration), + getKerberosTicketRenewWindow(serverName, configuration)); } catch (LoginException le) { KerberosAuthException kae = new KerberosAuthException(LOGIN_FAILURE, le); @@ -148,7 +151,7 @@ public void reloginFromKeytab(String serverName, LoginSession loginSession) thro Subject subject = loginSession.getSubject(); KerberosTicket tgt = getTGT(subject); //Return if TGT is valid and is not going to expire soon. - if (tgt != null && now < getRefreshTime(tgt)) { + if (tgt != null && now < getRefreshTime(tgt, loginSession.getKerberosTicketRenewWindow())) { return; } @@ -197,7 +200,7 @@ public void reloginFromKeytab(String serverName, LoginSession loginSession) thro public long getKerberosMinMillisBeforeRelogin(String serverName, Configuration configuration) { try { - return 1000L * configuration.getLong(HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN, 60L); + return 1000L * configuration.getLong(HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN, HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN_DEFAULT); } catch (NumberFormatException e) { throw new IllegalArgumentException(String.format("Invalid attribute value for %s of %s for server %s", HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN, @@ -206,6 +209,18 @@ public long getKerberosMinMillisBeforeRelogin(String serverName, Configuration c } } + public float getKerberosTicketRenewWindow(String serverName, Configuration configuration) { + float ticketRenewWindow = configuration.getFloat(CONFIG_KEY_TICKET_RENEW_WINDOW, DEFAULT_TICKET_RENEW_WINDOW); + if (ticketRenewWindow < 0f || ticketRenewWindow > 1f) { + throw new IllegalArgumentException( + String.format("Invalid value for %s of %f for server %s. Please choose a value between 0 and 1.", + CONFIG_KEY_TICKET_RENEW_WINDOW, + ticketRenewWindow, + serverName)); + } + return ticketRenewWindow; + } + // if the first kerberos ticket is not TGT, then remove and destroy it since // the kerberos library of jdk always use the first kerberos ticket as TGT. // See HADOOP-13433 for more details. @@ -237,10 +252,10 @@ private void fixKerberosTicketOrder(Subject subject) { LOG.warn("Warning, no kerberos tickets found while attempting to renew ticket"); } - private long getRefreshTime(KerberosTicket tgt) { + private long getRefreshTime(KerberosTicket tgt, float ticketRenewWindow) { long start = tgt.getStartTime().getTime(); long end = tgt.getEndTime().getTime(); - return start + (long) ((end - start) * TICKET_RENEW_WINDOW); + return start + (long) ((end - start) * ticketRenewWindow); } private static String getOSLoginModuleName() { diff --git a/server/pxf-api/src/main/java/org/greenplum/pxf/api/security/SecureLogin.java b/server/pxf-api/src/main/java/org/greenplum/pxf/api/security/SecureLogin.java index 2b2a81d08e..620ffe7a8d 100644 --- a/server/pxf-api/src/main/java/org/greenplum/pxf/api/security/SecureLogin.java +++ b/server/pxf-api/src/main/java/org/greenplum/pxf/api/security/SecureLogin.java @@ -218,11 +218,13 @@ private LoginSession getServerLoginSession(final String serverName, final String LoginSession expectedLoginSession; if (Utilities.isSecurityEnabled(configuration)) { long kerberosMinMillisBeforeRelogin = pxfUserGroupInformation.getKerberosMinMillisBeforeRelogin(serverName, configuration); + float kerberosTicketRenewWindow = pxfUserGroupInformation.getKerberosTicketRenewWindow(serverName, configuration); expectedLoginSession = new LoginSession( configDirectory, getServicePrincipal(serverName, configuration), getServiceKeytab(serverName, configuration), - kerberosMinMillisBeforeRelogin); + kerberosMinMillisBeforeRelogin, + kerberosTicketRenewWindow); } else { expectedLoginSession = new LoginSession(configDirectory); } diff --git a/server/pxf-api/src/test/java/org/apache/hadoop/security/LoginSessionTest.java b/server/pxf-api/src/test/java/org/apache/hadoop/security/LoginSessionTest.java index bea6f31bfd..0f4db2152f 100644 --- a/server/pxf-api/src/test/java/org/apache/hadoop/security/LoginSessionTest.java +++ b/server/pxf-api/src/test/java/org/apache/hadoop/security/LoginSessionTest.java @@ -54,30 +54,33 @@ public void setup() { public void testLoginSessionConfigurationConstructor() { session = new LoginSession("config"); assertEquals(0, session.getKerberosMinMillisBeforeRelogin()); + assertEquals(0, session.getKerberosTicketRenewWindow()); assertNull(session.getKeytabPath()); assertNull(session.getPrincipalName()); assertNull(session.getLoginUser()); assertNull(session.getSubject()); assertNull(session.getUser()); - assertEquals("LoginSession[config=config,principal=,keytab=,kerberosMinMillisBeforeRelogin=0]", session.toString()); + assertEquals("LoginSession[config=config,principal=,keytab=,kerberosMinMillisBeforeRelogin=0,kerberosTicketRenewWindow=0.0]", session.toString()); } @Test public void testLoginSessionConfigurationAndLoginUserConstructor() { session = new LoginSession("config", ugiFoo); assertEquals(0, session.getKerberosMinMillisBeforeRelogin()); + assertEquals(0, session.getKerberosTicketRenewWindow()); assertSame(ugiFoo, session.getLoginUser()); assertNull(session.getKeytabPath()); assertNull(session.getPrincipalName()); assertNull(session.getSubject()); assertNull(session.getUser()); - assertEquals("LoginSession[config=config,principal=,keytab=,kerberosMinMillisBeforeRelogin=0]", session.toString()); + assertEquals("LoginSession[config=config,principal=,keytab=,kerberosMinMillisBeforeRelogin=0,kerberosTicketRenewWindow=0.0]", session.toString()); } @Test public void testLoginSessionShortConstructor() { - session = new LoginSession("config", "principal", "keytab", 0); + session = new LoginSession("config", "principal", "keytab", 0, 0.8f); assertEquals(0, session.getKerberosMinMillisBeforeRelogin()); + assertEquals(0.8f, session.getKerberosTicketRenewWindow()); assertEquals("keytab", session.getKeytabPath()); assertEquals("principal", session.getPrincipalName()); assertNull(session.getLoginUser()); @@ -87,8 +90,9 @@ public void testLoginSessionShortConstructor() { @Test public void testLoginSessionFullConstructor() { - session = new LoginSession("config", "principal", "keytab", ugiFoo, subjectFoo, 1); + session = new LoginSession("config", "principal", "keytab", ugiFoo, subjectFoo, 1, 0.8f); assertEquals(1, session.getKerberosMinMillisBeforeRelogin()); + assertEquals(0.8f, session.getKerberosTicketRenewWindow()); assertEquals("keytab", session.getKeytabPath()); assertEquals("principal", session.getPrincipalName()); assertSame(ugiFoo, session.getLoginUser()); @@ -98,35 +102,39 @@ public void testLoginSessionFullConstructor() { @Test public void testEquality() { - sessionFoo = new LoginSession("config", "principal", "keytab", ugiFoo, subjectFoo, 1); + sessionFoo = new LoginSession("config", "principal", "keytab", ugiFoo, subjectFoo, 1, 0.8f); - sessionBar = new LoginSession("config", "principal", "keytab", ugiBar, subjectBar, 1); + sessionBar = new LoginSession("config", "principal", "keytab", ugiBar, subjectBar, 1, 0.8f); assertEquals(sessionFoo, sessionBar); assertEquals(sessionFoo.hashCode(), sessionBar.hashCode()); - sessionBar = new LoginSession("DIFFERENT", "principal", "keytab", ugiBar, subjectBar, 1); + sessionBar = new LoginSession("DIFFERENT", "principal", "keytab", ugiBar, subjectBar, 1, 0.8f); assertNotEquals(sessionFoo, sessionBar); assertNotEquals(sessionFoo.hashCode(), sessionBar.hashCode()); - sessionBar = new LoginSession("config", "DIFFERENT", "keytab", ugiBar, subjectBar, 1); + sessionBar = new LoginSession("config", "DIFFERENT", "keytab", ugiBar, subjectBar, 1, 0.8f); assertNotEquals(sessionFoo, sessionBar); assertNotEquals(sessionFoo.hashCode(), sessionBar.hashCode()); - sessionBar = new LoginSession("config", "principal", "DIFFERENT", ugiBar, subjectBar, 1); + sessionBar = new LoginSession("config", "principal", "DIFFERENT", ugiBar, subjectBar, 1, 0.8f); assertNotEquals(sessionFoo, sessionBar); assertNotEquals(sessionFoo.hashCode(), sessionBar.hashCode()); - sessionBar = new LoginSession("config", "principal", "keytab", ugiBar, subjectBar, 999); + sessionBar = new LoginSession("config", "principal", "keytab", ugiBar, subjectBar, 999, 0.8f); + assertNotEquals(sessionFoo, sessionBar); + assertNotEquals(sessionFoo.hashCode(), sessionBar.hashCode()); + + sessionBar = new LoginSession("config", "principal", "keytab", ugiBar, subjectBar, 1, 0.4f); assertNotEquals(sessionFoo, sessionBar); assertNotEquals(sessionFoo.hashCode(), sessionBar.hashCode()); } @Test public void testToString() { - session = new LoginSession("config", "principal", "keytab", ugiFoo, subjectFoo, 1); - assertEquals("LoginSession[config=config,principal=principal,keytab=keytab,kerberosMinMillisBeforeRelogin=1]", session.toString()); + session = new LoginSession("config", "principal", "keytab", ugiFoo, subjectFoo, 1, 0.8f); + assertEquals("LoginSession[config=config,principal=principal,keytab=keytab,kerberosMinMillisBeforeRelogin=1,kerberosTicketRenewWindow=0.8]", session.toString()); } private static void resetProperty(String key, String val) { diff --git a/server/pxf-api/src/test/java/org/apache/hadoop/security/PxfUserGroupInformationTest.java b/server/pxf-api/src/test/java/org/apache/hadoop/security/PxfUserGroupInformationTest.java index e2316e0c36..3290c4c18f 100644 --- a/server/pxf-api/src/test/java/org/apache/hadoop/security/PxfUserGroupInformationTest.java +++ b/server/pxf-api/src/test/java/org/apache/hadoop/security/PxfUserGroupInformationTest.java @@ -111,14 +111,7 @@ public void testLoginFromKeytabMinMillisFromConfig() throws Exception { // assert that the login session was created with properly wired up ugi/subject/user/loginContext assertEquals(33000, session.getKerberosMinMillisBeforeRelogin()); // will pick from configuration - assertEquals("/path/to/keytab", session.getKeytabPath()); - assertEquals("principal/some.host.com@EXAMPLE.COM", session.getPrincipalName()); - assertEquals(ugi, session.getLoginUser()); // UGI equality only compares enclosed subjects - assertNotSame(ugi, session.getLoginUser()); // UGI equality only compares enclosed subjects - assertSame(subject, session.getSubject()); - assertSame(user, session.getUser()); - assertSame(mockLoginContext, session.getUser().getLogin()); - assertEquals(UserGroupInformation.AuthenticationMethod.KERBEROS, session.getLoginUser().getAuthenticationMethod()); + assertSessionInfo(session, mockLoginContext, ugi, subject, user); // verify that login() was called verify(mockLoginContext).login(); @@ -132,14 +125,49 @@ public void testLoginFromKeytabMinMillisFromDefault() throws Exception { // assert that the login session was created with properly wired up ugi/subject/user/loginContext assertEquals(60000, session.getKerberosMinMillisBeforeRelogin()); // will pick from default - assertEquals("/path/to/keytab", session.getKeytabPath()); - assertEquals("principal/some.host.com@EXAMPLE.COM", session.getPrincipalName()); - assertEquals(ugi, session.getLoginUser()); // UGI equality only compares enclosed subjects - assertNotSame(ugi, session.getLoginUser()); // UGI equality only compares enclosed subjects - assertSame(subject, session.getSubject()); - assertSame(user, session.getUser()); - assertSame(mockLoginContext, session.getUser().getLogin()); - assertEquals(UserGroupInformation.AuthenticationMethod.KERBEROS, session.getLoginUser().getAuthenticationMethod()); + assertSessionInfo(session, mockLoginContext, ugi, subject, user); + + // verify that login() was called + verify(mockLoginContext).login(); + } + + @Test + public void testLoginFromKeytabRenewWindowFromConfig() throws Exception { + configuration.setFloat("pxf.service.kerberos.ticket-renew-window", 0.2f); + ugi = new UserGroupInformation(subject); + + session = pxfUserGroupInformation.loginUserFromKeytab(configuration, "server", "config-dir", "principal/some.host.com@EXAMPLE.COM", "/path/to/keytab"); + + // assert that the login session was created with properly wired up ugi/subject/user/loginContext + assertEquals(0.2f, session.getKerberosTicketRenewWindow()); // will pick from configuration + assertSessionInfo(session, mockLoginContext, ugi, subject, user); + + // verify that login() was called + verify(mockLoginContext).login(); + } + + @Test + public void testLoginFromKeytabRenewWindowDefault() throws Exception { + ugi = new UserGroupInformation(subject); + + session = pxfUserGroupInformation.loginUserFromKeytab(configuration, "server", "config-dir", "principal/some.host.com@EXAMPLE.COM", "/path/to/keytab"); + + // assert that the login session was created with properly wired up ugi/subject/user/loginContext + assertEquals(0.8f, session.getKerberosTicketRenewWindow()); // will pick from default + assertSessionInfo(session, mockLoginContext, ugi, subject, user); + + // verify that login() was called + verify(mockLoginContext).login(); + } + + @Test + public void testLoginFromKeytabRenewWindowInvalidValue() throws Exception { + configuration.set("pxf.service.kerberos.ticket-renew-window", "1.2"); + ugi = new UserGroupInformation(subject); + + Exception e = assertThrows(IllegalArgumentException.class, + () -> pxfUserGroupInformation.loginUserFromKeytab(configuration, "server", "config-dir", "principal/some.host.com@EXAMPLE.COM", "/path/to/keytab")); + assertEquals("Invalid value for pxf.service.kerberos.ticket-renew-window of 1.200000 for server server. Please choose a value between 0 and 1.", e.getMessage()); // verify that login() was called verify(mockLoginContext).login(); @@ -152,7 +180,7 @@ public void testReloginFromKeytabNoopForNonKerberos() throws KerberosAuthExcepti user.setLogin(mockLoginContext); ugi = new UserGroupInformation(subjectWithKerberosKeyTab); // do NOT set authentication method of UGI to KERBEROS, will cause NOOP for relogin - session = new LoginSession("config", "principal", "keytab", ugi, subjectWithKerberosKeyTab, 1); + session = new LoginSession("config", "principal", "keytab", ugi, subjectWithKerberosKeyTab, 1, 0.8f); pxfUserGroupInformation.reloginFromKeytab(serverName, session); @@ -164,7 +192,7 @@ public void testReloginFromKeytabNoopForNonKeytab() throws KerberosAuthException user.setLogin(mockLoginContext); ugi = new UserGroupInformation(subject); ugi.setAuthenticationMethod(UserGroupInformation.AuthenticationMethod.KERBEROS); - session = new LoginSession("config", "principal", "keytab", ugi, subject, 1); + session = new LoginSession("config", "principal", "keytab", ugi, subject, 1, 0.8f); pxfUserGroupInformation.reloginFromKeytab(serverName, session); @@ -178,7 +206,7 @@ public void testReloginFromKeytabNoopInsufficientTimeElapsed() throws KerberosAu ugi.setAuthenticationMethod(UserGroupInformation.AuthenticationMethod.KERBEROS); user.setLastLogin(nowMs); // simulate just logged in // set 33 secs between re-login attempts - session = new LoginSession("config", "principal", "keytab", ugi, subjectWithKerberosKeyTab, 55000L); + session = new LoginSession("config", "principal", "keytab", ugi, subjectWithKerberosKeyTab, 55000L, 0.8f); pxfUserGroupInformation.reloginFromKeytab(serverName, session); @@ -197,7 +225,7 @@ public void testReloginFromKeytabNoopTGTValidForLongTime() throws KerberosAuthEx ugi = new UserGroupInformation(subjectWithKerberosKeyTab); ugi.setAuthenticationMethod(UserGroupInformation.AuthenticationMethod.KERBEROS); // leave user.lastLogin at 0 to simulate old login - session = new LoginSession("config", "principal", "keytab", ugi, subjectWithKerberosKeyTab, 1); + session = new LoginSession("config", "principal", "keytab", ugi, subjectWithKerberosKeyTab, 1, 0.8f); pxfUserGroupInformation.reloginFromKeytab(serverName, session); @@ -210,7 +238,7 @@ public void testReloginFromKeytabFailsNoLogin() { ugi = new UserGroupInformation(subjectWithKerberosKeyTab); ugi.setAuthenticationMethod(UserGroupInformation.AuthenticationMethod.KERBEROS); // leave user.lastLogin at 0 to simulate old login - session = new LoginSession("config", "principal", "keytab", ugi, subjectWithKerberosKeyTab, 1); + session = new LoginSession("config", "principal", "keytab", ugi, subjectWithKerberosKeyTab, 1, 0.8f); Exception e = assertThrows(KerberosAuthException.class, () -> pxfUserGroupInformation.reloginFromKeytab(serverName, session)); @@ -223,7 +251,7 @@ public void testReloginFromKeytabFailsNoKeytab() { ugi = new UserGroupInformation(subjectWithKerberosKeyTab); ugi.setAuthenticationMethod(UserGroupInformation.AuthenticationMethod.KERBEROS); // leave user.lastLogin at 0 to simulate old login - session = new LoginSession("config", "principal", null, ugi, subjectWithKerberosKeyTab, 1); + session = new LoginSession("config", "principal", null, ugi, subjectWithKerberosKeyTab, 1, 0.8f); Exception e = assertThrows(KerberosAuthException.class, () -> pxfUserGroupInformation.reloginFromKeytab(serverName, session)); @@ -243,7 +271,7 @@ public void testReloginFromKeytabNoValidTGT() throws Exception { ugi = new UserGroupInformation(subjectWithKerberosKeyTab); ugi.setAuthenticationMethod(UserGroupInformation.AuthenticationMethod.KERBEROS); // leave user.lastLogin at 0 to simulate old login - session = new LoginSession("config", "principal", "keytab", ugi, subjectWithKerberosKeyTab, 1); + session = new LoginSession("config", "principal", "keytab", ugi, subjectWithKerberosKeyTab, 1, 0.8f); // train to return another LoginContext when it is constructed during re-login mockAnotherLoginContext = mock(LoginContext.class); @@ -277,12 +305,49 @@ public void testReloginFromKeytabValidTGTWillExpireSoon() throws Exception { ugi = new UserGroupInformation(subjectWithKerberosKeyTab); ugi.setAuthenticationMethod(UserGroupInformation.AuthenticationMethod.KERBEROS); // leave user.lastLogin at 0 to simulate old login - session = new LoginSession("config", "principal", "keytab", ugi, subjectWithKerberosKeyTab, 1); + session = new LoginSession("config", "principal", "keytab", ugi, subjectWithKerberosKeyTab, 1, 0.8f); + + // train to return another LoginContext when it is constructed during re-login + mockAnotherLoginContext = mock(LoginContext.class); + when(mockLoginContextProvider.newLoginContext(anyString(), any(), any())).thenReturn(mockAnotherLoginContext); + + pxfUserGroupInformation.reloginFromKeytab(serverName, session); + + assertNotSame(mockLoginContext, user.getLogin()); + assertSame(mockAnotherLoginContext, user.getLogin()); + assertTrue(user.getLastLogin() > 0); // login timestamp is updated + + verify(mockLoginContext).logout(); + verify(mockAnotherLoginContext).login(); + } + + @Test + public void testReloginFromKeytabDifferentRenewWindow() throws Exception { + user.setLogin(mockLoginContext); + when(mockTGT.getServer()).thenReturn(tgtPrincipal); + + // TGT validity started 1 hr ago, valid for another 1 hr, we are at 6/12 or 50% which is less + // than the default renew window of 80% + when(mockTGT.getStartTime()).thenReturn(new Date(nowMs - 3600 * 1000L)); + when(mockTGT.getEndTime()).thenReturn(new Date(nowMs + 3600 * 1000L)); + + ugi = new UserGroupInformation(subjectWithKerberosKeyTab); + ugi.setAuthenticationMethod(UserGroupInformation.AuthenticationMethod.KERBEROS); + // leave user.lastLogin at 0 to simulate old login + session = new LoginSession("config", "principal", "keytab", ugi, subjectWithKerberosKeyTab, 1, 0.8f); + + // with the default threshold, there should be no change to the login + pxfUserGroupInformation.reloginFromKeytab(serverName, session); + + assertSame(mockLoginContext, user.getLogin()); + assertTrue(user.getLastLogin() == 0); // login timestamp is updated // train to return another LoginContext when it is constructed during re-login mockAnotherLoginContext = mock(LoginContext.class); when(mockLoginContextProvider.newLoginContext(anyString(), any(), any())).thenReturn(mockAnotherLoginContext); + // change the renew threshold to 50% + session = new LoginSession("config", "principal", "keytab", ugi, subjectWithKerberosKeyTab, 1, 0.5f); pxfUserGroupInformation.reloginFromKeytab(serverName, session); assertNotSame(mockLoginContext, user.getLogin()); @@ -302,7 +367,7 @@ public void testReloginFromKeytabThrowsExceptionOnLoginFailure() throws Exceptio ugi = new UserGroupInformation(subjectWithKerberosKeyTab); ugi.setAuthenticationMethod(UserGroupInformation.AuthenticationMethod.KERBEROS); // leave user.lastLogin at 0 to simulate old login - session = new LoginSession("config", "principal", "keytab", ugi, subjectWithKerberosKeyTab, 1); + session = new LoginSession("config", "principal", "keytab", ugi, subjectWithKerberosKeyTab, 1, 0.8f); // train to return another LoginContext when it is constructed during re-login mockAnotherLoginContext = mock(LoginContext.class); @@ -314,6 +379,17 @@ public void testReloginFromKeytabThrowsExceptionOnLoginFailure() throws Exceptio assertEquals("Login failure for principal: principal from keytab keytab javax.security.auth.login.LoginException: foo", e.getMessage()); } + private static void assertSessionInfo(LoginSession session, LoginContext loginContext, UserGroupInformation ugi, Subject subject, User user) { + assertEquals("/path/to/keytab", session.getKeytabPath()); + assertEquals("principal/some.host.com@EXAMPLE.COM", session.getPrincipalName()); + assertEquals(ugi, session.getLoginUser()); // UGI equality only compares enclosed subjects + assertNotSame(ugi, session.getLoginUser()); // UGI equality only compares enclosed subjects + assertSame(subject, session.getSubject()); + assertSame(user, session.getUser()); + assertSame(loginContext, session.getUser().getLogin()); + assertEquals(UserGroupInformation.AuthenticationMethod.KERBEROS, session.getLoginUser().getAuthenticationMethod()); + } + private static void resetProperty(String key, String val) { if (val != null) { System.setProperty(key, val); diff --git a/server/pxf-api/src/test/java/org/greenplum/pxf/api/security/SecureLoginTest.java b/server/pxf-api/src/test/java/org/greenplum/pxf/api/security/SecureLoginTest.java index 7bbb5b5d1a..ca4cb5e745 100644 --- a/server/pxf-api/src/test/java/org/greenplum/pxf/api/security/SecureLoginTest.java +++ b/server/pxf-api/src/test/java/org/greenplum/pxf/api/security/SecureLoginTest.java @@ -136,7 +136,7 @@ public void testLoginNoKerberosNoServiceUserWhenConfigurationValuesAreProvided() @Test public void testLoginNoKerberosWithServiceUser() throws IOException { - expectedLoginSession = new LoginSession("config", null, null, null, null, 0); + expectedLoginSession = new LoginSession("config", null, null, null, null, 0, 0); configuration.set("pxf.service.user.name", "foo"); UserGroupInformation loginUGI = secureLogin.getLoginUser("server", "config", configuration); @@ -174,7 +174,7 @@ public void testLoginKerberosFailsWhenNoKeytab() { @Test public void testLoginKerberosFirstTime() throws IOException { expectedUGI = UserGroupInformation.createUserForTesting("some", new String[]{}); - expectedLoginSession = new LoginSession("config", "principal", "/path/to/keytab", expectedUGI, null, 0); + expectedLoginSession = new LoginSession("config", "principal", "/path/to/keytab", expectedUGI, null, 0, 0.8f); configuration.set("hadoop.security.authentication", "kerberos"); configuration.set(PROPERTY_KEY_SERVICE_PRINCIPAL, "principal"); configuration.set(PROPERTY_KEY_SERVICE_KEYTAB, "/path/to/keytab"); @@ -198,13 +198,14 @@ public void testLoginKerberosFirstTime() throws IOException { @Test public void testLoginKerberosSameSession() throws IOException { expectedUGI = UserGroupInformation.createUserForTesting("some", new String[]{}); - expectedLoginSession = new LoginSession("config", "principal", "/path/to/keytab", expectedUGI, null, 90000); + expectedLoginSession = new LoginSession("config", "principal", "/path/to/keytab", expectedUGI, null, 90000, 0.8f); configuration.set("hadoop.security.authentication", "kerberos"); configuration.set(PROPERTY_KEY_SERVICE_PRINCIPAL, "principal"); configuration.set(PROPERTY_KEY_SERVICE_KEYTAB, "/path/to/keytab"); configuration.set("hadoop.kerberos.min.seconds.before.relogin", "90"); when(pxfUserGroupInformationMock.loginUserFromKeytab(configuration, "server", "config", "principal", "/path/to/keytab")).thenReturn(expectedLoginSession); when(pxfUserGroupInformationMock.getKerberosMinMillisBeforeRelogin("server", configuration)).thenReturn(90000L); + when(pxfUserGroupInformationMock.getKerberosTicketRenewWindow("server", configuration)).thenReturn(0.8f); UserGroupInformation loginUGI = secureLogin.getLoginUser("server", "config", configuration); @@ -226,6 +227,7 @@ public void testLoginKerberosSameSession() throws IOException { verify(pxfUserGroupInformationMock).loginUserFromKeytab(configuration, "server", "config", "principal", "/path/to/keytab"); verify(pxfUserGroupInformationMock).getKerberosMinMillisBeforeRelogin("server", configuration); + verify(pxfUserGroupInformationMock).getKerberosTicketRenewWindow("server", configuration); // 1 extra relogin call verify(pxfUserGroupInformationMock, times(2)).reloginFromKeytab("server", expectedLoginSession); @@ -236,7 +238,7 @@ public void testLoginKerberosSameSession() throws IOException { public void testLoginKerberosDifferentServer() throws IOException { expectedUGI = UserGroupInformation.createUserForTesting("some", new String[]{}); - expectedLoginSession = new LoginSession("config", "principal", "/path/to/keytab", expectedUGI, null, 0); + expectedLoginSession = new LoginSession("config", "principal", "/path/to/keytab", expectedUGI, null, 0, 0.8f); configuration.set("hadoop.security.authentication", "kerberos"); configuration.set(PROPERTY_KEY_SERVICE_PRINCIPAL, "principal"); @@ -254,7 +256,7 @@ public void testLoginKerberosDifferentServer() throws IOException { // ------------------- now login for another server ------------------------- - LoginSession expectedDiffLoginSession = new LoginSession("diff-config", "principal", "/path/to/keytab", expectedUGI, null, 0); + LoginSession expectedDiffLoginSession = new LoginSession("diff-config", "principal", "/path/to/keytab", expectedUGI, null, 0, 0.8f); Configuration diffConfiguration = new Configuration(); diffConfiguration.set("hadoop.security.authentication", "kerberos"); diffConfiguration.set(PROPERTY_KEY_SERVICE_PRINCIPAL, "principal"); @@ -283,13 +285,14 @@ public void testLoginKerberosDifferentServer() throws IOException { public void testLoginKerberosSameServerDifferentPrincipal() throws IOException { expectedUGI = UserGroupInformation.createUserForTesting("some", new String[]{}); - expectedLoginSession = new LoginSession("config", "principal", "/path/to/keytab", expectedUGI, null, 0); + expectedLoginSession = new LoginSession("config", "principal", "/path/to/keytab", expectedUGI, null, 0, 0.8f); configuration.set("hadoop.security.authentication", "kerberos"); configuration.set(PROPERTY_KEY_SERVICE_PRINCIPAL, "principal"); configuration.set(PROPERTY_KEY_SERVICE_KEYTAB, "/path/to/keytab"); configuration.set("hadoop.kerberos.min.seconds.before.relogin", "90"); when(pxfUserGroupInformationMock.loginUserFromKeytab(configuration, "server", "config", "principal", "/path/to/keytab")).thenReturn(expectedLoginSession); + when(pxfUserGroupInformationMock.getKerberosTicketRenewWindow("server", configuration)).thenReturn(0.8f); UserGroupInformation loginUGI = secureLogin.getLoginUser("server", "config", configuration); @@ -301,7 +304,7 @@ public void testLoginKerberosSameServerDifferentPrincipal() throws IOException { // ------------------- now change the principal in the configuration, login again ------------------------- - LoginSession expectedDiffLoginSession = new LoginSession("config", "diff-principal", "/path/to/keytab", expectedUGI, null, 90000); + LoginSession expectedDiffLoginSession = new LoginSession("config", "diff-principal", "/path/to/keytab", expectedUGI, null, 90000, 0.8f); Configuration diffConfiguration = new Configuration(); diffConfiguration.set("hadoop.security.authentication", "kerberos"); diffConfiguration.set(PROPERTY_KEY_SERVICE_PRINCIPAL, "diff-principal"); @@ -310,6 +313,7 @@ public void testLoginKerberosSameServerDifferentPrincipal() throws IOException { when(pxfUserGroupInformationMock.loginUserFromKeytab(configuration, "server", "config", "principal", "/path/to/keytab")).thenReturn(expectedLoginSession); when(pxfUserGroupInformationMock.loginUserFromKeytab(diffConfiguration, "server", "config", "diff-principal", "/path/to/keytab")).thenReturn(expectedDiffLoginSession); when(pxfUserGroupInformationMock.getKerberosMinMillisBeforeRelogin("server", diffConfiguration)).thenReturn(180000L); + when(pxfUserGroupInformationMock.getKerberosTicketRenewWindow("server", diffConfiguration)).thenReturn(0.8f); UserGroupInformation diffLoginUGI = secureLogin.getLoginUser("server", "config", diffConfiguration); @@ -325,6 +329,7 @@ public void testLoginKerberosSameServerDifferentPrincipal() throws IOException { verify(pxfUserGroupInformationMock).loginUserFromKeytab(diffConfiguration, "server", "config", "diff-principal", "/path/to/keytab"); verify(pxfUserGroupInformationMock, times(2)).getKerberosMinMillisBeforeRelogin("server", diffConfiguration); + verify(pxfUserGroupInformationMock, times(2)).getKerberosTicketRenewWindow("server", diffConfiguration); verify(pxfUserGroupInformationMock).reloginFromKeytab("server", expectedDiffLoginSession); verifyNoMoreInteractions(pxfUserGroupInformationMock); @@ -333,10 +338,11 @@ public void testLoginKerberosSameServerDifferentPrincipal() throws IOException { @Test public void testLoginKerberosReuseExistingLoginSessionWithResolvedHostnameInPrincipal() throws IOException { when(pxfUserGroupInformationMock.getKerberosMinMillisBeforeRelogin("server", configuration)).thenReturn(90L); + when(pxfUserGroupInformationMock.getKerberosTicketRenewWindow("server", configuration)).thenReturn(0.8f); expectedUGI = UserGroupInformation.createUserForTesting("some", new String[]{}); - expectedLoginSession = new LoginSession("config", RESOLVED_PRINCIPAL, "/path/to/keytab", expectedUGI, null, 90); + expectedLoginSession = new LoginSession("config", RESOLVED_PRINCIPAL, "/path/to/keytab", expectedUGI, null, 90, 0.8f); SecureLogin.addToCache("server", expectedLoginSession); configuration.set("hadoop.security.authentication", "kerberos"); diff --git a/server/pxf-service/src/templates/templates/pxf-site.xml b/server/pxf-service/src/templates/templates/pxf-site.xml index 503be11693..8020d3f5ef 100644 --- a/server/pxf-service/src/templates/templates/pxf-site.xml +++ b/server/pxf-service/src/templates/templates/pxf-site.xml @@ -26,6 +26,16 @@ Set to true to enable, false to disable. + + pxf.service.kerberos.ticket-renew-window + 0.8 + + By default, PXF refreshes the Kerberos ticket when 80% (0.8) of its lifespan has elapsed. + Specifies the window, as a float, at which PXF will actively refresh the ticket. This value + must be between 0 and 1. Setting this value to 0 means that every PXF request will get a new + ticket. + +