Merge branch 'pxf-6.x' into 6.8.0-sync

arenadata/Dockerfile

@@ -0,0 +1,34 @@
+FROM hub.adsw.io/library/gpdb6_regress:latest as base
+
+# install go, ginkgo and keep env variables which may be used as a part of base image
+RUN yum install -y go
+ENV GOPATH=$HOME/go
+ENV PATH=$PATH:/usr/local/go/bin:$GOPATH/bin
+RUN go install github.com/onsi/ginkgo/ginkgo@latest
+
+# leave pxf artifacts dir env also
+ENV OUTPUT_ARTIFACT_DIR="pxf_tarball"
+
+# remove unnecessary artifacts and create symlinks
+# concource scripts expects gpdb and pxf placed in the same folder
+RUN rm /home/gpadmin/bin_gpdb/server-*.tar.gz && \
+    mkdir /tmp/build && \
+    ln -s /home/gpadmin/gpdb_src /tmp/build/gpdb_src && \
+    ln -s /home/gpadmin/bin_gpdb /tmp/build/bin_gpdb
+# default working dir - the place where all sources and artifacts are placed
+WORKDIR /tmp/build
+
+# create separate image with files we don't want to keep in base image
+FROM base as build
+COPY . /tmp/build/pxf_src
+RUN source gpdb_src/concourse/scripts/common.bash && \
+    install_gpdb && \
+    source '/usr/local/greenplum-db-devel/greenplum_path.sh' && \
+    mkdir ${OUTPUT_ARTIFACT_DIR} && \
+    export SKIP_FDW_BUILD_REASON=0 && \
+    pxf_src/concourse/scripts/compile_pxf.bash
+
+# create test image which prepares base image and keeps only pxf artifacts from build image
+FROM base as test
+COPY --from=build /tmp/build/${OUTPUT_ARTIFACT_DIR}/pxf.tar.gz /tmp/build/${OUTPUT_ARTIFACT_DIR}/
+COPY --from=build /tmp/build/pxf_src /tmp/build/pxf_src

arenadata/README.md

@@ -0,0 +1,15 @@
+## How to build PXF Docker image
+From the root pxf folder run:
+```bash
+docker build -t gpdb6_pxf_regress:latest -f arenadata/Dockerfile .
+```
+This will build an image called `gpdb6_pxf_regress` with the tag `latest`. This image is based on `gpdb6_regress:latest`, which additionally contains pxf sources and pxf artifacts tarball in `/tmp/build/pxf_src` and `/tmp/build/pxf_tarball` folders respectively.
+
+## How to test PXF
+During the image building phase `compile_pxf.bash` script additionally calls `test` make target, which calls `make -C cli/go/src/pxf-cli test` and `make -C server test` commands.
+To additionally test `fdw` and `external-table` parts you may call:
+```bash
+docker run --rm -it \
+  --privileged --sysctl kernel.sem="500 1024000 200 4096" \
+  gpdb6_pxf_regress:latest /tmp/build/pxf_src/arenadata/test_in_docker.sh
+```

arenadata/test_in_docker.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# This script depends on hub.adsw.io/library/gpdb6_pxf_regress
+set -exo pipefail
+
+# manually prepare gpadmin user; test_pxf.bash doesn't tweak gpadmin folder permissions and ssh keys
+./gpdb_src/concourse/scripts/setup_gpadmin_user.bash
+# unpack gpdb and pxf; run gpdb cluster and pxf server
+/tmp/build/pxf_src/concourse/scripts/test_pxf.bash
+# tweak necessary folders to run regression tests later
+chown gpadmin:gpadmin -R /usr/local/greenplum-db-devel
+chown gpadmin:gpadmin -R /tmp/build/pxf_src
+
+# test fdw and external-table
+su - gpadmin -c "
+    source '/usr/local/greenplum-db-devel/greenplum_path.sh';
+    source '/home/gpadmin/gpdb_src/gpAux/gpdemo/gpdemo-env.sh';
+    cd /tmp/build/pxf_src/fdw &&
+    make install &&
+    make installcheck &&
+    cd ../external-table/ &&
+    make install &&
+    make installcheck;
+"

external-table/src/libchurl.c

@@ -473,7 +473,6 @@ churl_read_check_connectivity(CHURL_HANDLE handle)
 	Assert(!context->upload);
 
 	fill_internal_buffer(context, 1);
-	check_response(context);
 }
 
 /*
@@ -626,6 +625,8 @@ multi_perform(churl_context *context)
 	if (curl_error != CURLM_OK)
 		elog(ERROR, "internal error: curl_multi_perform failed (%d - %s)",
 			 curl_error, curl_easy_strerror(curl_error));
+
+	check_response(context);
 }
 
 static bool
@@ -653,8 +654,6 @@ flush_internal_buffer(churl_context *context)
 		multi_perform(context);
 	}
 
-	check_response(context);
-
 	if ((context->curl_still_running == 0) &&
 		((context_buffer->top - context_buffer->bot) > 0))
 		elog(ERROR, "failed sending to remote component %s", get_dest_address(context->curl_handle));
@@ -709,8 +708,6 @@ finish_upload(churl_context *context)
 	 */
 	while (context->curl_still_running != 0)
 		multi_perform(context);
-
-	check_response(context);
 }
 
 static void

external-table/src/pxfbridge.c

@@ -40,7 +40,7 @@ gpbridge_cleanup(gphadoop_context *context)
 	if (context == NULL)
 		return;
 
-	churl_cleanup(context->churl_handle, false);
+	churl_cleanup(context->churl_handle, context->after_error);
 	context->churl_handle = NULL;
 
 	churl_headers_cleanup(context->churl_headers);

external-table/src/pxfbridge.h

@@ -43,6 +43,7 @@ typedef struct
 	ProjectionInfo *proj_info;
 	List           *quals;
 	bool           completed;
+	bool           after_error;
 } gphadoop_context;
 
 /*

external-table/src/pxfprotocol.c

@@ -26,6 +26,7 @@
 #include "access/fileam.h"
 #endif
 #include "utils/elog.h"
+#include "utils/resowner.h"
 
 /* define magic module unless run as a part of test cases */
 #ifndef UNIT_TESTING
@@ -154,6 +155,21 @@ pxfprotocol_import(PG_FUNCTION_ARGS)
 	PG_RETURN_INT32(bytes_read);
 }
 
+static void
+url_curl_abort_callback(ResourceReleasePhase phase,
+						bool isCommit,
+						bool isTopLevel,
+						void *arg)
+{
+	gphadoop_context *context = arg;
+
+	if (phase != RESOURCE_RELEASE_AFTER_LOCKS || isCommit || !isTopLevel)
+		return;
+
+	context->after_error = true;
+	cleanup_context(context);
+}
+
 /*
  * Allocates context and sets values for the segment
  */
@@ -190,6 +206,8 @@ create_context(PG_FUNCTION_ARGS, bool is_import)
 	context->proj_info = proj_info;
 	context->quals     = filter_quals;
 	context->completed = false;
+	context->after_error = false;
+	RegisterResourceReleaseCallback(url_curl_abort_callback, context);
 	return context;
 }
 
@@ -201,6 +219,7 @@ cleanup_context(gphadoop_context *context)
 {
 	if (context != NULL)
 	{
+		UnregisterResourceReleaseCallback(url_curl_abort_callback, context);
 		gpbridge_cleanup(context);
 		pfree(context->uri.data);
 		pfree(context);

fdw/libchurl.c

@@ -480,7 +480,6 @@ churl_read_check_connectivity(CHURL_HANDLE handle)
 	Assert(!context->upload);
 
 	fill_internal_buffer(context, 1);
-	check_response(context);
 }
 
 /*
@@ -633,6 +632,8 @@ multi_perform(churl_context *context)
 	if (curl_error != CURLM_OK)
 		elog(ERROR, "internal error: curl_multi_perform failed (%d - %s)",
 			 curl_error, curl_easy_strerror(curl_error));
+
+	check_response(context);
 }
 
 static bool
@@ -717,8 +718,6 @@ finish_upload(churl_context *context)
 	 */
 	while (context->curl_still_running != 0)
 		multi_perform(context);
-
-	check_response(context);
 }
 
 static void

server/build.gradle

@@ -20,6 +20,18 @@
 buildscript {
     repositories {
         mavenCentral()
+        maven {
+            url "https://rt.adsw.io/artifactory/maven-arenadata-release/"
+            mavenContent {
+                releasesOnly()
+            }
+        }
+        maven {
+            url "https://rt.adsw.io/artifactory/maven-arenadata-snapshot/"
+            mavenContent {
+                snapshotsOnly()
+            }
+        }
     }
 }
 
@@ -40,6 +52,18 @@ allprojects {
 
     repositories {
         mavenCentral()
+        maven {
+            url "https://rt.adsw.io/artifactory/maven-arenadata-release/"
+            mavenContent {
+                releasesOnly()
+            }
+        }
+        maven {
+            url "https://rt.adsw.io/artifactory/maven-arenadata-snapshot/"
+            mavenContent {
+                snapshotsOnly()
+            }
+        }
     }
 }
 
@@ -95,6 +119,18 @@ configure(javaProjects) {
             dependency("org.apache.htrace:htrace-core:3.1.0-incubating")
             dependency("org.apache.htrace:htrace-core4:4.0.1-incubating")
 
+            // --- bump log4j2 to 2.17.1 for CVE-2021-44228, CVE-2021-45046, and CVE-2021-45105 fixes,
+            // more details: https://logging.apache.org/log4j/2.x/security.html
+            // revert once org.springframework.boot:spring-boot-starter-log4j2 is upgraded to bundle log4j2:2.17.1+
+            dependencySet(group:"org.apache.logging.log4j", version:"2.17.1") {
+                entry("log4j-jul")
+                entry("log4j-api")
+                entry("log4j-core")
+                entry("log4j-spring-boot")
+            }
+            dependency("org.apache.logging.log4j:log4j-slf4j-impl:2.17.1")
+            // --- end of CVE patch
+
             dependency("org.apache.zookeeper:zookeeper:3.4.6")
             dependency("org.codehaus.woodstox:stax2-api:3.1.4")
             dependency("org.datanucleus:datanucleus-api-jdo:4.2.4")
@@ -111,7 +147,12 @@ configure(javaProjects) {
             dependency("org.threeten:threeten-extra:1.5.0")
             dependency("org.tukaani:xz:1.8")
             dependency("org.wildfly.openssl:wildfly-openssl:1.0.7.Final")
-            dependency("org.xerial.snappy:snappy-java:1.1.10.1")
+
+            // The snappy-java:1.1.8.4 is required for PPC64LE arch.
+            dependency("org.xerial.snappy:snappy-java:1.1.8.4")
+
+            // Arenadata encryption
+            dependency("io.arenadata.security:encryption:1.0.0")
 
             // Hadoop dependencies
             dependencySet(group:"org.apache.hadoop", version:"${hadoopVersion}") {
@@ -143,11 +184,13 @@ configure(javaProjects) {
                 entry("hive-metastore")
                 entry("hive-serde")
                 entry("hive-common")
+                entry("hive-service")
+                entry("hive-service-rpc")
             }
             // 1.2.2 breaks on CDH-5.x
-            dependencySet(group:"org.apache.hive", version:"1.1.0") {
+            // We use custom hive-jdbc driver from Arenadata
+            dependencySet(group:"io.arenadata.hive", version:"2.3.8-arenadata-pxf-3") {
                 entry("hive-jdbc")
-                entry("hive-service")
             }
             dependencySet(group:"org.apache.hive.shims", version:"${hiveVersion}") {
                 entry("hive-shims-common")

server/gradle.properties

@@ -25,7 +25,7 @@ hbaseVersion=1.3.2
 junitVersion=4.11
 parquetVersion=1.11.1
 awsJavaSdk=1.12.261
-springBootVersion=2.7.12
+springBootVersion=2.4.3
 org.gradle.daemon=true
 org.gradle.parallel=false
 orcVersion=1.6.13

server/pxf-api/build.gradle

@@ -39,6 +39,7 @@ dependencies {
     implementation("commons-configuration:commons-configuration")
     implementation("commons-lang:commons-lang")
     implementation("org.apache.commons:commons-lang3")
+    implementation("io.arenadata.security:encryption")
 
     implementation("org.apache.hadoop:hadoop-auth")                  { transitive = false }
     implementation("org.codehaus.woodstox:stax2-api")                { transitive = false }

server/pxf-api/src/main/java/org/greenplum/pxf/api/configuration/PxfJksTextEncryptorConfiguration.java

@@ -0,0 +1,37 @@
+package org.greenplum.pxf.api.configuration;
+
+import io.arenadata.security.encryption.client.configuration.JksTextEncryptorConfiguration;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@ConditionalOnProperty({"pxf.ssl.jks-store.path", "pxf.ssl.jks-store.password", "pxf.ssl.salt.key"})
+public class PxfJksTextEncryptorConfiguration extends JksTextEncryptorConfiguration {
+    private final String path;
+    private final String password;
+    private final String key;
+
+    public PxfJksTextEncryptorConfiguration(@Value("${pxf.ssl.jks-store.path}") String path,
+                                            @Value("${pxf.ssl.jks-store.password}") String password,
+                                            @Value("${pxf.ssl.salt.key}") String key) {
+        this.path = path;
+        this.password = password;
+        this.key = key;
+    }
+
+    @Override
+    protected String jksStorePath() {
+        return path;
+    }
+
+    @Override
+    protected char[] jksStorePassword() {
+        return password.toCharArray();
+    }
+
+    @Override
+    protected String secretKeyAlias() {
+        return key;
+    }
+}

server/pxf-api/src/main/java/org/greenplum/pxf/api/examples/DemoTextResolver.java

@@ -22,7 +22,10 @@
 import org.greenplum.pxf.api.OneField;
 import org.greenplum.pxf.api.OneRow;
 import org.greenplum.pxf.api.io.DataType;
+import org.greenplum.pxf.api.model.InputStreamHandler;
 
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
 import java.util.LinkedList;
 import java.util.List;
 
@@ -31,6 +34,7 @@
  * <p>
  * Demo implementation of resolver that returns text format
  */
+@InputStreamHandler
 public class DemoTextResolver extends DemoResolver {
 
     /**
@@ -62,8 +66,16 @@ public OneRow setFields(List<OneField> record) throws Exception {
             throw new Exception("Unexpected record format, expected 1 field, found " +
                     (record == null ? 0 : record.size()));
         }
-        byte[] value = (byte[]) record.get(0).val;
+        int readCount;
+        byte[] data = new byte[1024];
+        ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+        DataInputStream dis = (DataInputStream) record.get(0).val;
+        while ((readCount = dis.read(data, 0, data.length)) != -1) {
+            buffer.write(data, 0, readCount);
+        }
+        buffer.flush();
+        byte[] bytes= buffer.toByteArray();
         // empty array means the end of input stream, return null to stop iterations
-        return value.length == 0 ? null : new OneRow(value);
+        return bytes.length == 0 ? null : new OneRow(bytes);
     }
 }

server/pxf-api/src/main/java/org/greenplum/pxf/api/utilities/SpringContext.java

@@ -26,6 +26,14 @@ public static <T extends Object> T getBean(Class<T> requiredType) {
         return context.getBean(requiredType);
     }
 
+    public static <T extends Object> T getNullableBean(Class<T> requiredType) {
+        try {
+            return context.getBean(requiredType);
+        } catch (Exception e) {
+            return null;
+        }
+    }
+
     @Override
     public void setApplicationContext(ApplicationContext context) throws BeansException {