Skip to content

Commit

Permalink
Merge pull request #100 from arenadata/feature/ADBDEV-5595
Browse files Browse the repository at this point in the history 
[ADBDEV-5595] - Add support for new types in PXF
  • Loading branch information
@iamlapa
iamlapa authored Jul 30, 2024
2 parents b43cf67 + bd92bd0 commit 1743548
Show file tree
Hide file tree
Showing 71 changed files with 17,643 additions and 3,217 deletions.
3,925 changes: 3,296 additions & 629 deletions automation/sqlrepo/features/filterpushdown/checkFilterPushDown/expected/query01.ans
View file

Large diffs are not rendered by default.

3,913 changes: 3,290 additions & 623 deletions automation/sqlrepo/features/filterpushdown/checkFilterPushDown/expected/query02.ans
View file

Large diffs are not rendered by default.

486 changes: 408 additions & 78 deletions automation/sqlrepo/features/filterpushdown/checkFilterPushDown/sql/query01.sql
View file

Large diffs are not rendered by default.

486 changes: 408 additions & 78 deletions automation/sqlrepo/features/filterpushdown/checkFilterPushDown/sql/query02.sql
View file

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion automation/sqlrepo/features/filterpushdown/checkFilterPushDownDisabled/expected/query01.ans
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
-- @description query01 for PXF filter pushdown disabled case
SET gp_external_enable_filter_pushdown = off;
SET
SELECT * FROM test_filter WHERE t0 = 'C' AND a1 = 2 ORDER BY t0, a1;
SELECT t0, a1, b2, c3, d4, e5, filterValue FROM test_filter WHERE t0 = 'C' AND a1 = 2 ORDER BY t0, a1;
t0 | a1 | b2 | c3 | d4 | e5 | filtervalue
----+----+----+------+-----+----+-------------
C | 2 | | 2.21 | CC | CC | No filter
Expand Down
2 changes: 1 addition & 1 deletion automation/sqlrepo/features/filterpushdown/checkFilterPushDownDisabled/sql/query01.sql
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
-- @description query01 for PXF filter pushdown disabled case

SET gp_external_enable_filter_pushdown = off;
SELECT * FROM test_filter WHERE t0 = 'C' AND a1 = 2 ORDER BY t0, a1;
SELECT t0, a1, b2, c3, d4, e5, filterValue FROM test_filter WHERE t0 = 'C' AND a1 = 2 ORDER BY t0, a1;
4 changes: 2 additions & 2 deletions ...tion/sqlrepo/features/filterpushdown/checkFilterPushDownHexDelimiter/expected/query01.ans
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,15 @@ SET gp_external_enable_filter_pushdown = true;
SET
SET optimizer = off;
SET
SELECT * FROM test_filter WHERE t0 = 'C' AND a1 = 2 ORDER BY t0, a1;
SELECT t0, a1, b2, c3, d4, e5, filterValue FROM test_filter WHERE t0 = 'C' AND a1 = 2 ORDER BY t0, a1;
t0 | a1 | b2 | c3 | d4 | e5 | filtervalue
----+----+----+------+-----+----+--------------------------
C | 2 | | 2.21 | CC | CC | a0c25s1dCo5a1c23s1d2o5l0
(1 row)

SET optimizer = on;
SET
SELECT * FROM test_filter WHERE t0 = 'C' AND a1 = 2 ORDER BY t0, a1;
SELECT t0, a1, b2, c3, d4, e5, filterValue FROM test_filter WHERE t0 = 'C' AND a1 = 2 ORDER BY t0, a1;
t0 | a1 | b2 | c3 | d4 | e5 | filtervalue
----+----+----+------+-----+----+--------------------------
C | 2 | | 2.21 | CC | CC | a0c25s1dCo5a1c23s1d2o5l0
Expand Down
4 changes: 2 additions & 2 deletions automation/sqlrepo/features/filterpushdown/checkFilterPushDownHexDelimiter/sql/query01.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ SET gp_external_enable_filter_pushdown = true;

SET optimizer = off;

SELECT * FROM test_filter WHERE t0 = 'C' AND a1 = 2 ORDER BY t0, a1;
SELECT t0, a1, b2, c3, d4, e5, filterValue FROM test_filter WHERE t0 = 'C' AND a1 = 2 ORDER BY t0, a1;

SET optimizer = on;

SELECT * FROM test_filter WHERE t0 = 'C' AND a1 = 2 ORDER BY t0, a1;
SELECT t0, a1, b2, c3, d4, e5, filterValue FROM test_filter WHERE t0 = 'C' AND a1 = 2 ORDER BY t0, a1;
7 changes: 4 additions & 3 deletions ...mation/src/test/java/org/greenplum/pxf/automation/arenadata/FragmentDistributionTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,12 +41,14 @@ public class FragmentDistributionTest extends BaseFeature {
private String pxfHome;
private String pxfLogFile;
private String hdfsPath;
Table dataTable;
private Table dataTable;
private Table postgresSourceTable;
private String restartCommand;

@Override
protected void beforeClass() throws Exception {
pxfHome = cluster.getPxfHome();
restartCommand = pxfHome + "/bin/pxf restart";
if (cluster instanceof MultiNodeCluster) {
pxfNodes = ((MultiNodeCluster) cluster).getNode(SegmentNode.class, PhdCluster.EnumClusterServices.pxf);
}
Expand Down Expand Up @@ -223,8 +225,7 @@ public void testFragmentDistributionForHdfsWithSegLimit() throws Exception {
}

private void changeLogLevel(String level) throws Exception {
cluster.runCommandOnNodes(pxfNodes, String.format("export PXF_LOG_LEVEL=%s", level));
cluster.restart(PhdCluster.EnumClusterServices.pxf);
cluster.runCommandOnNodes(pxfNodes, String.format("export PXF_LOG_LEVEL=%s;%s", level, restartCommand));
}

private void cleanLogs() throws Exception {
Expand Down
5 changes: 3 additions & 2 deletions automation/src/test/java/org/greenplum/pxf/automation/arenadata/JdbcBackPressureTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ public class JdbcBackPressureTest extends BaseFeature {
private String pxfLogFile;
private List<Node> pxfNodes;
private Oracle oracle;
private String restartCommand;

@Override
public void beforeClass() throws Exception {
Expand All @@ -76,6 +77,7 @@ public void beforeClass() throws Exception {
if (cluster instanceof MultiNodeCluster) {
pxfNodes = ((MultiNodeCluster) cluster).getNode(SegmentNode.class, PhdCluster.EnumClusterServices.pxf);
}
restartCommand = pxfHome + "/bin/pxf restart";
oracle = (Oracle) SystemManagerImpl.getInstance().getSystemObject("oracle");
prepareData();
changeLogLevel("trace");
Expand Down Expand Up @@ -187,8 +189,7 @@ public void checkBatchTimeoutSuccess() throws Exception {
}

private void changeLogLevel(String level) throws Exception {
cluster.runCommandOnNodes(pxfNodes, String.format("export PXF_LOG_LEVEL=%s", level));
cluster.restart(PhdCluster.EnumClusterServices.pxf);
cluster.runCommandOnNodes(pxfNodes, String.format("export PXF_LOG_LEVEL=%s;%s", level, restartCommand));
}

private void cleanPxfLog() throws Exception {
Expand Down
5 changes: 3 additions & 2 deletions automation/src/test/java/org/greenplum/pxf/automation/arenadata/PushdownPredicateInTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,12 @@ public class PushdownPredicateInTest extends BaseFeature {
private String pxfLogFile;
private Table gpdbPredicateInSourceTable;
private Table oraclePredicateInSourceTable;
private String restartCommand;

@Override
protected void beforeClass() throws Exception {
pxfHome = cluster.getPxfHome();
restartCommand = pxfHome + "/bin/pxf restart";
String pxfJdbcSiteConfPath = String.format(PXF_JDBC_SITE_CONF_FILE_PATH_TEMPLATE, pxfHome, PXF_ORACLE_SERVER_PROFILE);
pxfJdbcSiteConfFile = pxfJdbcSiteConfPath + "/" + PXF_JDBC_SITE_CONF_FILE_NAME;
String pxfJdbcSiteConfTemplate = pxfHome + "/" + PXF_JDBC_SITE_CONF_TEMPLATE_RELATIVE_PATH;
Expand Down Expand Up @@ -150,8 +152,7 @@ public void testPredicateInOracle() throws Exception {
}

private void changeLogLevel(String level) throws Exception {
cluster.runCommandOnNodes(pxfNodes, String.format("export PXF_LOG_LEVEL=%s", level));
cluster.restart(PhdCluster.EnumClusterServices.pxf);
cluster.runCommandOnNodes(pxfNodes, String.format("export PXF_LOG_LEVEL=%s;%s", level, restartCommand));
}

private void cleanLogs() throws Exception {
Expand Down
47 changes: 40 additions & 7 deletions ...rc/test/java/org/greenplum/pxf/automation/features/filterpushdown/FilterPushDownTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,46 @@ public class FilterPushDownTest extends BaseFeature {
private static final String COMMA = ",";

private static final String[] FIELDS = new String[]{
"t0 text",
"a1 integer",
"b2 boolean",
"c3 numeric",
"d4 char(3)",
"e5 varchar(2)",
"filterValue text"
"t0 text",
"a1 integer",
"b2 boolean",
"c3 numeric",
"d4 char(2)",
"e5 varchar(2)",
"x1 bpchar(2)",
"x2 smallint",
"x3 bigint",
"x4 real",
"x5 float8",
"x6 bytea",
"x7 date",
"x8 time",
"x9 timestamp",
"x10 timestamp with time zone",
"x11 interval",
"x12 uuid",
"x13 json",
"x14 jsonb",
"x15 int2[]",
"x16 int4[]",
"x17 int8[]",
"x18 bool[]",
"x19 text[]",
"x20 float4[]",
"x21 float8[]",
"x22 bytea[]",
"x23 bpchar[]",
"x24 varchar(2)[]",
"x25 date[]",
"x26 uuid[]",
"x27 numeric[]",
"x28 time[]",
"x29 timestamp[]",
"x30 timestamp with time zone[]",
"x31 interval[]",
"x32 json[]",
"x33 jsonb[]",
"filterValue text"
};

/**
Expand Down
2 changes: 1 addition & 1 deletion automation/src/test/java/org/greenplum/pxf/automation/features/hbase/HBaseTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -260,7 +260,7 @@ public void multipleQualifiersPushdownFilter() throws Exception {
public void partialFilterPushdown() throws Exception {

String whereClause = " WHERE \"cf1:q3\" > 6 AND \"cf1:q7\" = '42'";
String partialfilterString = "No filter";
String partialfilterString = "a3c23s1d6o2a7c17s2d42o5l0";
verifyFilterResults(hbaseTable, exTable, whereClause, partialfilterString,
"partialFilterPushdown", false);
}
Expand Down
4 changes: 3 additions & 1 deletion automation/src/test/java/org/greenplum/pxf/automation/features/hive/HiveOrcTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import org.greenplum.pxf.automation.structures.tables.basic.Table;
import org.greenplum.pxf.automation.structures.tables.hive.HiveTable;
import org.greenplum.pxf.automation.structures.tables.utils.TableFactory;
import org.junit.Ignore;
import org.testng.annotations.Test;

import java.nio.charset.StandardCharsets;
Expand Down Expand Up @@ -356,7 +357,7 @@ public void defaultAnalyze() throws Exception {

// Perform Analyze on external table and check suitable Warnings.
gpdb.runQueryWithExpectedWarning("ANALYZE " + exTable.getName(),
"ANALYZE for HiveRc, HiveText, and HiveOrc plugins is not supported", true);
"skipping \"" + exTable.getName() + "\"", true);

runSqlTest("features/hive/default_analyze");
}
Expand Down Expand Up @@ -402,6 +403,7 @@ public void hiveCollectionTypes() throws Exception {
* @throws Exception if test fails to run
*/
@Test(groups = { "features", "hcatalog" })
@Ignore
public void aggregateQueries() throws Exception {

createExternalTable(PXF_HIVE_SMALL_DATA_TABLE,
Expand Down
Loading

0 comments on commit 1743548

Please sign in to comment.