diff --git a/.github/workflows/create-release-on-tag.yml b/.github/workflows/create-release-on-tag.yml index 922c17d4ab..21fea525bd 100644 --- a/.github/workflows/create-release-on-tag.yml +++ b/.github/workflows/create-release-on-tag.yml @@ -22,17 +22,23 @@ jobs: tag_name: ${{ github.ref }} release_name: PXF Version ${{ github.ref }} body: | - ## 6.6.0 (04/06/2023) + ## 6.7.0 (07/13/2023) ### Enhancements: - - - [#949](https://github.com/greenplum-db/pxf/pull/949) Support for fixedwidth formatter with new `*:fixedwidth` PXF profiles - - [#954](https://github.com/greenplum-db/pxf/pull/954) Update table options names to not include dash character - - [#955](https://github.com/greenplum-db/pxf/pull/955) Bump jackson-databind from 2.13.4.1 to 2.13.4.2 in /automation - + + - [#956](https://github.com/greenplum-db/pxf/pull/956) Add pxfdelimited_import formatter to support multibyte delimiters for TEXT and CSV profiles + - [#960](https://github.com/greenplum-db/pxf/pull/960) Add support year with more than 4 digits in 'date' or 'timestamp' + - [#973](https://github.com/greenplum-db/pxf/pull/973) Enable write flow for FDW for non-text/csv formats + - [#976](https://github.com/greenplum-db/pxf/pull/976) Restrict PXF to listen to local requests only + - [#979](https://github.com/greenplum-db/pxf/pull/979) Add logging to the LineBreakAccessor for the write + - [#983](https://github.com/greenplum-db/pxf/pull/983) Bump Springboot to 2.7.12 + - [#984](https://github.com/greenplum-db/pxf/pull/984) Enable writing data in JSON format using *:json profiles + - [#989](https://github.com/greenplum-db/pxf/pull/989) Bump snappy to 1.1.10.1 + ### Bug Fixes: - - - [#940](https://github.com/greenplum-db/pxf/pull/940) Introduced options to handle decimal overflow when writing Parquet files + + - [#967](https://github.com/greenplum-db/pxf/pull/967) FDW: Fix for skipping the dropped and correctly counting Projection Index + - [#978](https://github.com/greenplum-db/pxf/pull/978) Added erroring out logic for decimal overflow for ORC draft: false prerelease: false diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e673b962f..6b8616c8c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## 6.7.0 (07/13/2023) + +### Enhancements: + +- [#956](https://github.com/greenplum-db/pxf/pull/956) Add pxfdelimited_import formatter to support multibyte delimiters for TEXT and CSV profiles +- [#960](https://github.com/greenplum-db/pxf/pull/960) Add support year with more than 4 digits in 'date' or 'timestamp' +- [#973](https://github.com/greenplum-db/pxf/pull/973) Enable write flow for FDW for non-text/csv formats +- [#976](https://github.com/greenplum-db/pxf/pull/976) Restrict PXF to listen to local requests only +- [#979](https://github.com/greenplum-db/pxf/pull/979) Add logging to the LineBreakAccessor for the write +- [#983](https://github.com/greenplum-db/pxf/pull/983) Bump Springboot to 2.7.12 +- [#984](https://github.com/greenplum-db/pxf/pull/984) Enable writing data in JSON format using *:json profiles +- [#989](https://github.com/greenplum-db/pxf/pull/989) Bump snappy to 1.1.10.1 + +### Bug Fixes: + +- [#967](https://github.com/greenplum-db/pxf/pull/967) FDW: Fix for skipping the dropped and correctly counting Projection Index +- [#978](https://github.com/greenplum-db/pxf/pull/978) Added erroring out logic for decimal overflow for ORC + ## 6.6.0 (04/06/2023) ### Enhancements: diff --git a/automation/README.Linux.md b/automation/README.Linux.md index 6f7acfa3f3..4b6f4de70c 100644 --- a/automation/README.Linux.md +++ b/automation/README.Linux.md @@ -1,6 +1,7 @@ # Running Automation on Linux **Note:** This information was documented based on the steps taken to get automation running on a Debian Bookworm (12) system. +They are intended to be used in tandem with the information in the main README file. ## Locale Setup @@ -19,19 +20,6 @@ gpstop -a gpstart -a ``` -## SSH Setup - -```sh -sudo tee /etc/ssh/sshd_config.d/pxf-automation.conf >/dev/null /dev/null <_.log -TestNg report will generated into target/surefire-reports +TestNg report will be generated into target/surefire-reports # IDE Setup (IntelliJ) and Automation Debugging diff --git a/automation/pom.xml b/automation/pom.xml index 2b637710dd..26dae47130 100644 --- a/automation/pom.xml +++ b/automation/pom.xml @@ -21,7 +21,7 @@ 1.3.2.1 1.1.0 1.6.13 - 1.1.1.7 + 1.1.10.1 1.6.4 diff --git a/automation/src/main/java/org/greenplum/pxf/automation/components/common/DbSystemObject.java b/automation/src/main/java/org/greenplum/pxf/automation/components/common/DbSystemObject.java index efa5ab22ed..c389ea5b6b 100755 --- a/automation/src/main/java/org/greenplum/pxf/automation/components/common/DbSystemObject.java +++ b/automation/src/main/java/org/greenplum/pxf/automation/components/common/DbSystemObject.java @@ -15,6 +15,7 @@ import jsystem.framework.report.Reporter; import org.greenplum.pxf.automation.structures.tables.basic.Table; +import org.greenplum.pxf.automation.structures.tables.pxf.ExternalTable; import org.postgresql.util.PSQLException; import org.greenplum.pxf.automation.utils.exception.ExceptionUtils; @@ -158,7 +159,31 @@ public void insertData(Table source, Table target) throws Exception { dataStringBuilder.append(","); } } - runQuery("INSERT INTO " + target.getName() + " VALUES " + dataStringBuilder); + insertData(dataStringBuilder.toString(), target); + } + + /** + * Inserts data from the provided string into the target Table. The string is expected to contain data + * in SQL format that follows the 'INSERT INTO [table] VALUES ' clause. + * + * @param data string containing data to insert + * @param target table to insert data into, can be an internal, an external or a foreign table + * @throws Exception is operation fails + */ + public void insertData(String data, Table target) throws Exception { + if (!data.startsWith("(")) { + data = "(" + data; + } + if (!data.endsWith(")")) { + data = data + ")"; + } + + String query = "INSERT INTO " + target.getName() + " VALUES " + data; + if (target instanceof ExternalTable) { + runQueryInsertIntoExternalTable(query); + } else { + runQuery(query); + } } @Override @@ -191,6 +216,16 @@ public long runQueryTiming(String query) throws Exception { return System.currentTimeMillis() - startTimeInMillis; } + /** + * Run query that inserts data into an external or a foreign table and ignores a warning about not being able to + * analyze a foreign table (if applicable) because PXF FDW does not yet support analyzing foreign tables. + * @param query query to run + * @throws Exception + */ + protected void runQueryInsertIntoExternalTable(String query) throws Exception { + runQueryWithExpectedWarning(query, ".* --- cannot analyze this foreign table", true, true); + } + /** * Run query which expected to get warning in execution and match it to expected one. * diff --git a/automation/src/main/java/org/greenplum/pxf/automation/components/gpdb/Gpdb.java b/automation/src/main/java/org/greenplum/pxf/automation/components/gpdb/Gpdb.java index 0d29cde978..b7cf7c66e5 100755 --- a/automation/src/main/java/org/greenplum/pxf/automation/components/gpdb/Gpdb.java +++ b/automation/src/main/java/org/greenplum/pxf/automation/components/gpdb/Gpdb.java @@ -5,6 +5,7 @@ import org.greenplum.pxf.automation.components.common.DbSystemObject; import org.greenplum.pxf.automation.components.common.ShellSystemObject; import org.greenplum.pxf.automation.structures.tables.basic.Table; +import org.greenplum.pxf.automation.structures.tables.pxf.ExternalTable; import org.greenplum.pxf.automation.utils.jsystem.report.ReportUtils; import org.greenplum.pxf.automation.utils.system.FDWUtils; import org.springframework.util.Assert; @@ -52,9 +53,7 @@ public void init() throws Exception { * connecting it. */ driver = "org.postgresql.Driver"; - address = "jdbc:postgresql://" + getHost() + ":" + getPort() + "/template1"; - - connect(); + connectToDataBase("template1"); version = determineVersion(); if (!checkDataBaseExists(getDb())) { @@ -71,11 +70,7 @@ public void init() throws Exception { } } - super.close(); - - address = "jdbc:postgresql://" + getHost() + ":" + getPort() + "/" + getDb(); - - connect(); + connectToDataBase(getDb()); // Create the extensions if they don't exist String extensionName = FDWUtils.useFDW ? "pxf_fdw" : "pxf"; @@ -112,16 +107,100 @@ private void initPort() { /** * Copies data from source table into target table - * @param source - * @param target - * @throws Exception + * @param source source table + * @param target target table + * @throws Exception if the operation fails + */ + public void copyData(Table source, Table target) throws Exception { + + copyData(source, target, false); + } + + /** + * Copies data from source table into target table + * @param sourceName name of the source table + * @param target target table + * @throws Exception if the operation fails + */ + public void copyData(String sourceName, Table target) throws Exception { + + copyData(sourceName, target, false); + } + + /** + * Copies data from source table into target table + * @param sourceName name of the source table + * @param target target table + * @param columns columns to select from the source table, if null then all columns will be selected + * @throws Exception if the operation fails + */ + public void copyData(String sourceName, Table target, String[] columns) throws Exception { + + copyData(sourceName, target, columns,false, null); + } + + /** + * Copies data from source table into target table + * @param sourceName name of the source table + * @param target target table + * @param columns columns to select from the source table, if null then all columns will be selected + * @param context extra context to add before the SQL query + * @throws Exception if the operation fails + */ + public void copyData(String sourceName, Table target, String[] columns, String context) throws Exception { + + copyData(sourceName, target, columns, false, context); + } + + /** + * Copies data from source table into target table + * @param source source table + * @param target target table + * @param ignoreFail whether to ignore any failures + * @throws Exception if the operation fails + */ + public void copyData(Table source, Table target, boolean ignoreFail) throws Exception { + copyData(source.getName(), target, ignoreFail); + } + + /** + * Copies data from source table into target table + * @param sourceName name of the source table + * @param target target table + * @param ignoreFail whether to ignore any failures + * @throws Exception if the operation fails */ - public void copyData(Table source, Table target) throws Exception { + public void copyData(String sourceName, Table target, boolean ignoreFail) throws Exception { + copyData(sourceName, target, null, ignoreFail, null); + } + /** + * Copies data from the source table into the target table with a possibility of selecting specific columns. + * If the columns are specified, the schema of the target table should correspond to the projection achieved + * by specifying the columns. If the columns are not provided, all columns are selected from the source table. + * @param sourceName name of the source table + * @param target target table + * @param columns columns to select from the source table, if null then all columns will be selected + * @param ignoreFail whether to ignore any failures + * @param context extra context to add before the SQL query + * @throws Exception if the operation fails + */ + public void copyData(String sourceName, Table target, String[] columns, boolean ignoreFail, String context) throws Exception { + String columnList = (columns == null || columns.length == 0) ? "*" : String.join(",", columns); + String query = String.format("%sINSERT INTO %s SELECT %s FROM %s", + StringUtils.isBlank(context) ? "" : context + "; " ,target.getName(), columnList, sourceName); + if (target instanceof ExternalTable) { + runQueryInsertIntoExternalTable(query); + } else { + runQuery(query, ignoreFail, false); + } + } - runQuery("INSERT INTO " + target.getName() + " SELECT * FROM " - + source.getName()); - } + public void connectToDataBase(String dbName) throws Exception { + super.close(); + address = "jdbc:postgresql://" + getHost() + ":" + getPort() + "/" + dbName; + connect(); + } @Override public void createDataBase(String schemaName, boolean ignoreFail) throws Exception { @@ -173,14 +252,19 @@ private void createForeignServers(boolean ignoreFail) throws Exception { List servers = Lists.newArrayList( "default_hdfs", "default_hive", + "db_hive_jdbc", // Needed for JdbcHiveTest "default_hbase", - "default_jdbc", + "default_jdbc", // Needed for JdbcHiveTest and other JdbcTest which refers to the default server. + "database_jdbc", + "db-session-params_jdbc", "default_file", "default_s3", "default_gs", "default_adl", "default_wasbs", "s3_s3", + "s3-invalid_s3", + "s3-non-existent_s3", "hdfs-non-secure_hdfs", "hdfs-secure_hdfs", "hdfs-ipa_hdfs", @@ -298,29 +382,20 @@ public void copyFromStdin(Table from, Table to, String delim, boolean csv) throw } StringBuilder dataStringBuilder = new StringBuilder(); - List> data = from.getData(); - for (int i = 0; i < data.size(); i++) { - List row = data.get(i); - for (int j = 0; j < row.size(); j++) { - dataStringBuilder.append(row.get(j)); - if (j != row.size() - 1) { dataStringBuilder.append(delimeter); } } - dataStringBuilder.append("\n"); - } - dataStringBuilder.append("\\."); - copy(to.getName(), "STDIN", dataStringBuilder.toString(), delim, null, csv); + copyWithOptionalCTAS("STDIN", to, dataStringBuilder.toString(), delim, null, csv); } /** @@ -335,7 +410,21 @@ public void copyFromStdin(Table from, Table to, String delim, boolean csv) throw public void copyFromFile(Table to, File path, String delim, boolean csv) throws Exception { String from = "'" + path.getAbsolutePath() + "'"; copyLocalFileToRemoteGpdb(from); - copy(to.getName(), from, null, delim, null, csv); + copyWithOptionalCTAS(from, to, null, delim, null, csv); + } + + private void copyWithOptionalCTAS(String from, Table to, String dataToCopy, String delim, String nullChar, boolean csv) throws Exception { + // COPY TO is not supported in PXF FDW with GP6, so we will have to do a workaround by + // creating a native table, copying data from the file into it and then performing a CTAS into the foreign table + if (FDWUtils.useFDW && getVersion() < 7) { + Table nativeTable = createTableLike(to.getName() + "_native", to); + // copy data into the native table + copy(nativeTable.getName(), from, dataToCopy, delim, null, csv); + // CTAS into the foreign table + copyData(nativeTable, to, true); + } else { + copy(to.getName(), from, dataToCopy, delim, null, csv); + } } private void copyLocalFileToRemoteGpdb(String from) throws Exception { @@ -495,4 +584,25 @@ SELECT COUNT(*) FROM pg_catalog.pg_user_mapping um return count > 0; } + /** + * Create a table like the other table, only schema / distribution is copied, not the data. + * @param name name of table to create + * @param source source table + * @return table that got created + * @throws Exception if the operation fails + */ + private Table createTableLike(String name, Table source) throws Exception { + Table table = new Table(name, source.getFields()); + String[] distributionFields = source.getDistributionFields(); + if (distributionFields != null && distributionFields.length > 0) { + table.setDistributionFields(distributionFields); + } else { + // set distribution field as the first one so that PSQL does not issue a warning + // extract the name of the first table field and type, split off the type that follows the name after whitespace + table.setDistributionFields(new String[]{table.getFields()[0].split("\\s+")[0]}); + } + createTableAndVerify(table); + return table; + } + } diff --git a/automation/src/main/java/org/greenplum/pxf/automation/components/hdfs/Hdfs.java b/automation/src/main/java/org/greenplum/pxf/automation/components/hdfs/Hdfs.java index 5038d3369d..80a25cc2e4 100755 --- a/automation/src/main/java/org/greenplum/pxf/automation/components/hdfs/Hdfs.java +++ b/automation/src/main/java/org/greenplum/pxf/automation/components/hdfs/Hdfs.java @@ -37,6 +37,7 @@ import java.io.DataOutputStream; import java.io.File; import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; @@ -51,6 +52,7 @@ import java.util.List; import java.util.UUID; +import static java.lang.Thread.sleep; import static org.testng.Assert.assertEquals; /** @@ -237,10 +239,37 @@ private Path getDatapath(String path) { @Override public ArrayList list(String path) throws Exception { ReportUtils.startLevel(report, getClass(), "List From " + path); - RemoteIterator list = fs.listFiles(getDatapath(path), true); + RemoteIterator list = null; + // build resilience to long delays for cloud and NFS mounts in CI + Exception savedException = null; + int attempt = 0; + while (list == null && attempt++ < 120) { + try { + list = fs.listFiles(getDatapath(path), true); + } catch (FileNotFoundException e) { + savedException = e; + ReportUtils.report(report, getClass(), + String.format("Directory %s does not exist, attempt %d, will retry in 1 sec", path, attempt)); + sleep(1000); + } + } + if (list == null) { + ReportUtils.report(report, getClass(), + String.format("Directory %s does not exist, max attempts exceeded, throwing exception", path)); + throw savedException; + + } ArrayList filesList = new ArrayList<>(); while (list.hasNext()) { - filesList.add(list.next().getPath().toString()); + String pathToFile = list.next().getPath().toString(); + // make sure the file is available, saw flakes on Cloud that files were not available even if they were listed + int fileAttempt = 0; + while (!doesFileExist(pathToFile) && fileAttempt++ < 120) { + ReportUtils.report(report, getClass(), + String.format("File %s does not exist, attempt %d, will retry in 1 sec", pathToFile, fileAttempt)); + sleep(1000); + } + filesList.add(pathToFile); } ReportUtils.report(report, getClass(), filesList.toString()); ReportUtils.stopLevel(report); diff --git a/automation/src/main/java/org/greenplum/pxf/automation/structures/tables/pxf/ExternalTable.java b/automation/src/main/java/org/greenplum/pxf/automation/structures/tables/pxf/ExternalTable.java index 7e35999838..ff8b03d822 100755 --- a/automation/src/main/java/org/greenplum/pxf/automation/structures/tables/pxf/ExternalTable.java +++ b/automation/src/main/java/org/greenplum/pxf/automation/structures/tables/pxf/ExternalTable.java @@ -180,32 +180,32 @@ protected void appendParameter(StringBuilder sBuilder, String parameter) { @Override public String constructCreateStmt() { - String createStatment = ""; + String createStatement = ""; - createStatment += createHeader(); - createStatment += createFields(); - createStatment += createLocation(); + createStatement += createHeader(); + createStatement += createFields(); + createStatement += createLocation(); if (getFormat() != null) { - createStatment += " FORMAT '" + getFormat() + "'"; + createStatement += " FORMAT '" + getFormat() + "'"; } if (getFormatter() != null) { String formatterOption = isFormatterMixedCase() ? "FoRmAtTeR" : "formatter"; - createStatment += String.format(" (%s='%s'", formatterOption, getFormatter()); + createStatement += String.format(" (%s='%s'", formatterOption, getFormatter()); if (formatterOptions.size() > 0) { - createStatment += ", "; - createStatment += formatterOptions.stream().collect(Collectors.joining(", ")); + createStatement += ", "; + createStatement += formatterOptions.stream().collect(Collectors.joining(", ")); } - createStatment += ")"; + createStatement += ")"; } boolean hasDelimiterOrEscapeOrNewLine = getDelimiter() != null || getEscape() != null || getNewLine() != null; if (hasDelimiterOrEscapeOrNewLine) { - createStatment += " ("; + createStatement += " ("; } if (getDelimiter() != null) { @@ -215,7 +215,7 @@ public String constructCreateStmt() { if (!parsedDelimiter.startsWith("E")) { parsedDelimiter = "'" + parsedDelimiter + "'"; } - createStatment += " DELIMITER " + parsedDelimiter ; + createStatement += " DELIMITER " + parsedDelimiter ; } if (getEscape() != null) { @@ -225,34 +225,35 @@ public String constructCreateStmt() { if (!parsedEscapeCharacter.startsWith("E")) { parsedEscapeCharacter = "'" + parsedEscapeCharacter + "'"; } - createStatment += " ESCAPE " + parsedEscapeCharacter; + createStatement += " ESCAPE " + parsedEscapeCharacter; } if (getNewLine() != null) { String newLineCharacter = getNewLine(); - createStatment += " NEWLINE '" + newLineCharacter + "'"; + createStatement += " NEWLINE '" + newLineCharacter + "'"; } if (hasDelimiterOrEscapeOrNewLine) { - createStatment += ")"; + createStatement += ")"; } if (getEncoding() != null) { - createStatment += " ENCODING '" + getEncoding() + "'"; + createStatement += " ENCODING '" + getEncoding() + "'"; } if (getErrorTable() != null) { - createStatment += " LOG ERRORS"; + createStatement += " LOG ERRORS"; } if (getSegmentRejectLimit() > 0) { - createStatment += " SEGMENT REJECT LIMIT " + createStatement += " SEGMENT REJECT LIMIT " + getSegmentRejectLimit() + " " + getSegmentRejectLimitType(); } - return createStatment; + createStatement += distribution(); + return createStatement; } public String getHost() { diff --git a/automation/src/main/java/org/greenplum/pxf/automation/structures/tables/pxf/ForeignTable.java b/automation/src/main/java/org/greenplum/pxf/automation/structures/tables/pxf/ForeignTable.java index 6b5ff265b2..39c9334aef 100644 --- a/automation/src/main/java/org/greenplum/pxf/automation/structures/tables/pxf/ForeignTable.java +++ b/automation/src/main/java/org/greenplum/pxf/automation/structures/tables/pxf/ForeignTable.java @@ -39,20 +39,24 @@ protected String createServer() { String[] serverParameters = StringUtils.defaultIfBlank(getServer(), "default").split("="); // getServer() might return a string "server=<..>", strip the prefix int index = serverParameters.length > 1 ? 1 : 0; - return String.format(" SERVER %s_%s", serverParameters[index], getProtocol()); + // foreign server names will have underscores instead of dashes + return String.format(" SERVER %s_%s", serverParameters[index].replace("-","_"), getProtocol()); } protected String createOptions() { // foreign tables do not have locations, parameters go into options // path (resource option for FDW) should always be present StringJoiner joiner = new StringJoiner(",", " OPTIONS (", ")"); - appendOption(joiner,"resource", getPath(), true); + appendOption(joiner,"resource ", getPath(), !getPath().startsWith("E")); String formatOption = getFormatOption(); if (formatOption != null) { appendOption(joiner, "format", formatOption); } + if (getCompressionCodec() != null) { + appendOption(joiner, "compression_codec", getCompressionCodec()); + } // process F/A/R as options, they are used in tests to test column projection / predicate pushdown if (getFragmenter() != null) { appendOption(joiner, "fragmenter", getFragmenter()); @@ -72,7 +76,7 @@ protected String createOptions() { if (getEscape() != null) { // if Escape character, no need for "'" - appendOption(joiner,"delimiter", getEscape(), !getEscape().startsWith("E")); + appendOption(joiner,"escape", getEscape(), !getEscape().startsWith("E")); } if (getNewLine() != null) { @@ -184,7 +188,8 @@ private String getProtocol() { private String[] getProfileParts() { if (getProfile() == null) { - // TODO: what will we do with tests that set F/A/R directly without a profile ? + // tests that set F/A/R directly without a profile need to be registered to 'test_fdw' created for testing + // specifically that defines pseudo protocol 'test' throw new IllegalStateException("Cannot create foreign table when profile is not specified"); } return getProfile().split(":"); diff --git a/automation/src/main/java/org/greenplum/pxf/automation/structures/tables/utils/TableFactory.java b/automation/src/main/java/org/greenplum/pxf/automation/structures/tables/utils/TableFactory.java index 36a7b9fd77..fb9155dbbf 100755 --- a/automation/src/main/java/org/greenplum/pxf/automation/structures/tables/utils/TableFactory.java +++ b/automation/src/main/java/org/greenplum/pxf/automation/structures/tables/utils/TableFactory.java @@ -3,6 +3,7 @@ import java.util.ArrayList; import java.util.List; +import org.apache.commons.lang.StringUtils; import org.greenplum.pxf.automation.enums.EnumPartitionType; import org.greenplum.pxf.automation.enums.EnumPxfDefaultProfiles; @@ -497,7 +498,7 @@ private static ExternalTable getPxfJdbcReadableTable(String name, userParameters.add("USER=" + user); } if (server != null) { - userParameters.add("SERVER=" + server); + exTable.setServer("SERVER=" + server); } if (customParameters != null) { userParameters.add(customParameters); @@ -655,6 +656,29 @@ public static ExternalTable getPxfJdbcWritableTable(String name, String[] fields return getPxfJdbcWritableTable(name, fields, dataSourcePath, null, null, null, customParameter); } + /** + * Prepares PXF Readable External or Foreign Table for custom data format. + * + * @param name name of the external table which will be generated + * @param fields fields of the external table + * @param path for external table path + * @param dataFormat dataFormat for the external table + * @return PXF Readable External or Foreign table + */ + public static ReadableExternalTable getPxfReadableCustomTable(String name, + String[] fields, + String path, + String dataFormat) { + ReadableExternalTable exTable = getReadableExternalOrForeignTable(name, fields, path, "CUSTOM"); + exTable.setFormatter("pxfwritable_import"); + + if (StringUtils.isNotBlank(dataFormat)) { + exTable.setProfile(ProtocolUtils.getProtocol().value() + ":" + dataFormat); + } + + return exTable; + } + // ============ FDW Adapter ============ private static ReadableExternalTable getReadableExternalOrForeignTable (String name, String[] fields, String path, String format) { return FDWUtils.useFDW ? diff --git a/automation/src/main/java/org/greenplum/pxf/automation/utils/csv/CsvUtils.java b/automation/src/main/java/org/greenplum/pxf/automation/utils/csv/CsvUtils.java index a9b0df601b..39d0f65a54 100755 --- a/automation/src/main/java/org/greenplum/pxf/automation/utils/csv/CsvUtils.java +++ b/automation/src/main/java/org/greenplum/pxf/automation/utils/csv/CsvUtils.java @@ -1,9 +1,14 @@ package org.greenplum.pxf.automation.utils.csv; -import java.io.File; +import java.io.FileOutputStream; import java.io.FileReader; -import java.io.FileWriter; import java.io.IOException; +import java.io.OutputStreamWriter; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Arrays; import java.util.Iterator; import java.util.List; @@ -46,20 +51,44 @@ public static Table getTable(String pathToCsvFile) throws IOException { return dataTable; } + /** + * Update the delimiter in a CSV file. This helper function is required as CSVWriter only allows + * for single-character delimiters. As such, for test cases that have multi-character delimiters + * we have to update the delimiter in the file after the CSV has been created + * + * @param originalDelim Original single char delimiter + * @param newDelimiter Desired multi-char delimiter + * @throws IOException + */ + public static void updateDelim(String targetCsvFile, char originalDelim, String newDelimiter) + throws IOException { + + Path path = Paths.get(targetCsvFile); + + String content = new String(Files.readAllBytes(path), StandardCharsets.UTF_8); + content = content.replace(String.valueOf(originalDelim), newDelimiter); + Files.write(path, content.getBytes(StandardCharsets.UTF_8)); + } /** * Write {@link Table} to a CSV file. * * @param table {@link Table} contains required data list to write to CSV file * @param targetCsvFile to write the data Table + * @param charset the encoding charset to write in + * @param delimiter the separator value to use between columns + * @param quotechar the quote value to use for each col + * @param escapechar the escape value to use + * @param eol the eol value to indicate end of row * @throws IOException */ - public static void writeTableToCsvFile(Table table, String targetCsvFile) + public static void writeTableToCsvFile(Table table, String targetCsvFile, Charset charset, + char delimiter, char quotechar, + char escapechar, String eol) throws IOException { - // create CsvWriter using FileWriter - CSVWriter csvWriter = new CSVWriter(new FileWriter(targetCsvFile)); - + // create CsvWriter using OutputStreamWriter to allow for user given values + CSVWriter csvWriter = new CSVWriter(new OutputStreamWriter(new FileOutputStream(targetCsvFile), charset), delimiter, quotechar, escapechar, eol); try { // go over list and write each inner list to csv file for (List currentList : table.getData()) { @@ -75,4 +104,27 @@ public static void writeTableToCsvFile(Table table, String targetCsvFile) csvWriter.close(); } } + + /** + * Write {@link Table} to a CSV file with the default separator (delimiter), quote, escape and eol values + * + * @param table {@link Table} contains required data list to write to CSV file + * @param targetCsvFile to write the data Table + * @throws IOException + */ + public static void writeTableToCsvFile(Table table, String targetCsvFile) + throws IOException { + + // the default separator is , + // the default quote and escape values are both " + // the default eol value is \n + writeTableToCsvFile( + table, + targetCsvFile, + StandardCharsets.UTF_8, + CSVWriter.DEFAULT_SEPARATOR, + CSVWriter.DEFAULT_QUOTE_CHARACTER, + CSVWriter.DEFAULT_ESCAPE_CHARACTER, + CSVWriter.DEFAULT_LINE_END); + } } \ No newline at end of file diff --git a/automation/src/test/java/org/greenplum/pxf/automation/features/avro/HdfsReadableAvroTest.java b/automation/src/test/java/org/greenplum/pxf/automation/features/avro/HdfsReadableAvroTest.java index 07c8ce59a6..db0f79189f 100755 --- a/automation/src/test/java/org/greenplum/pxf/automation/features/avro/HdfsReadableAvroTest.java +++ b/automation/src/test/java/org/greenplum/pxf/automation/features/avro/HdfsReadableAvroTest.java @@ -1,16 +1,13 @@ package org.greenplum.pxf.automation.features.avro; -import annotations.FailsWithFDW; import annotations.WorksWithFDW; import org.greenplum.pxf.automation.components.cluster.PhdCluster; import org.greenplum.pxf.automation.datapreparer.CustomAvroPreparer; import org.greenplum.pxf.automation.features.BaseFeature; import org.greenplum.pxf.automation.fileformats.IAvroSchema; import org.greenplum.pxf.automation.structures.tables.basic.Table; -import org.greenplum.pxf.automation.structures.tables.pxf.ReadableExternalTable; import org.greenplum.pxf.automation.structures.tables.utils.TableFactory; import org.greenplum.pxf.automation.utils.fileformats.FileFormatsUtils; -import org.greenplum.pxf.automation.utils.system.ProtocolEnum; import org.greenplum.pxf.automation.utils.system.ProtocolUtils; import org.testng.annotations.Test; diff --git a/automation/src/test/java/org/greenplum/pxf/automation/features/avro/HdfsWritableAvroTest.java b/automation/src/test/java/org/greenplum/pxf/automation/features/avro/HdfsWritableAvroTest.java index 4aecd6df71..dd042f748a 100644 --- a/automation/src/test/java/org/greenplum/pxf/automation/features/avro/HdfsWritableAvroTest.java +++ b/automation/src/test/java/org/greenplum/pxf/automation/features/avro/HdfsWritableAvroTest.java @@ -1,11 +1,10 @@ package org.greenplum.pxf.automation.features.avro; -import annotations.FailsWithFDW; +import annotations.WorksWithFDW; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import org.greenplum.pxf.automation.features.BaseFeature; -import org.greenplum.pxf.automation.structures.tables.pxf.ReadableExternalTable; -import org.greenplum.pxf.automation.structures.tables.pxf.WritableExternalTable; +import org.greenplum.pxf.automation.features.BaseWritableFeature; +import org.greenplum.pxf.automation.structures.tables.basic.Table; import org.greenplum.pxf.automation.structures.tables.utils.TableFactory; import org.greenplum.pxf.automation.utils.jsystem.report.ReportUtils; import org.greenplum.pxf.automation.utils.system.ProtocolEnum; @@ -24,10 +23,9 @@ import static java.lang.Thread.sleep; import static org.junit.Assert.assertEquals; -@FailsWithFDW -public class HdfsWritableAvroTest extends BaseFeature { +@WorksWithFDW +public class HdfsWritableAvroTest extends BaseWritableFeature { - private ReadableExternalTable readableExternalTable; private ArrayList filesToDelete; private static final String[] AVRO_PRIMITIVE_WRITABLE_TABLE_COLS = new String[]{ "type_int int", @@ -105,8 +103,7 @@ public class HdfsWritableAvroTest extends BaseFeature { "type_numeric_array TEXT[]", "type_string_array TEXT[]" }; - private String gpdbTable; - private String hdfsPath; + private String tableNamePrefix; private String publicStage; private String resourcePath; private String fullTestPath; @@ -115,7 +112,7 @@ public class HdfsWritableAvroTest extends BaseFeature { @Override public void beforeClass() throws Exception { // path for storing data on HDFS (for processing by PXF) - hdfsPath = hdfs.getWorkingDirectory() + "/writableAvro/"; + hdfsWritePath = hdfs.getWorkingDirectory() + "/writableAvro/"; String absolutePath = Objects.requireNonNull(getClass().getClassLoader().getResource("data")).getPath(); resourcePath = absolutePath + "/avro/"; @@ -130,17 +127,16 @@ public void beforeMethod() throws Exception { @Test(groups = {"features", "gpdb", "hcfs", "security"}) public void generateSchemaPrimitive() throws Exception { - gpdbTable = "writable_avro_primitive_generate_schema"; - fullTestPath = hdfsPath + "generate_schema_primitive_types"; - prepareWritableExternalTable(gpdbTable, AVRO_PRIMITIVE_WRITABLE_TABLE_COLS, fullTestPath); - exTable.setUserParameters(new String[]{"COMPRESSION_CODEC=xz"}); + tableNamePrefix = "writable_avro_primitive_generate_schema"; + fullTestPath = hdfsWritePath + "generate_schema_primitive_types"; + prepareWritableExternalTable(tableNamePrefix, AVRO_PRIMITIVE_WRITABLE_TABLE_COLS, fullTestPath); + writableExTable.setUserParameters(new String[]{"COMPRESSION_CODEC=xz"}); + gpdb.createTableAndVerify(writableExTable); - prepareReadableExternalTable(gpdbTable, AVRO_PRIMITIVE_READABLE_TABLE_COLS, fullTestPath); + prepareReadableExternalTable(tableNamePrefix, AVRO_PRIMITIVE_READABLE_TABLE_COLS, fullTestPath); + gpdb.createTableAndVerify(readableExTable); - gpdb.createTableAndVerify(readableExternalTable); - gpdb.createTableAndVerify(exTable); - - insertPrimitives(gpdbTable); + insertPrimitives(writableExTable); publicStage += "generateSchemaPrimitive/"; // fetch all the segment-generated avro files and make them into json records @@ -153,17 +149,16 @@ public void generateSchemaPrimitive() throws Exception { @Test(groups = {"features", "gpdb", "hcfs", "security"}) public void generateSchemaPrimitive_withNoCompression() throws Exception { - gpdbTable = "writable_avro_primitive_no_compression"; - fullTestPath = hdfsPath + "generate_schema_primitive_types_with_no_compression"; - prepareWritableExternalTable(gpdbTable, AVRO_PRIMITIVE_WRITABLE_TABLE_COLS, fullTestPath); - exTable.setUserParameters(new String[]{"COMPRESSION_CODEC=uncompressed"}); - - prepareReadableExternalTable(gpdbTable, AVRO_PRIMITIVE_READABLE_TABLE_COLS, fullTestPath); + tableNamePrefix = "writable_avro_primitive_no_compression"; + fullTestPath = hdfsWritePath + "generate_schema_primitive_types_with_no_compression"; + prepareWritableExternalTable(tableNamePrefix, AVRO_PRIMITIVE_WRITABLE_TABLE_COLS, fullTestPath); + writableExTable.setUserParameters(new String[]{"COMPRESSION_CODEC=uncompressed"}); + gpdb.createTableAndVerify(writableExTable); - gpdb.createTableAndVerify(readableExternalTable); - gpdb.createTableAndVerify(exTable); + prepareReadableExternalTable(tableNamePrefix, AVRO_PRIMITIVE_READABLE_TABLE_COLS, fullTestPath); + gpdb.createTableAndVerify(readableExTable); - insertPrimitives(gpdbTable); + insertPrimitives(writableExTable); publicStage += "generateSchemaPrimitive_withNoCompression/"; // fetch all the segment-generated avro files and make them into json records @@ -176,18 +171,17 @@ public void generateSchemaPrimitive_withNoCompression() throws Exception { @Test(groups = {"features", "gpdb", "hcfs", "security"}) public void generateSchemaComplex() throws Exception { - gpdbTable = "writable_avro_complex_generate_schema"; + tableNamePrefix = "writable_avro_complex_generate_schema"; createComplexTypes(); - fullTestPath = hdfsPath + "generate_schema_complex_types"; - prepareWritableExternalTable(gpdbTable, AVRO_COMPLEX_TABLE_COLS_WRITABLE, fullTestPath); - exTable.setUserParameters(new String[]{"COMPRESSION_CODEC=bzip2"}); - - prepareReadableExternalTable(gpdbTable, AVRO_COMPLEX_TABLE_COLS_W_ARRAYS_READABLE, fullTestPath); + fullTestPath = hdfsWritePath + "generate_schema_complex_types"; + prepareWritableExternalTable(tableNamePrefix, AVRO_COMPLEX_TABLE_COLS_WRITABLE, fullTestPath); + writableExTable.setUserParameters(new String[]{"COMPRESSION_CODEC=bzip2"}); + gpdb.createTableAndVerify(writableExTable); - gpdb.createTableAndVerify(readableExternalTable); - gpdb.createTableAndVerify(exTable); + prepareReadableExternalTable(tableNamePrefix, AVRO_COMPLEX_TABLE_COLS_W_ARRAYS_READABLE, fullTestPath); + gpdb.createTableAndVerify(readableExTable); - insertComplex(gpdbTable); + insertComplex(writableExTable); publicStage += "generateSchemaComplex/"; // fetch all the segment-generated avro files and make them into json records @@ -200,24 +194,24 @@ public void generateSchemaComplex() throws Exception { @Test(groups = {"features", "gpdb", "hcfs", "security"}) public void userProvidedSchemaFileOnHcfsPrimitive() throws Exception { - gpdbTable = "writable_avro_primitive_user_provided_schema_on_hcfs"; - fullTestPath = hdfsPath + "primitive_user_provided_schema_on_hcfs"; - prepareWritableExternalTable(gpdbTable, AVRO_PRIMITIVE_WRITABLE_TABLE_COLS, fullTestPath); - exTable.setUserParameters(new String[]{"COMPRESSION_CODEC=snappy"}); + tableNamePrefix = "writable_avro_primitive_user_provided_schema_on_hcfs"; + fullTestPath = hdfsWritePath + "primitive_user_provided_schema_on_hcfs"; + prepareWritableExternalTable(tableNamePrefix, AVRO_PRIMITIVE_WRITABLE_TABLE_COLS, fullTestPath); + writableExTable.setUserParameters(new String[]{"COMPRESSION_CODEC=snappy"}); - prepareReadableExternalTable(gpdbTable, AVRO_PRIMITIVE_READABLE_TABLE_COLS, fullTestPath); - gpdb.createTableAndVerify(readableExternalTable); - - String schemaPath = hdfsPath.replaceFirst("/$", "_schema/primitives_no_union.avsc"); + String schemaPath = hdfsWritePath.replaceFirst("/$", "_schema/primitives_no_union.avsc"); // copy a schema file to HCFS that has no UNION types, just the raw underlying types. // the Avro files should thus be different from those without user-provided schema hdfs.copyFromLocal(resourcePath + "primitives_no_union.avsc", schemaPath); schemaPath = "/" + schemaPath; - exTable.setExternalDataSchema(schemaPath); - gpdb.createTableAndVerify(exTable); + writableExTable.setExternalDataSchema(schemaPath); + gpdb.createTableAndVerify(writableExTable); + + prepareReadableExternalTable(tableNamePrefix, AVRO_PRIMITIVE_READABLE_TABLE_COLS, fullTestPath); + gpdb.createTableAndVerify(readableExTable); - insertPrimitives(gpdbTable); + insertPrimitives(writableExTable); publicStage += "userProvidedSchemaFileOnHcfsPrimitive/"; // fetch all the segment-generated avro files and make them into json records @@ -231,17 +225,12 @@ public void userProvidedSchemaFileOnHcfsPrimitive() throws Exception { @Test(groups = {"features", "gpdb", "hcfs", "security"}) public void userProvidedSchemaFileOnClasspathComplexTypesAsText() throws Exception { createComplexTypes(); - gpdbTable = "writable_avro_complex_user_schema_on_classpath"; - fullTestPath = hdfsPath + "complex_user_schema_on_classpath"; - prepareWritableExternalTable(gpdbTable, + tableNamePrefix = "writable_avro_complex_user_schema_on_classpath"; + fullTestPath = hdfsWritePath + "complex_user_schema_on_classpath"; + prepareWritableExternalTable(tableNamePrefix, AVRO_COMPLEX_TABLE_COLS_WRITABLE, fullTestPath); - prepareReadableExternalTable(gpdbTable, - AVRO_COMPLEX_TABLE_COLS_AS_TEXT_READABLE, - fullTestPath); - gpdb.createTableAndVerify(readableExternalTable); - // copy a schema file to PXF's classpath on cluster that has no UNION types, just the raw underlying types. // the Avro files should thus be different from those without user-provided schema // this file is generated using Avro tools: http://avro.apache.org/releases.html @@ -253,10 +242,15 @@ public void userProvidedSchemaFileOnClasspathComplexTypesAsText() throws Excepti cluster.copyFileToNodes(new File(resourcePath + "complex_no_union.avro").getAbsolutePath(), cluster.getPxfConfLocation(), false, false); - exTable.setExternalDataSchema("complex_no_union.avro"); - gpdb.createTableAndVerify(exTable); + writableExTable.setExternalDataSchema("complex_no_union.avro"); + gpdb.createTableAndVerify(writableExTable); + + prepareReadableExternalTable(tableNamePrefix, + AVRO_COMPLEX_TABLE_COLS_AS_TEXT_READABLE, + fullTestPath); + gpdb.createTableAndVerify(readableExTable); - insertComplex(gpdbTable); + insertComplex(writableExTable); publicStage += "userProvidedSchemaFileOnClasspathComplex/"; // fetch all the segment-generated avro files and make them into json records @@ -269,24 +263,24 @@ public void userProvidedSchemaFileOnClasspathComplexTypesAsText() throws Excepti @Test(groups = {"features", "gpdb", "hcfs", "security"}) public void userProvidedSchemaFileGPDBArraysAsAvroArraysWithNulls() throws Exception { - gpdbTable = "writable_avro_array_user_schema_w_nulls"; - fullTestPath = hdfsPath + "array_user_schema_w_nulls"; - prepareWritableExternalTable(gpdbTable, + tableNamePrefix = "writable_avro_array_user_schema_w_nulls"; + fullTestPath = hdfsWritePath + "array_user_schema_w_nulls"; + prepareWritableExternalTable(tableNamePrefix, AVRO_ARRAY_TABLE_COLS_WRITABLE, fullTestPath); - prepareReadableExternalTable(gpdbTable, - AVRO_ARRAY_TABLE_COLS_READABLE, - fullTestPath); - gpdb.createTableAndVerify(readableExternalTable); - cluster.copyFileToNodes(new File(resourcePath + "array_with_nulls.avsc").getAbsolutePath(), cluster.getPxfConfLocation(), false, false); - exTable.setExternalDataSchema("array_with_nulls.avsc"); - gpdb.createTableAndVerify(exTable); + writableExTable.setExternalDataSchema("array_with_nulls.avsc"); + gpdb.createTableAndVerify(writableExTable); + + prepareReadableExternalTable(tableNamePrefix, + AVRO_ARRAY_TABLE_COLS_READABLE, + fullTestPath); + gpdb.createTableAndVerify(readableExTable); - insertComplexNullArrays(gpdbTable); + insertComplexNullArrays(writableExTable); publicStage += "userProvidedSchemaArrayWithNullsComplex/"; // fetch all the segment-generated avro files and make them into json records @@ -299,17 +293,16 @@ public void userProvidedSchemaFileGPDBArraysAsAvroArraysWithNulls() throws Excep @Test(groups = {"features", "gpdb", "hcfs", "security"}) public void generateSchemaWithNullValuesComplex() throws Exception { - gpdbTable = "writable_avro_null_values"; + tableNamePrefix = "writable_avro_null_values"; createComplexTypes(); - fullTestPath = hdfsPath + "null_values"; - prepareWritableExternalTable(gpdbTable, AVRO_COMPLEX_TABLE_COLS_WRITABLE, fullTestPath); - - prepareReadableExternalTable(gpdbTable, AVRO_COMPLEX_TABLE_COLS_W_ARRAYS_READABLE, fullTestPath); + fullTestPath = hdfsWritePath + "null_values"; + prepareWritableExternalTable(tableNamePrefix, AVRO_COMPLEX_TABLE_COLS_WRITABLE, fullTestPath); + gpdb.createTableAndVerify(writableExTable); - gpdb.createTableAndVerify(readableExternalTable); - gpdb.createTableAndVerify(exTable); + prepareReadableExternalTable(tableNamePrefix, AVRO_COMPLEX_TABLE_COLS_W_ARRAYS_READABLE, fullTestPath); + gpdb.createTableAndVerify(readableExTable); - insertComplexWithNulls(gpdbTable); + insertComplexWithNulls(writableExTable); publicStage += "nullValues/"; // fetch all the segment-generated avro files and make them into json records @@ -322,10 +315,10 @@ public void generateSchemaWithNullValuesComplex() throws Exception { @Override protected void afterMethod() throws Exception { - super.afterMethod(); if (ProtocolUtils.getPxfTestKeepData().equals("true")) { return; } + super.afterMethod(); if (filesToDelete != null) { for (File file : filesToDelete) { if (!file.delete()) { @@ -349,62 +342,62 @@ private void dropComplexTypes() throws Exception { } } - private void insertPrimitives(String exTable) throws Exception { - gpdb.runQuery("INSERT INTO " + exTable + "_writable " + "SELECT " + - "i, " + // type_int - "i, " + // type_smallint - "i*100000000000, " + // type_long - "i+1.0001, " + // type_float - "i*100000.0001, " + // type_double - "'row_' || i::varchar(255), " + // type_string - "('bytes for ' || i::varchar(255))::bytea, " + // type_bytes - "CASE WHEN (i%2) = 0 THEN TRUE ELSE FALSE END, " + // type_boolean - "'character row ' || i::char(3)," + // type_character - "'character varying row ' || i::varchar(3)" + // type_varchar - "from generate_series(1, 100) s(i);"); + private void insertPrimitives(Table exTable) throws Exception { + gpdb.copyData("generate_series(1, 100) s(i)", exTable, new String[]{ + "i", // type_int + "i", // type_smallint + "i*100000000000", // type_long + "i+1.0001", // type_float + "i*100000.0001", // type_double + "'row_' || i::varchar(255)", // type_string + "('bytes for ' || i::varchar(255))::bytea", // type_bytes + "CASE WHEN (i%2) = 0 THEN TRUE ELSE FALSE END", // type_boolean + "'character row ' || i::char(3)", // type_character + "'character varying row ' || i::varchar(3)" // type_varchar + }); } - private void insertComplex(String gpdbTable) throws Exception { - gpdb.runQuery("SET TIMEZONE=-7;" + "INSERT INTO " + gpdbTable + "_writable " + " SELECT " + - "i, " + - "('(' || CASE WHEN (i%2) = 0 THEN FALSE ELSE TRUE END || ',' || (i*2)::VARCHAR(255) || ')')::struct, " + - "CASE WHEN (i%2) = 0 THEN 'sad' ELSE 'happy' END::mood," + - "('{' || i::VARCHAR(255) || ',' || (i*10)::VARCHAR(255) || ',' || (i*100)::VARCHAR(255) || '}')::BIGINT[], " + - "('{' || (i*1.0001)::VARCHAR(255) || ',' || ((i*10.00001)*10)::VARCHAR(255) || ',' || ((i*100.000001)*100)::VARCHAR(255) || '}')::NUMERIC(8,1)[], " + - "('{\"item ' || ((i-1)*10)::VARCHAR(255) || '\",\"item ' || (i*10)::VARCHAR(255) || '\",\"item ' || ((i+1)*10)::VARCHAR(255) || '\"}')::TEXT[], " + - "DATE '2001-09-28' + i, " + - "TIME '00:00:00' + (i::VARCHAR(255) || ' seconds')::interval, " + - "TIMESTAMP '2001-09-28 01:00' + (i::VARCHAR(255) || ' days')::interval + (i::VARCHAR(255) || ' hours')::interval, " + - "TIMESTAMP WITH TIME ZONE '2001-09-28 01:00-07' + (i::VARCHAR(255) || ' days')::interval + (i::VARCHAR(255) || ' hours')::interval " + - "FROM generate_series(1, 100) s(i);"); + private void insertComplex(Table exTable) throws Exception { + gpdb.copyData("generate_series(1, 100) s(i)", exTable, new String[]{ + "i", + "('(' || CASE WHEN (i%2) = 0 THEN FALSE ELSE TRUE END || ',' || (i*2)::VARCHAR(255) || ')')::struct", + "CASE WHEN (i%2) = 0 THEN 'sad' ELSE 'happy' END::mood", + "('{' || i::VARCHAR(255) || ',' || (i*10)::VARCHAR(255) || ',' || (i*100)::VARCHAR(255) || '}')::BIGINT[]", + "('{' || (i*1.0001)::VARCHAR(255) || ',' || ((i*10.00001)*10)::VARCHAR(255) || ',' || ((i*100.000001)*100)::VARCHAR(255) || '}')::NUMERIC(8,1)[]", + "('{\"item ' || ((i-1)*10)::VARCHAR(255) || '\",\"item ' || (i*10)::VARCHAR(255) || '\",\"item ' || ((i+1)*10)::VARCHAR(255) || '\"}')::TEXT[]", + "DATE '2001-09-28' + i", + "TIME '00:00:00' + (i::VARCHAR(255) || ' seconds')::interval", + "TIMESTAMP '2001-09-28 01:00' + (i::VARCHAR(255) || ' days')::interval + (i::VARCHAR(255) || ' hours')::interval", + "TIMESTAMP WITH TIME ZONE '2001-09-28 01:00-07' + (i::VARCHAR(255) || ' days')::interval + (i::VARCHAR(255) || ' hours')::interval" + }, "SET TIMEZONE=-7"); } - private void insertComplexWithNulls(String gpdbTable) throws Exception { - gpdb.runQuery("SET TIMEZONE=-7;" + "INSERT INTO " + gpdbTable + "_writable " + " SELECT " + - "i, " + - "('(' || CASE WHEN (i%2) = 0 THEN FALSE ELSE TRUE END || ', ' || (i*2)::VARCHAR(255) || ')')::struct, " + - "CASE WHEN (i%3) = 0 THEN 'sad' WHEN (i%2) = 0 THEN 'happy' ELSE NULL END::mood, " + - "('{' || i::VARCHAR(255) || ',' || (i*10)::VARCHAR(255) || ',' || (i*100)::VARCHAR(255) || '}')::BIGINT[], " + - "('{' || (i*1.0001)::VARCHAR(255) || ',' || ((i*10.00001)*10)::VARCHAR(255) || ',' || ((i*100.000001)*100)::VARCHAR(255) || '}')::NUMERIC(8,1)[], " + - "('{\"item ' || ((i-1)*10)::VARCHAR(255) || '\",\"item ' || (i*10)::VARCHAR(255) || '\",\"item ' || ((i+1)*10)::VARCHAR(255) || '\"}')::TEXT[], " + - "DATE '2001-09-28' + i, " + - "TIME '00:00:00' + (i::VARCHAR(255) || ' seconds')::interval, " + - "TIMESTAMP '2001-09-28 01:00' + (i::VARCHAR(255) || ' days')::interval + (i::VARCHAR(255) || ' hours')::interval, " + - "TIMESTAMP WITH TIME ZONE '2001-09-28 01:00-07' + (i::VARCHAR(255) || ' days')::interval + (i::VARCHAR(255) || ' hours')::interval " + - "FROM generate_series(1, 100) s(i);"); + private void insertComplexWithNulls(Table exTable) throws Exception { + gpdb.copyData("generate_series(1, 100) s(i)", exTable, new String[]{ + "i", + "('(' || CASE WHEN (i%2) = 0 THEN FALSE ELSE TRUE END || ', ' || (i*2)::VARCHAR(255) || ')')::struct", + "CASE WHEN (i%3) = 0 THEN 'sad' WHEN (i%2) = 0 THEN 'happy' ELSE NULL END::mood", + "('{' || i::VARCHAR(255) || ',' || (i*10)::VARCHAR(255) || ',' || (i*100)::VARCHAR(255) || '}')::BIGINT[]", + "('{' || (i*1.0001)::VARCHAR(255) || ',' || ((i*10.00001)*10)::VARCHAR(255) || ',' || ((i*100.000001)*100)::VARCHAR(255) || '}')::NUMERIC(8,1)[]", + "('{\"item ' || ((i-1)*10)::VARCHAR(255) || '\",\"item ' || (i*10)::VARCHAR(255) || '\",\"item ' || ((i+1)*10)::VARCHAR(255) || '\"}')::TEXT[]", + "DATE '2001-09-28' + i", + "TIME '00:00:00' + (i::VARCHAR(255) || ' seconds')::interval", + "TIMESTAMP '2001-09-28 01:00' + (i::VARCHAR(255) || ' days')::interval + (i::VARCHAR(255) || ' hours')::interval", + "TIMESTAMP WITH TIME ZONE '2001-09-28 01:00-07' + (i::VARCHAR(255) || ' days')::interval + (i::VARCHAR(255) || ' hours')::interval" + }, "SET TIMEZONE=-7"); } private void - insertComplexNullArrays(String gpdbTable) throws Exception { - gpdb.runQuery("INSERT INTO " + gpdbTable + "_writable " + " VALUES " + - "(1, NULL, '{1.0001,10.00001,100.000001}', '{\"item 0\",\"item 10\",\"item 20\"}')," + - "(2, '{2,20,200}', NULL, '{\"item 0\",\"item 10\",\"item 20\"}')," + - "(3, '{3,30,300}', '{3.0001,30.00001,300.000001}', NULL )," + - "(4, '{NULL,40,400}', '{4.0001,40.00001,400.000001}', '{\"item 0\",\"item 10\",\"item 20\"}')," + - "(5, '{5,50,500}', '{5.0001,NULL,500.000001}', '{\"item 0\",\"item 10\",\"item 20\"}')," + - "(6, '{6,60,600}', '{6.0001,60.00001,600.000001}', '{\"item 0\",\"item 10\",NULL}' )," + - "(7, '{7,70,700}', '{7.0001,70.00001,700.000001}', '{\"item 0\",\"item 10\",\"item 20\"}');" - ); + insertComplexNullArrays(Table exTable) throws Exception { + String data = String.join(",", new String[] { + "(1, NULL, '{1.0001,10.00001,100.000001}', '{\"item 0\",\"item 10\",\"item 20\"}')", + "(2, '{2,20,200}', NULL, '{\"item 0\",\"item 10\",\"item 20\"}')", + "(3, '{3,30,300}', '{3.0001,30.00001,300.000001}', NULL )", + "(4, '{NULL,40,400}', '{4.0001,40.00001,400.000001}', '{\"item 0\",\"item 10\",\"item 20\"}')", + "(5, '{5,50,500}', '{5.0001,NULL,500.000001}', '{\"item 0\",\"item 10\",\"item 20\"}')", + "(6, '{6,60,600}', '{6.0001,60.00001,600.000001}', '{\"item 0\",\"item 10\",NULL}' )", + "(7, '{7,70,700}', '{7.0001,70.00001,700.000001}', '{\"item 0\",\"item 10\",\"item 20\"}')"}); + gpdb.insertData(data, exTable); } private void fetchAndVerifyAvroHcfsFiles(String compareFile, String codec) throws Exception { @@ -415,7 +408,7 @@ private void fetchAndVerifyAvroHcfsFiles(String compareFile, String codec) throw addJsonNodesToMap(jsonToCompare, resourcePath + compareFile); // for HCFS on Cloud, wait a bit for async write in previous steps to finish - if (protocol != ProtocolEnum.HDFS && protocol != ProtocolEnum.FILE) { + if (protocol != ProtocolEnum.HDFS) { sleep(10000); } for (String srcPath : hdfs.list(fullTestPath)) { @@ -423,11 +416,6 @@ private void fetchAndVerifyAvroHcfsFiles(String compareFile, String codec) throw final String filePath = publicStage + fileName; filesToDelete.add(new File(filePath + ".avro")); filesToDelete.add(new File(publicStage + "." + fileName + ".avro.crc")); - // make sure the file is available, saw flakes on Cloud that listed files were not available - int attempts = 0; - while (!hdfs.doesFileExist(srcPath) && attempts++ < 20) { - sleep(1000); - } hdfs.copyToLocal(srcPath, filePath + ".avro"); sleep(250); hdfs.writeJsonFileFromAvro("file://" + filePath + ".avro", filePath + ".json"); @@ -481,12 +469,12 @@ private void addJsonNodesToMap(Map map, String filePath) { } private void prepareWritableExternalTable(String name, String[] fields, String path) { - exTable = TableFactory.getPxfHcfsWritableTable(name + "_writable", + writableExTable = TableFactory.getPxfHcfsWritableTable(name + "_writable", fields, path, hdfs.getBasePath(), "avro"); } private void prepareReadableExternalTable(String name, String[] fields, String path) { - readableExternalTable = TableFactory.getPxfHcfsReadableTable(name + "_readable", + readableExTable = TableFactory.getPxfHcfsReadableTable(name + "_readable", fields, path, hdfs.getBasePath(),"avro"); } } diff --git a/automation/src/test/java/org/greenplum/pxf/automation/features/cloud/CloudAccessTest.java b/automation/src/test/java/org/greenplum/pxf/automation/features/cloud/CloudAccessTest.java index 4894439a8a..340229b31a 100644 --- a/automation/src/test/java/org/greenplum/pxf/automation/features/cloud/CloudAccessTest.java +++ b/automation/src/test/java/org/greenplum/pxf/automation/features/cloud/CloudAccessTest.java @@ -1,5 +1,6 @@ package org.greenplum.pxf.automation.features.cloud; +import annotations.WorksWithFDW; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.greenplum.pxf.automation.components.hdfs.Hdfs; @@ -15,6 +16,7 @@ /** * Functional CloudAccess Test */ +@WorksWithFDW public class CloudAccessTest extends BaseFeature { private static final String PROTOCOL_S3 = "s3a://"; @@ -115,7 +117,7 @@ public void testCloudAccessOkWhenServerCredsNoConfigFileExists() throws Exceptio /* * The tests below are for the case where there's a Hadoop cluster configured under "default" server - * both without and with Kerberos security, testing that clud access works in presence of "default" server + * both without and with Kerberos security, testing that cloud access works in presence of "default" server */ @Test(groups = {"gpdb", "security"}) diff --git a/automation/src/test/java/org/greenplum/pxf/automation/features/columnprojection/ColumnProjectionTest.java b/automation/src/test/java/org/greenplum/pxf/automation/features/columnprojection/ColumnProjectionTest.java index d8da5f3112..9ad9150b10 100755 --- a/automation/src/test/java/org/greenplum/pxf/automation/features/columnprojection/ColumnProjectionTest.java +++ b/automation/src/test/java/org/greenplum/pxf/automation/features/columnprojection/ColumnProjectionTest.java @@ -1,16 +1,17 @@ package org.greenplum.pxf.automation.features.columnprojection; -import annotations.FailsWithFDW; +import annotations.WorksWithFDW; import org.greenplum.pxf.automation.components.cluster.PhdCluster; import org.greenplum.pxf.automation.features.BaseFeature; import org.greenplum.pxf.automation.structures.tables.pxf.ReadableExternalTable; import org.greenplum.pxf.automation.structures.tables.utils.TableFactory; +import org.greenplum.pxf.automation.utils.system.FDWUtils; import org.testng.annotations.Test; import java.io.File; /** Functional PXF column projection cases */ -@FailsWithFDW +@WorksWithFDW public class ColumnProjectionTest extends BaseFeature { String testPackageLocation = "/org/greenplum/pxf/automation/testplugin/"; @@ -54,7 +55,91 @@ public void checkColumnProjection() throws Exception { // SELECT t0, colprojvalue FROM test_column_projection GROUP BY t0, colprojvalue HAVING AVG(a1) < 5 ORDER BY t0; // SELECT b.value, a.colprojvalue FROM test_column_projection a JOIN t0_values b ON a.t0 = b.key; if (gpdb.getVersion() >= 7) { - runTincTest("pxf.features.columnprojection.checkColumnProjection_gp7.runTest"); + /* The below query (mentioned in above comment as well) is propagating for FDW but not for external-table, + * so use a different test set for FDW. + * The Call stack is different in case of external-table and FDW. + + SELECT b.value, a.colprojvalue FROM test_column_projection a JOIN t0_values b ON a.t0 = b.key; + value | colprojvalue + -------+----------------- + 50 | t0|colprojvalue + (1 row) + + Following are the explain plans for the external-table and FDW for the same query, + The different explain plans explains that for one it is projecting and for other it's not. + External Table: + + pxfautomation=# \d+ e_test_column_projection + Foreign table "public.e_test_column_projection" + Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description + --------------+---------+-----------+----------+---------+-------------+----------+--------------+------------- + t0 | text | | | | | extended | | + a1 | integer | | | | | plain | | + b2 | boolean | | | | | plain | | + colprojvalue | text | | | | | extended | | + FDW options: (format 'text', delimiter ',', "null" E'\\N', escape E'\\', format_type 't', location_uris 'pxf://dummy_path?PROFILE=test:text&FRAGMENTER=org.greenplum.pxf.automation.testplugin.ColumnProjectionVerifyFragmenter&ACCESSOR=org.greenplum.pxf.automation.testplugin.ColumnProjectionVerifyAccessor&RESOLVER=org.greenplum.pxf.plugins.hdfs.StringPassResolver', execute_on 'ALL_SEGMENTS', log_errors 'f', encoding '6', is_writable 'false') + + pxfautomation=# explain analyze SELECT b.value, a.colprojvalue FROM e_test_column_projection a JOIN t0_values b ON a.t0 = b.key; + QUERY PLAN + --------------------------------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=2306.08..20789139.42 rows=77900000 width=36) (actual time=78.603..78.606 rows=1 loops=1) + -> Hash Join (cost=2306.08..19750472.75 rows=25966667 width=36) (actual time=52.602..63.398 rows=1 loops=1) + Hash Cond: (a.t0 = (b.key)::text) + Extra Text: (seg0) Hash chain length 1.0 avg, 1 max, using 1 of 524288 buckets. + -> Foreign Scan on e_test_column_projection a (cost=0.00..11000.00 rows=1000000 width=64) (actual time=51.045..51.076 rows=10 loops=1) + -> Hash (cost=1332.33..1332.33 rows=77900 width=12) (actual time=0.039..0.040 rows=1 loops=1) + Buckets: 524288 Batches: 1 Memory Usage: 4097kB + -> Broadcast Motion 3:3 (slice2; segments: 3) (cost=0.00..1332.33 rows=77900 width=12) (actual time=0.013..0.014 rows=1 loops=1) + -> Seq Scan on t0_values b (cost=0.00..293.67 rows=25967 width=12) (actual time=1.760..1.762 rows=1 loops=1) + Optimizer: Postgres query optimizer + Planning Time: 1.151 ms + (slice0) Executor memory: 106K bytes. + (slice1) Executor memory: 4253K bytes avg x 3 workers, 4253K bytes max (seg0). Work_mem: 4097K bytes max. + (slice2) Executor memory: 37K bytes avg x 3 workers, 37K bytes max (seg0). + Memory used: 128000kB + Execution Time: 8581.766 ms + (16 rows) + + FDW: + + pxfautomation=# \d+ test_column_projection + Foreign table "public.test_column_projection" + Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description + --------------+---------+-----------+----------+---------+-------------+----------+--------------+------------- + t0 | text | | | | | extended | | + a1 | integer | | | | | plain | | + b2 | boolean | | | | | plain | | + colprojvalue | text | | | | | extended | | + Server: default_test + FDW options: (resource 'dummy_path', format 'text', fragmenter 'org.greenplum.pxf.automation.testplugin.ColumnProjectionVerifyFragmenter', accessor 'org.greenplum.pxf.automation.testplugin.ColumnProjectionVerifyAccessor', resolver 'org.greenplum.pxf.plugins.hdfs.StringPassResolver', delimiter ',') + + pxfautomation=# explain analyze SELECT b.value, a.colprojvalue FROM test_column_projection a JOIN t0_values b ON a.t0 = b.key; + QUERY PLAN + ------------------------------------------------------------------------------------------------------------------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) (cost=50077.50..52328.93 rows=77900 width=36) (actual time=117.120..117.133 rows=1 loops=1) + -> Hash Join (cost=50077.50..51290.27 rows=25967 width=36) (actual time=111.568..112.709 rows=1 loops=1) + Hash Cond: ((b.key)::text = a.t0) + Extra Text: (seg2) Hash chain length 1.0 avg, 1 max, using 10 of 262144 buckets. + -> Seq Scan on t0_values b (cost=0.00..293.67 rows=25967 width=12) (actual time=0.964..0.966 rows=1 loops=1) + -> Hash (cost=50040.00..50040.00 rows=3000 width=64) (actual time=110.975..110.976 rows=10 loops=1) + Buckets: 262144 Batches: 1 Memory Usage: 2049kB + -> Broadcast Motion 3:3 (slice2; segments: 3) (cost=50000.00..50040.00 rows=3000 width=64) (actual time=110.902..110.906 rows=10 loops=1) + -> Foreign Scan on test_column_projection a (cost=50000.00..50000.00 rows=1000 width=64) (actual time=1.312..1.329 rows=10 loops=1) + Optimizer: Postgres query optimizer + Planning Time: 0.592 ms + (slice0) Executor memory: 38K bytes. + (slice1) Executor memory: 2101K bytes avg x 3 workers, 2101K bytes max (seg0). Work_mem: 2049K bytes max. + (slice2) Executor memory: 90K bytes avg x 3 workers, 97K bytes max (seg2). + Memory used: 128000kB + Execution Time: 117.692 ms + (16 rows) + */ + if (FDWUtils.useFDW) { + runTincTest("pxf.features.columnprojection.checkColumnProjection_fdw.runTest"); + } + else { + runTincTest("pxf.features.columnprojection.checkColumnProjection_gp7.runTest"); + } } else { runTincTest("pxf.features.columnprojection.checkColumnProjection.runTest"); } diff --git a/automation/src/test/java/org/greenplum/pxf/automation/features/extension/PxfExtensionTest.java b/automation/src/test/java/org/greenplum/pxf/automation/features/extension/PxfExtensionTest.java new file mode 100644 index 0000000000..301dcdf369 --- /dev/null +++ b/automation/src/test/java/org/greenplum/pxf/automation/features/extension/PxfExtensionTest.java @@ -0,0 +1,135 @@ +package org.greenplum.pxf.automation.features.extension; + +import org.greenplum.pxf.automation.BaseFunctionality; +import org.greenplum.pxf.automation.structures.tables.basic.Table; +import org.greenplum.pxf.automation.structures.tables.pxf.ReadableExternalTable; +import org.greenplum.pxf.automation.structures.tables.utils.TableFactory; +import org.testng.annotations.Test; + +public class PxfExtensionTest extends BaseFunctionality { + + public static final String[] FIELDS = { + "name text", + "num integer", + "dub double precision", + "longNum bigint", + "bool boolean" + }; + + private ReadableExternalTable externalTable; + private String location; + private String location_multi; + + @Override + public void beforeClass() throws Exception { + super.beforeClass(); + gpdb.dropDataBase("pxfautomation_extension", true, true); + gpdb.createDataBase("pxfautomation_extension", false); + gpdb.setDb("pxfautomation_extension"); + gpdb.connectToDataBase("pxfautomation_extension"); + + Table smallData = getSmallData("", 10); + + location = hdfs.getWorkingDirectory() + "/upgrade-test-data.txt"; + hdfs.writeTableToFile(location, smallData, ","); + + location_multi = hdfs.getWorkingDirectory() + "/upgrade-test-data_multibyte.txt"; + hdfs.writeTableToFile(location_multi, smallData, "停"); + } + + @Override + public void beforeMethod() throws Exception { + gpdb.setDb("pxfautomation_extension"); + gpdb.connectToDataBase("pxfautomation_extension"); + gpdb.runQuery("DROP EXTENSION IF EXISTS pxf CASCADE", true, false); + } + + @Test(groups = {"gpdb", "pxfExtensionVersion2_1"}) + public void testPxfCreateExtension() throws Exception { + gpdb.runQuery("CREATE EXTENSION pxf"); + // create a regular external table + createReadablePxfTable("default", location, false); + // create an external table with the multibyte formatter + createReadablePxfTable("default", location_multi, true); + runTincTest("pxf.features.extension_tests.create_extension.runTest"); + } + + @Test(groups = {"pxfExtensionVersion2"}) + public void testPxfCreateExtensionOldRPM() throws Exception { + gpdb.runQuery("CREATE EXTENSION pxf"); + // create a regular external table + createReadablePxfTable("default", location, false); + // create an external table with the multibyte formatter + createReadablePxfTable("default", location_multi, true); + runTincTest("pxf.features.extension_tests.create_extension_rpm.runTest"); + } + + @Test(groups = {"gpdb", "pxfExtensionVersion2_1"}) + public void testPxfUpgrade() throws Exception { + gpdb.runQuery("CREATE EXTENSION pxf VERSION '2.0'"); + createReadablePxfTable("default", location, false); + runTincTest("pxf.features.extension_tests.upgrade.step_1_create_extension_with_older_pxf_version.runTest"); + + // create an external table with the multibyte formatter + createReadablePxfTable("default", location_multi, true); + gpdb.runQuery("ALTER EXTENSION pxf UPDATE"); + runTincTest("pxf.features.extension_tests.upgrade.step_2_after_alter_extension.runTest"); + } + + @Test(groups = {"gpdb", "pxfExtensionVersion2_1"}) + public void testPxfExplicitUpgrade() throws Exception { + gpdb.runQuery("CREATE EXTENSION pxf VERSION '2.0'"); + createReadablePxfTable("default", location, false); + runTincTest("pxf.features.extension_tests.explicit_upgrade.step_1_create_extension_with_older_pxf_version.runTest"); + + // create an external table with the multibyte formatter + createReadablePxfTable("default", location_multi, true); + gpdb.runQuery("ALTER EXTENSION pxf UPDATE TO '2.1'"); + runTincTest("pxf.features.extension_tests.explicit_upgrade.step_2_after_alter_extension.runTest"); + } + + @Test(groups = {"gpdb", "pxfExtensionVersion2_1"}) + public void testPxfDowngrade() throws Exception { + gpdb.runQuery("CREATE EXTENSION pxf"); + + createReadablePxfTable("default", location, false); + // create an external table with the multibyte formatter + createReadablePxfTable("default", location_multi, true); + runTincTest("pxf.features.extension_tests.downgrade.step_1_create_extension.runTest"); + + gpdb.runQuery("ALTER EXTENSION pxf UPDATE TO '2.0'"); + runTincTest("pxf.features.extension_tests.downgrade.step_2_after_alter_extension_downgrade.runTest"); + } + + @Test(groups = {"gpdb", "pxfExtensionVersion2_1"}) + public void testPxfDowngradeThenUpgradeAgain() throws Exception { + gpdb.runQuery("CREATE EXTENSION pxf"); + + createReadablePxfTable("default", location, false); + // create an external table with the multibyte formatter + createReadablePxfTable("default", location_multi, true); + runTincTest("pxf.features.extension_tests.downgrade_then_upgrade.step_1_check_extension.runTest"); + + gpdb.runQuery("ALTER EXTENSION pxf UPDATE TO '2.0'"); + runTincTest("pxf.features.extension_tests.downgrade_then_upgrade.step_2_after_alter_extension_downgrade.runTest"); + + gpdb.runQuery("ALTER EXTENSION pxf UPDATE TO '2.1'"); + runTincTest("pxf.features.extension_tests.downgrade_then_upgrade.step_3_after_alter_extension_upgrade.runTest"); + } + + private void createReadablePxfTable(String serverName, String location, boolean multi) throws Exception { + if (multi) { + externalTable = TableFactory.getPxfReadableTextTable("pxf_upgrade_test_multibyte", FIELDS, location, null); + externalTable.setFormat("CUSTOM"); + externalTable.setFormatter("pxfdelimited_import"); + externalTable.addFormatterOption("delimiter='停'"); + } else { + externalTable = TableFactory.getPxfReadableTextTable("pxf_upgrade_test", FIELDS, location, ","); + } + externalTable.setHost(pxfHost); + externalTable.setPort(pxfPort); + externalTable.setServer("SERVER=" + serverName); + gpdb.createTableAndVerify(externalTable); + } + +} diff --git a/automation/src/test/java/org/greenplum/pxf/automation/features/general/AlterTableTest.java b/automation/src/test/java/org/greenplum/pxf/automation/features/general/AlterTableTest.java index af2fd8c94a..a7d0d652bf 100644 --- a/automation/src/test/java/org/greenplum/pxf/automation/features/general/AlterTableTest.java +++ b/automation/src/test/java/org/greenplum/pxf/automation/features/general/AlterTableTest.java @@ -1,13 +1,14 @@ package org.greenplum.pxf.automation.features.general; +import annotations.FailsWithFDW; +import annotations.WorksWithFDW; import org.greenplum.pxf.automation.features.BaseFeature; import org.greenplum.pxf.automation.structures.tables.basic.Table; -import org.greenplum.pxf.automation.structures.tables.pxf.ReadableExternalTable; -import org.greenplum.pxf.automation.structures.tables.pxf.WritableExternalTable; import org.greenplum.pxf.automation.structures.tables.utils.TableFactory; import org.greenplum.pxf.automation.utils.system.ProtocolUtils; import org.testng.annotations.Test; +@WorksWithFDW public class AlterTableTest extends BaseFeature { private static final String AVRO_TYPES_FILE_NAME = "supported_primitive_types"; @@ -89,47 +90,38 @@ public void beforeClass() throws Exception { @Test(groups = {"features", "gpdb", "security"}) public void dropAndAddColumnsPxfWritableImportWithColumnProjectionSupport() throws Exception { - exTable = new ReadableExternalTable(PXF_ALTER_PARQUET_TABLE, - PARQUET_TABLE_COLUMNS, hdfsPath + "/parquet/" + PARQUET_PRIMITIVE_TYPES, "custom"); - exTable.setHost(pxfHost); - exTable.setPort(pxfPort); - exTable.setFormatter("pxfwritable_import"); - exTable.setProfile(ProtocolUtils.getProtocol().value() + ":parquet"); - - gpdb.createTableAndVerify(exTable); + exTable = TableFactory.getPxfReadableCustomTable(PXF_ALTER_PARQUET_TABLE, + PARQUET_TABLE_COLUMNS, hdfsPath + "/parquet/" + PARQUET_PRIMITIVE_TYPES, "parquet"); + setParamsAndVerifyTable(); runTincTest("pxf.features.general.alter.pxfwritable_import.with_column_projection.runTest"); } + // TODO: Determine the reason why FDW is failing with the below class cast exception: + // + // ERROR: PXF server error : class java.io.DataInputStream cannot be cast to class java.lang.String + // (java.io.DataInputStream and java.lang.String are in module java.base of loader 'bootstrap') + @FailsWithFDW @Test(groups = {"features", "gpdb", "security"}) public void dropColumnsPxfWritableExport() throws Exception { // Create source table - exTable = new ReadableExternalTable(PXF_PARQUET_TABLE_SOURCE, - PARQUET_TABLE_COLUMNS, hdfsPath + "/parquet/" + PARQUET_PRIMITIVE_TYPES, "custom"); - exTable.setHost(pxfHost); - exTable.setPort(pxfPort); - exTable.setFormatter("pxfwritable_import"); - exTable.setProfile(ProtocolUtils.getProtocol().value() + ":parquet"); - gpdb.createTableAndVerify(exTable); + exTable = TableFactory.getPxfReadableCustomTable(PXF_PARQUET_TABLE_SOURCE, + PARQUET_TABLE_COLUMNS, hdfsPath + "/parquet/" + PARQUET_PRIMITIVE_TYPES, "parquet"); + setParamsAndVerifyTable(); // Create writable table - exTable = new WritableExternalTable(PXF_ALTER_WRITE_PARQUET_TABLE, - PARQUET_TABLE_COLUMNS, hdfsPath + "/parquet-write/" + PARQUET_WRITE_PRIMITIVES, "custom"); - exTable.setHost(pxfHost); - exTable.setPort(pxfPort); + exTable = TableFactory.getPxfWritableTextTable(PXF_ALTER_WRITE_PARQUET_TABLE, + PARQUET_TABLE_COLUMNS, hdfsPath + "/parquet-write/" + PARQUET_WRITE_PRIMITIVES, null); exTable.setFormatter("pxfwritable_export"); exTable.setProfile(ProtocolUtils.getProtocol().value() + ":parquet"); - gpdb.createTableAndVerify(exTable); + exTable.setFormat("CUSTOM"); + setParamsAndVerifyTable(); // Create validation table - exTable = new ReadableExternalTable(PXF_ALTER_WRITE_PARQUET_TABLE + "_r", - PARQUET_TABLE_SUBSET_COLUMNS, hdfsPath + "/parquet-write/" + PARQUET_WRITE_PRIMITIVES, "custom"); - exTable.setHost(pxfHost); - exTable.setPort(pxfPort); - exTable.setFormatter("pxfwritable_import"); - exTable.setProfile(ProtocolUtils.getProtocol().value() + ":parquet"); - gpdb.createTableAndVerify(exTable); + exTable = TableFactory.getPxfReadableCustomTable(PXF_ALTER_WRITE_PARQUET_TABLE + "_r", + PARQUET_TABLE_SUBSET_COLUMNS, hdfsPath + "/parquet-write/" + PARQUET_WRITE_PRIMITIVES, "parquet"); + setParamsAndVerifyTable(); runTincTest("pxf.features.general.alter.pxfwritable_export.parquet.runTest"); } @@ -137,7 +129,7 @@ public void dropColumnsPxfWritableExport() throws Exception { @Test(groups = {"features", "gpdb", "security"}) public void dropAndAddColumnsPxfWritableImportWithoutColumnProjectionSupport() throws Exception { // default external table with common settings - exTable = new ReadableExternalTable(PXF_ALTER_AVRO_TABLE, new String[]{ + exTable = TableFactory.getPxfReadableCustomTable(PXF_ALTER_AVRO_TABLE, new String[]{ "type_int int", "type_double float8", "type_string text", @@ -145,13 +137,8 @@ public void dropAndAddColumnsPxfWritableImportWithoutColumnProjectionSupport() t "col_does_not_exist text", "type_long bigint", "type_bytes bytea", - "type_boolean bool"}, hdfsPath + "/avro/" + AVRO_TYPES_FILE_NAME + SUFFIX_AVRO, "custom"); - exTable.setHost(pxfHost); - exTable.setPort(pxfPort); - exTable.setFormatter("pxfwritable_import"); - exTable.setProfile(ProtocolUtils.getProtocol().value() + ":avro"); - - gpdb.createTableAndVerify(exTable); + "type_boolean bool"}, hdfsPath + "/avro/" + AVRO_TYPES_FILE_NAME + SUFFIX_AVRO, "avro"); + setParamsAndVerifyTable(); // Verify results runTincTest("pxf.features.general.alter.pxfwritable_import.without_column_projection.runTest"); @@ -178,10 +165,15 @@ public void dropAndAddColumsCsv() throws Exception { "longNum bigint", "bool boolean" }, hdfsPath + "/csv/" + fileName, ","); + setParamsAndVerifyTable(); + + runTincTest("pxf.features.general.alter.csv.runTest"); + } + + private void setParamsAndVerifyTable() throws Exception + { exTable.setHost(pxfHost); exTable.setPort(pxfPort); gpdb.createTableAndVerify(exTable); - - runTincTest("pxf.features.general.alter.csv.runTest"); } } diff --git a/automation/src/test/java/org/greenplum/pxf/automation/features/general/FailOverTest.java b/automation/src/test/java/org/greenplum/pxf/automation/features/general/FailOverTest.java index 28e3ba954a..e52686acec 100644 --- a/automation/src/test/java/org/greenplum/pxf/automation/features/general/FailOverTest.java +++ b/automation/src/test/java/org/greenplum/pxf/automation/features/general/FailOverTest.java @@ -1,5 +1,6 @@ package org.greenplum.pxf.automation.features.general; +import annotations.FailsWithFDW; import org.greenplum.pxf.automation.components.cluster.PhdCluster; import org.greenplum.pxf.automation.features.BaseFeature; import org.greenplum.pxf.automation.structures.tables.pxf.ReadableExternalTable; @@ -8,6 +9,7 @@ import java.io.File; /** Tests how failures are handled **/ +@FailsWithFDW public class FailOverTest extends BaseFeature { String testPackageLocation = "/org/greenplum/pxf/automation/testplugin/"; diff --git a/automation/src/test/java/org/greenplum/pxf/automation/features/gpupgrade/GpupgradeTest.java b/automation/src/test/java/org/greenplum/pxf/automation/features/gpupgrade/GpupgradeTest.java index 9ebe53912d..ec04195dd5 100644 --- a/automation/src/test/java/org/greenplum/pxf/automation/features/gpupgrade/GpupgradeTest.java +++ b/automation/src/test/java/org/greenplum/pxf/automation/features/gpupgrade/GpupgradeTest.java @@ -34,6 +34,23 @@ protected void afterMethod() throws Exception { super.afterMethod(); } + @Test(groups = {"features", "gpdb"}) + public void testGpdbUpgradeExtensionVersion2_0Scenario() throws Exception { + + // Skipping this test for GP7 since this isn't passing for GP7 + if (gpdb.getVersion() >= 7) + throw new SkipException("Skipping testGpdbUpgradeScenario for GPDB7"); + + gpdb.runQuery("ALTER EXTENSION pxf UPDATE TO '2.0'"); + runTincTest("pxf.features.gpupgrade.extension2_0.step_1_before_running_pxf_pre_gpupgrade.runTest"); + + cluster.runCommand("pxf-pre-gpupgrade"); + runTincTest("pxf.features.gpupgrade.extension2_0.step_2_after_running_pxf_pre_gpupgrade.runTest"); + + cluster.runCommand("pxf-post-gpupgrade"); + runTincTest("pxf.features.gpupgrade.extension2_0.step_3_after_running_pxf_post_gpupgrade.runTest"); + } + @Test(groups = {"features", "gpdb"}) public void testGpdbUpgradeScenario() throws Exception { @@ -41,13 +58,14 @@ public void testGpdbUpgradeScenario() throws Exception { if (gpdb.getVersion() >= 7) throw new SkipException("Skipping testGpdbUpgradeScenario for GPDB7"); - runTincTest("pxf.features.gpupgrade.step_1_before_running_pxf_pre_gpupgrade.runTest"); + gpdb.runQuery("ALTER EXTENSION pxf UPDATE TO '2.1'"); + runTincTest("pxf.features.gpupgrade.extension2_1.step_1_before_running_pxf_pre_gpupgrade.runTest"); cluster.runCommand("pxf-pre-gpupgrade"); - runTincTest("pxf.features.gpupgrade.step_2_after_running_pxf_pre_gpupgrade.runTest"); + runTincTest("pxf.features.gpupgrade.extension2_1.step_2_after_running_pxf_pre_gpupgrade.runTest"); cluster.runCommand("pxf-post-gpupgrade"); - runTincTest("pxf.features.gpupgrade.step_3_after_running_pxf_post_gpupgrade.runTest"); + runTincTest("pxf.features.gpupgrade.extension2_1.step_3_after_running_pxf_post_gpupgrade.runTest"); } private String prepareData() throws Exception { diff --git a/automation/src/test/java/org/greenplum/pxf/automation/features/hcfs/FileAsRowTest.java b/automation/src/test/java/org/greenplum/pxf/automation/features/hcfs/FileAsRowTest.java index dff1970581..1c27159a9d 100644 --- a/automation/src/test/java/org/greenplum/pxf/automation/features/hcfs/FileAsRowTest.java +++ b/automation/src/test/java/org/greenplum/pxf/automation/features/hcfs/FileAsRowTest.java @@ -1,7 +1,9 @@ package org.greenplum.pxf.automation.features.hcfs; +import annotations.SkipForFDW; +import annotations.WorksWithFDW; import org.greenplum.pxf.automation.features.BaseFeature; -import org.greenplum.pxf.automation.structures.tables.pxf.ReadableExternalTable; +import org.greenplum.pxf.automation.structures.tables.utils.TableFactory; import org.greenplum.pxf.automation.utils.system.ProtocolEnum; import org.greenplum.pxf.automation.utils.system.ProtocolUtils; import org.testng.annotations.Test; @@ -9,6 +11,7 @@ /** * Functional File as Row Test */ +@WorksWithFDW public class FileAsRowTest extends BaseFeature { private static final String emptyTextFile = "empty"; @@ -63,6 +66,15 @@ public void testMultilineFile() throws Exception { hdfsBasePath + multiLineTextFile, srcPaths); } + /* + * This test fails for only GP6 with the Error: + * ERROR: set-valued function called in context that cannot accept a set (seg2 slice1 10.254.0.190:6002 pid=12873) + * + * Logged a BUG with GPDB: + * https://github.com/greenplum-db/gpdb/issues/15762 + * Once this is fixed, we can remove the "SkipForFDW" annotation + */ + @SkipForFDW @Test(groups = {"gpdb", "hcfs", "security"}) public void testMultilineJsonFile() throws Exception { String hdfsBasePath = hdfs.getWorkingDirectory() + "/file_as_row/"; @@ -131,7 +143,7 @@ private void runTestScenario(String name, String[] fields, String hdfsPath, Stri String tableName = "file_as_row_" + name; ProtocolEnum protocol = ProtocolUtils.getProtocol(); - exTable = new ReadableExternalTable(tableName, fields, protocol.getExternalTablePath(hdfs.getBasePath(), locationPath), "CSV"); + exTable = TableFactory.getPxfReadableCSVTable(tableName, fields, protocol.getExternalTablePath(hdfs.getBasePath(), locationPath), ","); exTable.setProfile(protocol.value() + ":text:multi"); exTable.setUserParameters(new String[]{"FILE_AS_ROW=true"}); gpdb.createTableAndVerify(exTable); diff --git a/automation/src/test/java/org/greenplum/pxf/automation/features/hcfs/HcfsGlobbingTest.java b/automation/src/test/java/org/greenplum/pxf/automation/features/hcfs/HcfsGlobbingTest.java index 094e2b4ab1..8440730e61 100644 --- a/automation/src/test/java/org/greenplum/pxf/automation/features/hcfs/HcfsGlobbingTest.java +++ b/automation/src/test/java/org/greenplum/pxf/automation/features/hcfs/HcfsGlobbingTest.java @@ -1,8 +1,10 @@ package org.greenplum.pxf.automation.features.hcfs; +import annotations.WorksWithFDW; import org.greenplum.pxf.automation.features.BaseFeature; import org.greenplum.pxf.automation.structures.tables.basic.Table; import org.greenplum.pxf.automation.structures.tables.utils.TableFactory; +import org.greenplum.pxf.automation.utils.system.FDWUtils; import org.greenplum.pxf.automation.utils.system.ProtocolEnum; import org.greenplum.pxf.automation.utils.system.ProtocolUtils; import org.testng.annotations.Test; @@ -11,6 +13,7 @@ * Functional Globbing Tests. Tests are based on Hadoop Glob Tests * https://github.com/apache/hadoop/blob/rel/release-3.2.1/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java */ +@WorksWithFDW public class HcfsGlobbingTest extends BaseFeature { public static final String[] FIELDS = { @@ -85,6 +88,7 @@ public void testMatchAStringFromStringSet() throws Exception { prepareTestScenario("match_string_from_string_set_4", "match_string_from_string_set_10", null, null, null, null, "}{ac,?}"); // test ill-formed curly prepareTestScenario("match_string_from_string_set_4", "match_string_from_string_set_11", null, null, null, null, "}{bc"); + // test escape curly prepareTestScenario("match_string_from_string_set_12", "}{bc", "}bc", null, null, "}\\\\{bc"); runTestScenario("match_string_from_string_set"); @@ -125,10 +129,16 @@ private void prepareTestScenario(String path, String testName, String data1, Str ProtocolEnum protocol = ProtocolUtils.getProtocol(); // Create GPDB external table directed to the HDFS file + + String updatedPath = protocol.getExternalTablePath(hdfs.getBasePath(), hdfs.getWorkingDirectory()) + "/" + path + "/" + glob; + if (FDWUtils.useFDW) { + updatedPath = "E'"+ updatedPath +"'"; + } + exTable = TableFactory.getPxfReadableTextTable( "hcfs_glob_" + testName, FIELDS, - protocol.getExternalTablePath(hdfs.getBasePath(), hdfs.getWorkingDirectory()) + "/" + path + "/" + glob, + updatedPath, ","); exTable.setHost(pxfHost); exTable.setPort(pxfPort); diff --git a/automation/src/test/java/org/greenplum/pxf/automation/features/hcfs/HcfsTextTest.java b/automation/src/test/java/org/greenplum/pxf/automation/features/hcfs/HcfsTextTest.java index af08f5c997..ff634f647a 100644 --- a/automation/src/test/java/org/greenplum/pxf/automation/features/hcfs/HcfsTextTest.java +++ b/automation/src/test/java/org/greenplum/pxf/automation/features/hcfs/HcfsTextTest.java @@ -1,8 +1,10 @@ package org.greenplum.pxf.automation.features.hcfs; - +import annotations.SkipForFDW; +import annotations.WorksWithFDW; import org.greenplum.pxf.automation.features.BaseFeature; -import org.greenplum.pxf.automation.structures.tables.pxf.ReadableExternalTable; +import org.greenplum.pxf.automation.structures.tables.utils.TableFactory; +import org.greenplum.pxf.automation.utils.system.FDWUtils; import org.greenplum.pxf.automation.utils.system.ProtocolEnum; import org.greenplum.pxf.automation.utils.system.ProtocolUtils; import org.testng.annotations.Test; @@ -10,11 +12,21 @@ /** * Functional Text Files Test in HCFS */ +@WorksWithFDW public class HcfsTextTest extends BaseFeature { private static final String[] PXF_SINGLE_COL = {"text_blob text"}; private static final String[] PXF_THREE_COLS = {"a text", "b text", "c text"}; + /* Since delimiter 'OFF' doesn't work for FDW, we can skip this test for FDW. + * + * CREATE FOREIGN TABLE hcfs_text_delimiter_off (text_blob text) SERVER default_hdfs + * OPTIONS (resource 'tmp/pxf_automation_data/b6afbc30-2a0e-40a8-9c4c-61a54be173e5/hcfs-text/no-delimiter', format 'text', delimiter 'OFF'); + * CREATE FOREIGN TABLE + * pxfautomation=# select * from hcfs_text_delimiter_off; + * ERROR: using no delimiter is only supported for external tables + */ + @SkipForFDW @Test(groups = {"gpdb", "hcfs", "security"}) public void testDelimiterOff() throws Exception { String hdfsPath = hdfs.getWorkingDirectory() + "/hcfs-text/no-delimiter"; @@ -31,6 +43,7 @@ public void testEscapeOff() throws Exception { runTestScenario("escape_off", PXF_THREE_COLS, srcPath, hdfsPath, null, "OFF"); } + @SkipForFDW @Test(groups = {"gpdb", "hcfs", "security"}) public void testDelimiterOffAndEscapeOff() throws Exception { String hdfsPath = hdfs.getWorkingDirectory() + "/hcfs-text/no-delimiter-or-escape"; @@ -44,7 +57,7 @@ private void runTestScenario(String name, String[] fields, String srcPath, Strin ProtocolEnum protocol = ProtocolUtils.getProtocol(); String tableName = "hcfs_text_" + name; - exTable = new ReadableExternalTable(tableName, fields, protocol.getExternalTablePath(hdfs.getBasePath(), hdfsPath), "TEXT"); + exTable = TableFactory.getPxfReadableTextTable(tableName, fields, protocol.getExternalTablePath(hdfs.getBasePath(), hdfsPath), delimiter); exTable.setProfile(protocol.value() + ":text"); if (delimiter != null) { diff --git a/automation/src/test/java/org/greenplum/pxf/automation/features/hdfs/HdfsReadableSequenceTest.java b/automation/src/test/java/org/greenplum/pxf/automation/features/hdfs/HdfsReadableSequenceTest.java index 801bcd6d34..c9883a7307 100755 --- a/automation/src/test/java/org/greenplum/pxf/automation/features/hdfs/HdfsReadableSequenceTest.java +++ b/automation/src/test/java/org/greenplum/pxf/automation/features/hdfs/HdfsReadableSequenceTest.java @@ -1,10 +1,11 @@ package org.greenplum.pxf.automation.features.hdfs; +import annotations.WorksWithFDW; import org.greenplum.pxf.automation.components.cluster.PhdCluster; import org.greenplum.pxf.automation.datapreparer.CustomSequencePreparer; import org.greenplum.pxf.automation.features.BaseFeature; import org.greenplum.pxf.automation.structures.tables.basic.Table; -import org.greenplum.pxf.automation.structures.tables.pxf.ReadableExternalTable; +import org.greenplum.pxf.automation.structures.tables.utils.TableFactory; import org.greenplum.pxf.automation.utils.fileformats.FileFormatsUtils; import org.greenplum.pxf.automation.utils.system.ProtocolEnum; import org.greenplum.pxf.automation.utils.system.ProtocolUtils; @@ -15,6 +16,7 @@ /** * Collection of Test cases for PXF ability to read SequenceFile. */ +@WorksWithFDW public class HdfsReadableSequenceTest extends BaseFeature { private String hdfsPath; @@ -107,14 +109,11 @@ public void readSequenceFile() throws Exception { ProtocolEnum protocol = ProtocolUtils.getProtocol(); // default external table with common settings - exTable = new ReadableExternalTable("writable_in_sequence", null, "", - "custom"); + exTable = TableFactory.getPxfReadableSequenceTable("writable_in_sequence", null, "", + schemaPackage + customSchemaFileName); exTable.setHost(pxfHost); exTable.setPort(pxfPort); - exTable.setProfile(protocol.value() + ":SequenceFile"); - exTable.setFormatter("pxfwritable_import"); exTable.setFields(customWritableFields); - exTable.setDataSchema(schemaPackage + customSchemaFileName); exTable.setPath(protocol.getExternalTablePath(hdfs.getBasePath(), hdfsPath + writableInsideSequenceFileName)); gpdb.createTableAndVerify(exTable); diff --git a/automation/src/test/java/org/greenplum/pxf/automation/features/jdbc/JdbcTest.java b/automation/src/test/java/org/greenplum/pxf/automation/features/jdbc/JdbcTest.java index fe71caec63..67046b264c 100755 --- a/automation/src/test/java/org/greenplum/pxf/automation/features/jdbc/JdbcTest.java +++ b/automation/src/test/java/org/greenplum/pxf/automation/features/jdbc/JdbcTest.java @@ -2,6 +2,8 @@ import java.io.File; +import annotations.FailsWithFDW; +import annotations.WorksWithFDW; import org.greenplum.pxf.automation.structures.tables.basic.Table; import org.greenplum.pxf.automation.structures.tables.pxf.ExternalTable; import org.greenplum.pxf.automation.structures.tables.utils.TableFactory; @@ -11,6 +13,7 @@ import org.greenplum.pxf.automation.features.BaseFeature; +@WorksWithFDW public class JdbcTest extends BaseFeature { private static final String POSTGRES_DRIVER_CLASS = "org.postgresql.Driver"; @@ -401,16 +404,22 @@ public void readViewSessionParams() throws Exception { runTincTest("pxf.features.jdbc.session_params.runTest"); } + @FailsWithFDW + // All the Writable Tests are failing with this Error: + // ERROR: PXF server error : class java.io.DataInputStream cannot be cast to class + // [B (java.io.DataInputStream and [B are in module java.base of loader 'bootstrap') @Test(groups = {"features", "gpdb", "security", "jdbc"}) public void jdbcWritableTable() throws Exception { runTincTest("pxf.features.jdbc.writable.runTest"); } + @FailsWithFDW @Test(groups = {"features", "gpdb", "security", "jdbc"}) public void jdbcWritableTableNoBatch() throws Exception { runTincTest("pxf.features.jdbc.writable_nobatch.runTest"); } + @FailsWithFDW @Test(groups = {"features", "gpdb", "security", "jdbc"}) public void jdbcWritableTablePool() throws Exception { runTincTest("pxf.features.jdbc.writable_pool.runTest"); diff --git a/automation/src/test/java/org/greenplum/pxf/automation/features/json/JsonTest.java b/automation/src/test/java/org/greenplum/pxf/automation/features/json/JsonReadTest.java similarity index 99% rename from automation/src/test/java/org/greenplum/pxf/automation/features/json/JsonTest.java rename to automation/src/test/java/org/greenplum/pxf/automation/features/json/JsonReadTest.java index e5596da57a..f1d6dc4539 100755 --- a/automation/src/test/java/org/greenplum/pxf/automation/features/json/JsonTest.java +++ b/automation/src/test/java/org/greenplum/pxf/automation/features/json/JsonReadTest.java @@ -10,7 +10,7 @@ /** * Tests for Json plugin to read HDFS files in JSON format. */ -public class JsonTest extends BaseFeature { +public class JsonReadTest extends BaseFeature { private String hdfsPath; private String resourcePath; diff --git a/automation/src/test/java/org/greenplum/pxf/automation/features/json/JsonWriteTest.java b/automation/src/test/java/org/greenplum/pxf/automation/features/json/JsonWriteTest.java new file mode 100644 index 0000000000..83d1bae605 --- /dev/null +++ b/automation/src/test/java/org/greenplum/pxf/automation/features/json/JsonWriteTest.java @@ -0,0 +1,239 @@ +package org.greenplum.pxf.automation.features.json; + +import org.greenplum.pxf.automation.features.BaseWritableFeature; +import org.greenplum.pxf.automation.structures.tables.basic.Table; +import org.greenplum.pxf.automation.structures.tables.utils.TableFactory; +import org.greenplum.pxf.automation.utils.system.ProtocolEnum; +import org.greenplum.pxf.automation.utils.system.ProtocolUtils; +import org.testng.annotations.Test; + +import java.io.File; +import java.util.StringJoiner; +import java.util.function.Consumer; + +import static java.lang.Thread.sleep; +import static org.testng.Assert.assertTrue; + +/** + * Test class for testing writing use cases of Json profiles. + */ +public class JsonWriteTest extends BaseWritableFeature { + + public static final String[] PRIMITIVE_TYPES_FIELDS = { + "id integer", // DataType.INTEGER + "name text", // DataType.TEXT + "sml smallint", // DataType.SMALLINT + "integ integer", // DataType.INTEGER + "bg bigint", // DataType.BIGINT + "r real", // DataType.REAL + "dp double precision", // DataType.FLOAT8 + "dec numeric", // DataType.NUMERIC + "bool boolean", // DataType.BOOLEAN + "cdate date", // DataType.DATE + "ctime time", // DataType.TIME + "tm timestamp without time zone", // DataType.TIMESTAMP + "tmz timestamp with time zone", // DataType.TIMESTAMP_WITH_TIME_ZONE + "c1 character(3)", // DataType.BPCHAR + "vc1 character varying(5)", // DataType.VARCHAR + "bin bytea" // DataType.BYTEA + }; + + public static final String[] ARRAY_TYPES_FIELDS = { + "id integer", // DataType.INTEGER + "name text[]", // DataType.TEXTARRAY + "sml smallint[]", // DataType.INT2ARRAY + "integ integer[]", // DataType.INT4ARRAY + "bg bigint[]", // DataType.INT8ARRAY + "r real[]", // DataType.FLOAT4ARRAY + "dp double precision[]", // DataType.FLOAT8ARRAY + "dec numeric[]", // DataType.NUMERICARRAY + "bool boolean[]", // DataType.BOOLARRAY + "cdate date[]", // DataType.DATEARRAY + "ctime time[]", // DataType.TIMEARRAY + "tm timestamp without time zone[]", // DataType.TIMESTAMPARRAY + "tmz timestamp with time zone[]", // DataType.TIMESTAMP_WITH_TIMEZONE_ARRAY + "c1 character(3)[]", // DataType.BPCHARARRAY + "vc1 character varying(5)[]", // DataType.VARCHARARRAY + "bin bytea[]" // DataType.BYTEAARRAY + }; + + public static final String[] ESCAPING_PRIMITIVE_TYPES_FIELDS = { + "id integer", // DataType.INTEGER + "\"col space\" text", // DataType.TEXT + "text_escape text", // DataType.TEXT + "char_escape character(14)", // DataType.BPCHAR + "varchar_escape character varying(14)" // DataType.VARCHAR + }; + + private static final String PRIMITIVE_TYPES = "primitive_types"; + private static final String ARRAY_TYPES = "array_types"; + + + private static final Consumer JSON_EXTENSION_ASSERTER = (filename) -> { + assertTrue(filename.endsWith(".json"), "file " + filename + " does not end with .json"); + }; + + private static final Consumer JSONL_EXTENSION_ASSERTER = (filename) -> { + assertTrue(filename.endsWith(".jsonl"), "file " + filename + " does not end with .jsonl"); + }; + + private static final Consumer JSON_GZ_EXTENSION_ASSERTER = (filename) -> { + assertTrue(filename.endsWith(".json.gz"), "file " + filename + " does not end with .json.gz"); + }; + + private static final Consumer JSONL_GZ_EXTENSION_ASSERTER = (filename) -> { + assertTrue(filename.endsWith(".jsonl.gz"), "file " + filename + " does not end with .jsonl.gz"); + }; + + private String resourcePath; + + // tables that store seed data for insertion into writable external tables + private Table gpdbPrimitiveTypesTable; + private Table gpdbArrayTypesTable; + + @Override + public void beforeClass() throws Exception { + // path for storing data on HDFS that will be auto-cleanup after each test method + hdfsWritePath = hdfs.getWorkingDirectory() + "/json/"; + resourcePath = localDataResourcesFolder + "/json/"; + + // seed the source data in GPDB internal table -- primitive types + gpdbPrimitiveTypesTable = new Table("gpdb_" + PRIMITIVE_TYPES, PRIMITIVE_TYPES_FIELDS); + gpdbPrimitiveTypesTable.setDistributionFields(new String[]{"id"}); + gpdb.createTableAndVerify(gpdbPrimitiveTypesTable); + gpdb.copyFromFile(gpdbPrimitiveTypesTable, new File(resourcePath + PRIMITIVE_TYPES + ".csv"), ",", true); + + // seed the source data in GPDB internal table -- array types + gpdbArrayTypesTable = new Table("gpdb_" + ARRAY_TYPES, ARRAY_TYPES_FIELDS); + gpdbArrayTypesTable.setDistributionFields(new String[]{"id"}); + gpdb.createTableAndVerify(gpdbArrayTypesTable); + gpdb.copyFromFile(gpdbArrayTypesTable, new File(resourcePath + ARRAY_TYPES + ".csv"), ",", true); + } + + @Test(groups = {"gpdb", "security", "hcfs"}) + public void writePrimitiveTypesRows() throws Exception { + runScenario(PRIMITIVE_TYPES + "_rows", PRIMITIVE_TYPES_FIELDS, gpdbPrimitiveTypesTable, + null, null, JSONL_EXTENSION_ASSERTER); + } + + @Test(groups = {"gpdb", "security", "hcfs"}) + public void writePrimitiveTypesObject() throws Exception { + runScenario(PRIMITIVE_TYPES + "_object", PRIMITIVE_TYPES_FIELDS, gpdbPrimitiveTypesTable, + new String[]{"ROOT=records"}, new String[]{"IDENTIFIER=id"}, JSON_EXTENSION_ASSERTER); + } + + @Test(groups = {"gpdb", "security", "hcfs"}) + public void writePrimitiveTypesRowsCompressed() throws Exception { + runScenario(PRIMITIVE_TYPES + "_rows_compressed", PRIMITIVE_TYPES_FIELDS, gpdbPrimitiveTypesTable, + new String[]{"COMPRESSION_CODEC=gzip"}, new String[]{"IDENTIFIER=id"}, JSONL_GZ_EXTENSION_ASSERTER); + } + + @Test(groups = {"gpdb", "security", "hcfs"}) + public void writePrimitiveTypesObjectCompressed() throws Exception { + runScenario(PRIMITIVE_TYPES + "_object_compressed", PRIMITIVE_TYPES_FIELDS, gpdbPrimitiveTypesTable, + new String[]{"ROOT=records","COMPRESSION_CODEC=gzip"}, new String[]{"IDENTIFIER=id"}, JSON_GZ_EXTENSION_ASSERTER); + } + + @Test(groups = {"gpdb", "security", "hcfs"}) + public void writePrimitiveTypesEscaping() throws Exception { + String escapingData = new StringJoiner(",") + .add("(1, 'col space 1', 'text', 'char', 'varchar')") + .add("(2, 'col space 2', 's\"b\\{},:[]', 's\"b\\{},:[]', 's\"b\\{},:[]')") // single sequence + .add("(3, 'col space 3', 'd\"\"b\\\\{},:[]', 'd\"\"b\\\\{},:[]', 'd\"\"b\\\\{},:[]')") // double sequence + .add("(4, 'col space 4', 't\"\"\"b\\\\\\{},:[]', 't\"\"\"b\\\\\\{},:[]', 't\"\"\"b\\\\\\{},:[]')") // triple sequence + .toString(); + runScenario(PRIMITIVE_TYPES + "_escaping", ESCAPING_PRIMITIVE_TYPES_FIELDS, escapingData, + null, null, JSONL_EXTENSION_ASSERTER); + } + + @Test(groups = {"gpdb", "security", "hcfs"}) + public void writeArrayTypesRows() throws Exception { + runScenario(ARRAY_TYPES + "_rows", ARRAY_TYPES_FIELDS, gpdbArrayTypesTable, + null, null, JSONL_EXTENSION_ASSERTER); + } + + @Test(groups = {"gpdb", "security", "hcfs"}) + public void writeArrayTypesObject() throws Exception { + runScenario(ARRAY_TYPES + "_object", ARRAY_TYPES_FIELDS, gpdbArrayTypesTable, + new String[]{"ROOT=records"}, new String[]{"IDENTIFIER=id"}, JSON_EXTENSION_ASSERTER); + } + + @Test(groups = {"gpdb", "security", "hcfs"}) + public void errorInvalidEncoding() throws Exception { + // 1. prepare writable external table ready to receive data for writing from internal table + writableExTable = TableFactory.getPxfHcfsWritableTable( + "pxf_invalid_encoding_json_write", PRIMITIVE_TYPES_FIELDS, hdfsWritePath + "invalid_encoding", hdfs.getBasePath(), "json"); + writableExTable.setEncoding("LATIN1"); // set a non-UTF8 encoding for the table + createTable(writableExTable); + + // 2. run the Tinc test that inserts data as CTAS, which should fail and verifies the error message + // for external table with pxfwritable_export formatter (default setup for this test) the actual error + // will be produced by the formatter, not PXF, but for FDW or CSV wire format PXF error should show + runTincTest("pxf.features.hdfs.writable.json.invalid_encoding.runTest"); + } + + @Test(groups = {"gpdb", "security", "hcfs"}) + public void errorEmptyRoot() throws Exception { + // 1. prepare writable external table ready to receive data for writing from internal table + writableExTable = TableFactory.getPxfHcfsWritableTable( + "pxf_empty_root_json_write", PRIMITIVE_TYPES_FIELDS, hdfsWritePath + "empty_root", hdfs.getBasePath(), "json"); + writableExTable.setUserParameters(new String[]{"ROOT= "}); + createTable(writableExTable); + + // 2. run the Tinc test that inserts data as CTAS, which should fail and verifies the error message + runTincTest("pxf.features.hdfs.writable.json.empty_root.runTest"); + } + + /** + * Runs a test scenario with a given name. The test creates a writable external table, inserts data into it, then + * creates a readable external table and calls a Tinc test to read the data and compare with the expected output. + * + * @param scenario name of the scenario, gets factored into the names of tables + * @param fields schema for the external tables + * @param source a string with data or a Table object for existing Greenplum table that serves as the source of data + * @param writeOptions options to add to the writable table, null if no additional options are needed + * @param readOptions options to add to the readable table, null if no additional options are needed + * @param filenameAsserter a function that asserts names of produced files + * @throws Exception if any operation fails + */ + private void runScenario(String scenario, String[] fields, Object source, String[] writeOptions, String[] readOptions, Consumer filenameAsserter) throws Exception { + + // 1. prepare writable external table ready to receive data for writing from internal table + writableExTable = TableFactory.getPxfHcfsWritableTable( + "pxf_" + scenario + "_json_write", fields, hdfsWritePath + scenario, hdfs.getBasePath(), "json"); + if (writeOptions != null) { + writableExTable.setUserParameters(writeOptions); + } + createTable(writableExTable); + + // 2. insert data into the writable table + if (source instanceof String) { + // as INSERT INTO .. VALUES (..) SQL query + gpdb.insertData((String) source, writableExTable); + } else if (source instanceof Table) { + // as CTAS from the corresponding internal table + gpdb.copyData((Table) source, writableExTable); + } else { + throw new UnsupportedOperationException("Invalid type of source parameter"); + } + + // 3. wait until the data is written (potentially asynchronously) to the backend system, assert filenames + if (ProtocolUtils.getProtocol() != ProtocolEnum.HDFS) { + // for HCFS on Cloud, wait a bit for async write in previous steps to finish + sleep(10000); + } + hdfs.list(hdfsWritePath + scenario).forEach(filenameAsserter); + + // 4. prepare readable external table ready to read data written by the writable external table + readableExTable = TableFactory.getPxfHcfsReadableTable( + "pxf_" + scenario + "_json_read", fields, hdfsWritePath + scenario, hdfs.getBasePath(), "json"); + if (readOptions != null) { + readableExTable.setUserParameters(readOptions); + } + createTable(readableExTable); + + // 5. run the Tinc test that queries the data back using the corresponding readable external table + runTincTest("pxf.features.hdfs.writable.json." + scenario + ".runTest"); + } + +} diff --git a/automation/src/test/java/org/greenplum/pxf/automation/features/multibytedelimiter/MultibyteDelimiterTest.java b/automation/src/test/java/org/greenplum/pxf/automation/features/multibytedelimiter/MultibyteDelimiterTest.java new file mode 100644 index 0000000000..6ac0ce539d --- /dev/null +++ b/automation/src/test/java/org/greenplum/pxf/automation/features/multibytedelimiter/MultibyteDelimiterTest.java @@ -0,0 +1,655 @@ +package org.greenplum.pxf.automation.features.multibytedelimiter; + +import au.com.bytecode.opencsv.CSVWriter; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.io.compress.BZip2Codec; +import org.greenplum.pxf.automation.datapreparer.CustomTextPreparer; +import org.greenplum.pxf.automation.features.BaseFeature; +import org.greenplum.pxf.automation.structures.tables.basic.Table; +import org.greenplum.pxf.automation.structures.tables.utils.TableFactory; +import org.greenplum.pxf.automation.utils.csv.CsvUtils; +import org.greenplum.pxf.automation.utils.exception.ExceptionUtils; +import org.greenplum.pxf.automation.utils.fileformats.FileFormatsUtils; +import org.greenplum.pxf.automation.utils.system.ProtocolEnum; +import org.greenplum.pxf.automation.utils.system.ProtocolUtils; +import org.junit.Assert; +import org.postgresql.util.PSQLException; +import org.testng.annotations.Test; + +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +import static java.lang.Thread.sleep; + +/** + * Collection of Test cases for PXF ability to read Text/CSV files with pxfdelimited_import. + */ +public class MultibyteDelimiterTest extends BaseFeature { + ProtocolEnum protocol; + + // holds data for file generation + Table dataTable = null; + + // holds data for encoded file generation + Table encodedDataTable = null; + + // path for storing data on HDFS + String hdfsFilePath = ""; + + private static final String[] ROW_WITH_ESCAPE = {"s_101", + "s_1001", + "s_10001", + "2299-11-28 05:46:40", + "101", + "1001", + "10001", + "10001", + "10001", + "10001", + "10001", + "s_101 | escaped!", + "s_1001", + "s_10001", + "2299-11-28 05:46:40", + "101", + "1001", + "10001", + "10001", + "10001", + "10001", + "10001"}; + + private class CsvSpec { + String delimiter; + char quote; + char escape; + String eol; + Charset encoding; + public CsvSpec(String delimiter, char quote, char escape, String eol) { + this.delimiter = delimiter; + this.quote = quote; + this.escape = escape; + this.eol = eol; + this.encoding = StandardCharsets.UTF_8; + } + + public CsvSpec(String delimiter, char quote, char escape) { + this(delimiter, quote, escape, CSVWriter.DEFAULT_LINE_END); + } + + public CsvSpec(String delimiter) { + this(delimiter, CSVWriter.NO_QUOTE_CHARACTER, CSVWriter.NO_ESCAPE_CHARACTER, CSVWriter.DEFAULT_LINE_END); + } + + public void setDelimiter(String delimiter) { + this.delimiter = delimiter; + } + + public void setQuote(char quote) { + this.quote = quote; + } + + public void setEscape(char escape) { + this.escape = escape; + } + + public void setEol(String eol) { + this.eol = eol; + } + + public void setEncoding(Charset encoding) { + this.encoding = encoding; + } + + /** + * This function takes the CsvSpec used for writing the file + * and clones it to be used as table formatter options. + * + * In the case of EOL handling, we do not want to include the + * EOL value as a formatter option if it is the default (\n). + * However, for '\r' the corresponding value is 'CR' and for '\r\n' + * the corresponding value is 'CRLF'. Anything else, we return as is. + * + * @return A clone of the CsvSpec to be used as formatter options for the table DDL + */ + public CsvSpec cloneForFormatting() { + // handle EOL situation + String eol = this.eol; + switch (eol) { + case "\r": + eol = "CR"; + break; + case "\r\n": + eol = "CRLF"; + break; + case CSVWriter.DEFAULT_LINE_END: + // for the default case, we do not want to set the eol value in the formatter options + eol = null; + break; + default: + eol = this.eol; + } + + CsvSpec clone = new CsvSpec(this.delimiter, this.quote, this.escape, eol); + // we do not care about the encoding value as a formatter option in the table DDL + clone.setEncoding(null); + + return clone; + } + } + + /** + * Prepare all components and all data flow (Hdfs to GPDB) + */ + @Override + public void beforeClass() throws Exception { + protocol = ProtocolUtils.getProtocol(); + } + + /** + * Before every method determine default hdfs data Path, default data, and + * default external table structure. Each case change it according to it + * needs. + */ + @Override + protected void beforeMethod() throws Exception { + super.beforeMethod(); + // path for storing data on HDFS + hdfsFilePath = hdfs.getWorkingDirectory() + "/multibyteDelimiter"; + // prepare data in table + dataTable = new Table("dataTable", null); + FileFormatsUtils.prepareData(new CustomTextPreparer(), 100, dataTable); + // default definition of external table + exTable = TableFactory.getPxfReadableTextTable("pxf_multibyte_small_data", + new String[]{ + "s1 text", + "s2 text", + "s3 text", + "d1 timestamp", + "n1 int", + "n2 int", + "n3 int", + "n4 int", + "n5 int", + "n6 int", + "n7 int", + "s11 text", + "s12 text", + "s13 text", + "d11 timestamp", + "n11 int", + "n12 int", + "n13 int", + "n14 int", + "n15 int", + "n16 int", + "n17 int"}, + protocol.getExternalTablePath(hdfs.getBasePath(), hdfsFilePath), + null); + exTable.setFormat("CUSTOM"); + exTable.setFormatter("pxfdelimited_import"); + exTable.setProfile(protocol.value() + ":csv"); + + encodedDataTable = new Table("data", null); + encodedDataTable.addRow(new String[]{"4", "tá sé seo le tástáil dea-"}); + encodedDataTable.addRow(new String[]{"3", "règles d'automation"}); + encodedDataTable.addRow(new String[]{"5", "minden amire szüksége van a szeretet"}); + } + + @Test(groups = {"gpdb", "hcfs", "security"}) + public void readTwoByteDelimiter() throws Exception { + // used for creating the CSV file + CsvSpec fileSpec = new CsvSpec("¤"); + + runScenario("pxf_multibyte_twobyte_data", dataTable, fileSpec); + + // create a new table with the SKIP_HEADER_COUNT parameter + exTable.setName("pxf_multibyte_twobyte_data_with_skip"); + exTable.setUserParameters(new String[]{"SKIP_HEADER_COUNT=10"}); + // create external table + gpdb.createTableAndVerify(exTable); + + // verify results + runTincTest("pxf.features.multibyte_delimiter.two_byte.runTest"); + } + + @Test(groups = {"gpdb", "hcfs", "security"}) + public void readThreeByteDelimiter() throws Exception { + CsvSpec fileSpec = new CsvSpec("停"); + + runScenario("pxf_multibyte_threebyte_data", dataTable, fileSpec); + + // create a new table with the SKIP_HEADER_COUNT parameter + exTable.setName("pxf_multibyte_threebyte_data_with_skip"); + exTable.setUserParameters(new String[]{"SKIP_HEADER_COUNT=10"}); + // create external table + gpdb.createTableAndVerify(exTable); + + // verify results + runTincTest("pxf.features.multibyte_delimiter.three_byte.runTest"); + } + + @Test(groups = {"gpdb", "hcfs", "security"}) + public void readFourByteDelimiter() throws Exception { + CsvSpec fileSpec = new CsvSpec("\uD83D\uDE42"); + + runScenario("pxf_multibyte_fourbyte_data", dataTable, fileSpec); + + // create a new table with the SKIP_HEADER_COUNT parameter + exTable.setName("pxf_multibyte_fourbyte_data_with_skip"); + exTable.setUserParameters(new String[]{"SKIP_HEADER_COUNT=10"}); + // create external table + gpdb.createTableAndVerify(exTable); + + // verify results + runTincTest("pxf.features.multibyte_delimiter.four_byte.runTest"); + } + + @Test(groups = {"gpdb", "hcfs", "security"}) + public void readMultiCharStringDelimiter() throws Exception { + CsvSpec fileSpec = new CsvSpec("DELIM"); + + runScenario("pxf_multibyte_multichar_data", dataTable, fileSpec); + + // create a new table with the SKIP_HEADER_COUNT parameter + exTable.setName("pxf_multibyte_multichar_data_with_skip"); + exTable.setUserParameters(new String[]{"SKIP_HEADER_COUNT=10"}); + // create external table + gpdb.createTableAndVerify(exTable); + + // verify results + runTincTest("pxf.features.multibyte_delimiter.multi_char.runTest"); + } + + @Test(groups = {"gpdb", "hcfs", "security"}) + public void readTwoByteDelimiterWithCRLF() throws Exception { + CsvSpec fileSpec = new CsvSpec("¤", CSVWriter.NO_QUOTE_CHARACTER, CSVWriter.NO_ESCAPE_CHARACTER, "\r\n"); + + runScenario("pxf_multibyte_twobyte_withcrlf_data", dataTable, fileSpec); + + // verify results + runTincTest("pxf.features.multibyte_delimiter.two_byte_with_crlf.runTest"); + } + + @Test(groups = {"gpdb", "hcfs", "security"}) + public void readTwoByteDelimiterWithCR() throws Exception { + CsvSpec fileSpec = new CsvSpec("¤", CSVWriter.NO_QUOTE_CHARACTER, CSVWriter.NO_ESCAPE_CHARACTER, "\r"); + + // we need to add the eol value to the URL to be able to parse the data on PXF Java side + exTable.setUserParameters(new String[] {"NEWLINE=CR"}); + + runScenario("pxf_multibyte_twobyte_withcr_data", dataTable, fileSpec); + + // verify results + runTincTest("pxf.features.multibyte_delimiter.two_byte_with_cr.runTest"); + } + + @Test(groups = {"gpdb", "hcfs", "security"}) + public void readTwoByteDelimiterWrongFormatter() throws Exception { + CsvSpec fileSpec = new CsvSpec("¤"); + + exTable.setFormatter("pxfwritable_import"); + + runScenario("pxf_multibyte_twobyte_wrongformatter_data", dataTable, fileSpec); + + // verify results + runTincTest("pxf.features.multibyte_delimiter.two_byte_wrong_formatter.runTest"); + } + + @Test(groups = {"gpdb", "hcfs", "security"}) + public void readTwoByteDelimiterDelimNotProvided() throws Exception { + CsvSpec fileSpec = new CsvSpec("¤"); + CsvSpec tableSpec = fileSpec.cloneForFormatting(); + // remove the delimiter for the formatterOptions + tableSpec.setDelimiter(null); + + runScenario("pxf_multibyte_twobyte_nodelim_data", tableSpec, dataTable, fileSpec); + + // verify results + runTincTest("pxf.features.multibyte_delimiter.two_byte_no_delim.runTest"); + } + + @Test(groups = {"gpdb", "hcfs", "security"}) + public void readTwoByteDelimiterWithWrongDelimiter() throws Exception { + CsvSpec fileSpec = new CsvSpec("¤"); + CsvSpec tableSpec = fileSpec.cloneForFormatting(); + // set the wrong delimiter for the formatterOptions + tableSpec.setDelimiter("停"); + + runScenario("pxf_multibyte_twobyte_wrong_delim_data", tableSpec, dataTable, fileSpec); + + // verify results + runTincTest("pxf.features.multibyte_delimiter.two_byte_with_wrong_delim.runTest"); + } + + @Test(groups = {"gpdb", "hcfs", "security"}) + public void readTwoByteDelimiterWithQuote() throws Exception { + CsvSpec fileSpec = new CsvSpec("¤", CSVWriter.DEFAULT_QUOTE_CHARACTER, CSVWriter.NO_ESCAPE_CHARACTER); + + runScenario("pxf_multibyte_twobyte_withquote_data", dataTable, fileSpec); + + // verify results + runTincTest("pxf.features.multibyte_delimiter.two_byte_with_quote.runTest"); + } + + @Test(groups = {"gpdb", "hcfs", "security"}) + public void readTwoByteDelimiterWithWrongEol() throws Exception { + CsvSpec fileSpec = new CsvSpec("¤", CSVWriter.DEFAULT_QUOTE_CHARACTER, CSVWriter.DEFAULT_ESCAPE_CHARACTER); + CsvSpec tableSpec = fileSpec.cloneForFormatting(); + // set the wrong eol for the formatterOptions + tableSpec.setEol("CR"); + + runScenario("pxf_multibyte_twobyte_wrong_eol_data", tableSpec, dataTable, fileSpec); + + // verify results + // in newer versions of GP6 and in GP7, GPDB calls into the formatter one more time to handle EOF properly + // however, this is not the case for GP5 and for versions of GP6 older than 6.24.0 + // therefore, we must run 2 different sets of tests to check for the expected error + if (gpdb.getVersion() >= 6) { + runTincTest("pxf.features.multibyte_delimiter.two_byte_with_wrong_eol.runTest"); + } else { + runTincTest("pxf.features.multibyte_delimiter.two_byte_with_wrong_eol_5X.runTest"); + } + } + + @Test(groups = {"gpdb", "hcfs", "security"}) + public void readTwoByteDelimiterWithWrongQuote() throws Exception { + CsvSpec fileSpec = new CsvSpec("¤", CSVWriter.DEFAULT_QUOTE_CHARACTER, CSVWriter.DEFAULT_ESCAPE_CHARACTER); + CsvSpec tableSpec = fileSpec.cloneForFormatting(); + // set the wrong quote for the formatterOptions + tableSpec.setQuote('|'); + + runScenario("pxf_multibyte_twobyte_wrong_quote_data", tableSpec, dataTable, fileSpec); + + // verify results + // in newer versions of GP6 and in GP7, GPDB calls into the formatter one more time to handle EOF properly + // however, this is not the case for GP5 and for versions of GP6 older than 6.24.0 + // therefore, we must run 2 different sets of tests to check for the expected error + if (gpdb.getVersion() >= 6) { + runTincTest("pxf.features.multibyte_delimiter.two_byte_with_wrong_quote.runTest"); + } else { + runTincTest("pxf.features.multibyte_delimiter.two_byte_with_wrong_quote_5X.runTest"); + } + } + + @Test(groups = {"gpdb", "hcfs", "security"}) + public void readTwoByteDelimiterWithQuoteAndEscape() throws Exception { + CsvSpec fileSpec = new CsvSpec("¤", '|', '\\'); + + dataTable.addRow(ROW_WITH_ESCAPE); + + runScenario("pxf_multibyte_twobyte_withquote_withescape_data", dataTable, fileSpec); + + // verify results + runTincTest("pxf.features.multibyte_delimiter.two_byte_with_quote_and_escape.runTest"); + } + + @Test(groups = {"gpdb", "hcfs", "security"}) + public void readTwoByteDelimiterWithWrongEscape() throws Exception { + CsvSpec fileSpec = new CsvSpec("¤", '|', '\\'); + CsvSpec tableSpec = fileSpec.cloneForFormatting(); + tableSpec.setEscape('#'); + + dataTable.addRow(ROW_WITH_ESCAPE); + + runScenario("pxf_multibyte_twobyte_wrong_escape_data", tableSpec, dataTable, fileSpec); + + // verify results + runTincTest("pxf.features.multibyte_delimiter.two_byte_with_wrong_escape.runTest"); + } + + // users should still be able to use a normal delimiter with this formatter + @Test(groups = {"gpdb", "hcfs", "security"}) + public void readOneByteDelimiter() throws Exception { + CsvSpec fileSpec = new CsvSpec("|"); + + runScenario("pxf_multibyte_onebyte_data", dataTable, fileSpec); + + // verify results + runTincTest("pxf.features.multibyte_delimiter.one_byte.runTest"); + } + + + @Test(groups = {"gpdb", "hcfs", "security"}) + public void readOneCol() throws Exception { + CsvSpec fileSpec = new CsvSpec("¤"); + + exTable.setFields(new String[]{"s1 text"}); + + dataTable = new Table("data", null); + dataTable.addRow(new String[]{"tá sé seo le tástáil dea-"}); + dataTable.addRow(new String[]{"règles d'automation"}); + dataTable.addRow(new String[]{"minden amire szüksége van a szeretet"}); + + runScenario("pxf_multibyte_onecol_data", dataTable, fileSpec); + + // verify results + runTincTest("pxf.features.multibyte_delimiter.one_col.runTest"); + } + + @Test(groups = {"gpdb", "hcfs", "security"}) + public void readOneColQuote() throws Exception { + CsvSpec fileSpec = new CsvSpec("¤", '|', CSVWriter.NO_ESCAPE_CHARACTER); + + exTable.setFields(new String[]{"s1 text"}); + + dataTable = new Table("data", null); + dataTable.addRow(new String[]{"tá sé seo le tástáil dea-"}); + dataTable.addRow(new String[]{"règles d'automation"}); + dataTable.addRow(new String[]{"minden amire szüksége van a szeretet"}); + + runScenario("pxf_multibyte_onecol_quote_data", dataTable, fileSpec); + + // verify results + runTincTest("pxf.features.multibyte_delimiter.one_col_quote.runTest"); + } + + @Test(groups = {"gpdb", "hcfs", "security"}) + public void readBzip2CompressedCsv() throws Exception { + BZip2Codec codec = new BZip2Codec(); + codec.setConf(hdfs.getConfiguration()); + char c = 'a'; + + for (int i = 0; i < 10; i++, c++) { + Table dataTable = getSmallData(StringUtils.repeat(String.valueOf(c), 2), 10); + hdfs.writeTableToFile(hdfs.getWorkingDirectory() + "/bzip2/" + c + "_" + fileName + ".bz2", + dataTable, "¤", StandardCharsets.UTF_8, codec); + } + + createCsvExternalTable("pxf_multibyte_twobyte_withbzip2_data", + new String[] { + "name text", + "num integer", + "dub double precision", + "longNum bigint", + "bool boolean" + }, + protocol.getExternalTablePath(hdfs.getBasePath(), hdfs.getWorkingDirectory())+ "/bzip2/", + new String[] {"delimiter='¤'"}); + + runTincTest("pxf.features.multibyte_delimiter.two_byte_with_bzip2.runTest"); + } + + @Test(groups = {"gpdb", "hcfs", "security"}) + public void readTwoByteWithQuoteEscapeNewLine() throws Exception { + CsvSpec fileSpec = new CsvSpec("¤", '|', '\\', "EOL"); + + dataTable.addRow(ROW_WITH_ESCAPE); + + runScenario("pxf_multibyte_quote_escape_newline_data", dataTable, fileSpec); + + // verify results + runTincTest("pxf.features.multibyte_delimiter.quote_escape_newline.runTest"); + } + + @Test(groups = {"gpdb", "hcfs", "security"}) + public void invalidCodePoint() throws Exception { + exTable.setName("pxf_multibyte_invalid_codepoint_data"); + exTable.setFormatterOptions(new String[] {"delimiter=E'\\xA4'"}); + + // create external table + try { + gpdb.createTableAndVerify(exTable); + Assert.fail("Insert data should fail because of unsupported type"); + } catch (PSQLException e) { + ExceptionUtils.validate(null, e, new PSQLException("ERROR.*invalid byte sequence for encoding.*?", null), true); + } + } + + @Test(groups = {"gpdb", "hcfs", "security"}) + public void readFileWithLatin1EncodingTextProfile() throws Exception { + CsvSpec fileSpec = new CsvSpec("¤"); + // set the encoding value since the default value in CsvSpec is UTF-8 + fileSpec.setEncoding(StandardCharsets.ISO_8859_1); + + exTable.setFields(new String[]{"num1 int", "word text"}); + exTable.setEncoding("LATIN1"); + + runScenario("pxf_multibyte_encoding", encodedDataTable, fileSpec); + + // verify results + runTincTest("pxf.features.multibyte_delimiter.encoding.runTest"); + } + + @Test(groups = {"gpdb", "hcfs", "security"}) + public void readFileWithLatin1EncodingByteRepresentationTextProfile() throws Exception { + CsvSpec fileSpec = new CsvSpec("¤"); + // set the encoding value since the default value in CsvSpec is UTF-8 + fileSpec.setEncoding(StandardCharsets.ISO_8859_1); + CsvSpec tableSpec = fileSpec.cloneForFormatting(); + // use byte encoding instead + tableSpec.setDelimiter("\\xC2\\xA4"); + + exTable.setFields(new String[]{"num1 int", "word text"}); + exTable.setEncoding("LATIN1"); + exTable.setProfile(protocol.value() + ":text"); + + runScenario("pxf_multibyte_encoding_bytes", tableSpec, encodedDataTable, fileSpec); + + // verify results + runTincTest("pxf.features.multibyte_delimiter.encoding_bytes.runTest"); + } + + @Test(groups = {"gpdb", "hcfs", "security"}) + public void readFileWithLatin1EncodingWithQuoteTextProfile() throws Exception { + CsvSpec fileSpec = new CsvSpec("¤", '|', '|'); + // set the encoding value since the default value in CsvSpec is UTF-8 + fileSpec.setEncoding(StandardCharsets.ISO_8859_1); + + exTable.setFields(new String[]{"num1 int", "word text"}); + exTable.setEncoding("LATIN1"); + exTable.setProfile(protocol.value() + ":text"); + + runScenario("pxf_multibyte_encoding_quote", encodedDataTable, fileSpec); + + // verify results + runTincTest("pxf.features.multibyte_delimiter.encoding_quote.runTest"); + } + + @Test(groups = {"gpdb", "hcfs", "security"}) + public void readFileWithLatin1EncodingWithQuoteAndEscapeTextProfile() throws Exception { + CsvSpec fileSpec = new CsvSpec("¤", '|', '\"'); + // set the encoding value since the default value in CsvSpec is UTF-8 + fileSpec.setEncoding(StandardCharsets.ISO_8859_1); + + exTable.setFields(new String[]{"num1 int", "word text"}); + exTable.setEncoding("LATIN1"); + exTable.setProfile(protocol.value() + ":text"); + + runScenario("pxf_multibyte_encoding_quote_escape", encodedDataTable, fileSpec); + + // verify results + runTincTest("pxf.features.multibyte_delimiter.encoding_quote_escape.runTest"); + } + + @Test(groups = {"gpdb", "hcfs", "security"}) + public void wrongProfileWithFormatter() throws Exception { + exTable.setName("pxf_multibyte_wrong_profile"); + exTable.setFormatterOptions(new String[] {"delimiter='¤'", "quote='|'", "escape='\"'"}); + exTable.setProfile(protocol.value() + ":avro"); + exTable.setFields(new String[]{"name text", "age int"}); + + // prepare data and write to HDFS + gpdb.createTableAndVerify(exTable); + // location of schema and data files + String absolutePath = getClass().getClassLoader().getResource("data").getPath(); + String resourcePath = absolutePath + "/avro/"; + hdfs.writeAvroFileFromJson(hdfsFilePath + "simple.avro", + "file://" + resourcePath + "simple.avsc", + "file://" + resourcePath + "simple.json", null); + + // verify results + runTincTest("pxf.features.multibyte_delimiter.wrong_profile.runTest"); + } + + @Test(groups = {"gpdb", "hcfs", "security"}) + public void noProfileWithFormatter() throws Exception { + exTable.setName("pxf_multibyte_no_profile"); + exTable.setFormatterOptions(new String[] {"delimiter='¤'", "quote='|'", "escape='\"'"}); + exTable.setProfile(null); + exTable.setFragmenter("default-fragmenter"); + exTable.setAccessor("default-accessor"); + exTable.setResolver("default-resolver"); + exTable.setFields(new String[]{"name text", "age int"}); + + gpdb.createTableAndVerify(exTable); + + // verify results + runTincTest("pxf.features.multibyte_delimiter.no_profile.runTest"); + } + + private void createCsvExternalTable(String name, String[] cols, String path, String[] formatterOptions) throws Exception { + exTable = TableFactory.getPxfReadableTextTable(name, cols, path, null); + exTable.setFormat("CUSTOM"); + exTable.setFormatter("pxfdelimited_import"); + exTable.setProfile(protocol.value() + ":csv"); + exTable.setFormatterOptions(formatterOptions); + + gpdb.createTableAndVerify(exTable); + } + + private void writeCsvFileToHdfs(Table dataTable, CsvSpec spec) throws Exception { + // create local CSV + String tempLocalDataPath = dataTempFolder + "/data.csv"; + if (spec.delimiter.length() > 1) { + CsvUtils.writeTableToCsvFile(dataTable, tempLocalDataPath, spec.encoding, + '|', spec.quote, spec.escape, spec.eol); + CsvUtils.updateDelim(tempLocalDataPath, '|', spec.delimiter); + } else { + CsvUtils.writeTableToCsvFile(dataTable, tempLocalDataPath, spec.encoding, + spec.delimiter.charAt(0), spec.quote, spec.escape, spec.eol); + } + + // copy local CSV to HDFS + hdfs.copyFromLocal(tempLocalDataPath, hdfsFilePath); + sleep(2500); + } + + private void runScenario(String tableName, CsvSpec tableSpec, Table dataTable, CsvSpec fileSpec) throws Exception { + exTable.setName(tableName); + // set the formatter options using the table spec + if (tableSpec.delimiter != null) { + exTable.addFormatterOption("delimiter=E'" + tableSpec.delimiter + "'"); + } + if (tableSpec.quote != CSVWriter.NO_QUOTE_CHARACTER) { + exTable.addFormatterOption("quote='" + tableSpec.quote + "'"); + } + if (tableSpec.escape != CSVWriter.NO_ESCAPE_CHARACTER) { + exTable.addFormatterOption("escape='" + tableSpec.escape + "'"); + } + if (tableSpec.eol != null) { + exTable.addFormatterOption("newline='" + tableSpec.eol + "'"); + } + + // create external table + gpdb.createTableAndVerify(exTable); + + // create CSV file in hdfs using the provided data table and file spec + writeCsvFileToHdfs(dataTable, fileSpec); + } + + private void runScenario(String tableName, Table dataTable, CsvSpec fileSpec) throws Exception { + runScenario(tableName, fileSpec.cloneForFormatting(), dataTable, fileSpec); + + } +} diff --git a/automation/src/test/java/org/greenplum/pxf/automation/features/orc/OrcWriteTest.java b/automation/src/test/java/org/greenplum/pxf/automation/features/orc/OrcWriteTest.java index c47c0fd233..3a467ac98c 100644 --- a/automation/src/test/java/org/greenplum/pxf/automation/features/orc/OrcWriteTest.java +++ b/automation/src/test/java/org/greenplum/pxf/automation/features/orc/OrcWriteTest.java @@ -7,8 +7,6 @@ import org.greenplum.pxf.automation.structures.tables.hive.HiveExternalTable; import org.greenplum.pxf.automation.structures.tables.hive.HiveTable; import org.greenplum.pxf.automation.structures.tables.pxf.ExternalTable; -import org.greenplum.pxf.automation.structures.tables.pxf.ReadableExternalTable; -import org.greenplum.pxf.automation.structures.tables.pxf.WritableExternalTable; import org.greenplum.pxf.automation.structures.tables.utils.TableFactory; import org.greenplum.pxf.automation.utils.system.ProtocolEnum; import org.greenplum.pxf.automation.utils.system.ProtocolUtils; @@ -47,6 +45,11 @@ public class OrcWriteTest extends BaseFeature { "c_timestamptz timestamptz", // DataType.TIMESTAMP_WITH_TIME_ZONE }; + private static final String[] ORC_DECIMAL_WITH_LARGE_PRECISION_TABLE_COLUMNS = { + "id integer" , + "c_numeric numeric(90)" // DataType.NUMERIC + }; + private static final String[] ORC_PRIMITIVE_TABLE_COLUMNS_HIVE = { "id INT" , "c_bool BOOLEAN" , // DataType.BOOLEAN @@ -307,6 +310,47 @@ public void orcWritePrimitiveArraysMultidimensional() throws Exception { runTincTest("pxf.features.orc.write.primitive_types_array_multi.runTest"); } + @Test(groups = {"features", "gpdb", "security", "hcfs"}) + public void orcWriteDecimalWithLargePrecisionDefined() throws Exception { + gpdbTableNamePrefix = "orc_decimals_with_large_precision"; + fullTestPath = hdfsPath + gpdbTableNamePrefix; + + prepareWritableExternalTable(gpdbTableNamePrefix, ORC_DECIMAL_WITH_LARGE_PRECISION_TABLE_COLUMNS, fullTestPath); + + runTincTest("pxf.features.orc.write.decimal_with_large_precision_defined.runTest"); + } + + @Test(groups = {"features", "gpdb", "security", "hcfs"}) + public void orcWriteDecimalWithLargePrecisionNotDefined() throws Exception { + gpdbTableNamePrefix = "orc_decimals_with_large_precision_not_defined"; + fullTestPath = hdfsPath + gpdbTableNamePrefix; + + prepareWritableExternalTable(gpdbTableNamePrefix, ORC_PRIMITIVE_TABLE_COLUMNS, fullTestPath); + + runTincTest("pxf.features.orc.write.decimal_with_large_precision_not_defined.runTest"); + } + + @Test(groups = {"features", "gpdb", "security", "hcfs"}) + public void orcWriteDecimalIntegerDigitCountOverflow() throws Exception { + gpdbTableNamePrefix = "orc_decimals_with_large_integer_digit_count"; + fullTestPath = hdfsPath + gpdbTableNamePrefix; + + prepareWritableExternalTable(gpdbTableNamePrefix, ORC_PRIMITIVE_TABLE_COLUMNS, fullTestPath); + + runTincTest("pxf.features.orc.write.decimal_with_large_integer_digit.runTest"); + } + + @Test(groups = {"features", "gpdb", "security", "hcfs"}) + public void orcWriteDecimalScaleOverflow() throws Exception { + gpdbTableNamePrefix = "orc_decimals_with_large_scale"; + fullTestPath = hdfsPath + gpdbTableNamePrefix; + + prepareWritableExternalTable(gpdbTableNamePrefix, ORC_PRIMITIVE_TABLE_COLUMNS, fullTestPath); + prepareReadableExternalTable(gpdbTableNamePrefix, ORC_PRIMITIVE_TABLE_COLUMNS, fullTestPath, false /*mapByPosition*/); + + runTincTest("pxf.features.orc.write.decimal_with_large_scale.runTest"); + } + private void insertDataWithoutNulls(String exTable, int numRows) throws Exception { StringBuilder statementBuilder = new StringBuilder("INSERT INTO " + exTable + "_writable VALUES "); for (int i = 0; i < numRows; i++) { diff --git a/automation/src/test/java/org/greenplum/pxf/automation/features/parquet/ParquetWriteTest.java b/automation/src/test/java/org/greenplum/pxf/automation/features/parquet/ParquetWriteTest.java index 1e6acf5bb8..651185a982 100644 --- a/automation/src/test/java/org/greenplum/pxf/automation/features/parquet/ParquetWriteTest.java +++ b/automation/src/test/java/org/greenplum/pxf/automation/features/parquet/ParquetWriteTest.java @@ -1,10 +1,11 @@ package org.greenplum.pxf.automation.features.parquet; +import annotations.WorksWithFDW; import com.google.common.collect.Lists; import jsystem.framework.system.SystemManagerImpl; import org.apache.commons.lang.StringUtils; import org.greenplum.pxf.automation.components.hive.Hive; -import org.greenplum.pxf.automation.features.BaseFeature; +import org.greenplum.pxf.automation.features.BaseWritableFeature; import org.greenplum.pxf.automation.structures.tables.basic.Table; import org.greenplum.pxf.automation.structures.tables.hive.HiveExternalTable; import org.greenplum.pxf.automation.structures.tables.hive.HiveTable; @@ -23,7 +24,8 @@ import static java.lang.Thread.sleep; import static org.junit.Assert.assertEquals; -public class ParquetWriteTest extends BaseFeature { +@WorksWithFDW +public class ParquetWriteTest extends BaseWritableFeature { private static final String NUMERIC_TABLE = "numeric_precision"; private static final String NUMERIC_UNDEFINED_PRECISION_TABLE = "numeric_undefined_precision"; private static final String PXF_PARQUET_PRIMITIVE_TABLE = "pxf_parquet_primitive_types"; @@ -126,6 +128,12 @@ public class ParquetWriteTest extends BaseFeature { "id INTEGER", "tm_arr TIMESTAMP[]" }; + + private final String[] PARQUET_PRIMITIVE_COLUMN_NAMES = new String[]{ + "s1", "s2", "n1", "d1", "dc1", "tm", "f", "bg", "b", "tn", "vc1", "sml", "c1", "bin"}; + + private final String[] PARQUET_ARRAY_COLUMN_NAMES = new String[]{"id", "bool_arr", "smallint_arr", "int_arr", + "bigint_arr", "real_arr", "double_arr", "text_arr", "bytea_arr", "char_arr", "varchar_arr", "numeric_arr", "date_arr"}; private String hdfsPath; private ProtocolEnum protocol; private Hive hive; @@ -152,12 +160,12 @@ public void parquetWritePaddedChar() throws Exception { runWritePrimitivesScenario("pxf_parquet_write_padded_char", "pxf_parquet_read_padded_char", "parquet_write_padded_char", null); /* 2. Insert data with chars that need padding */ - gpdb.runQuery("INSERT INTO pxf_parquet_write_padded_char VALUES ('row25_char_needs_padding', 's_17', 11, 37, 0.123456789012345679, " + - "'2013-07-23 21:00:05', 7.7, 23456789, false, 11, 'abcde', 1100, 'a ', '1')"); - gpdb.runQuery("INSERT INTO pxf_parquet_write_padded_char VALUES ('row26_char_with_tab', 's_17', 11, 37, 0.123456789012345679, " + - "'2013-07-23 21:00:05', 7.7, 23456789, false, 11, 'abcde', 1100, e'b\\t ', '1')"); - gpdb.runQuery("INSERT INTO pxf_parquet_write_padded_char VALUES ('row27_char_with_newline', 's_17', 11, 37, 0.123456789012345679, " + - "'2013-07-23 21:00:05', 7.7, 23456789, false, 11, 'abcde', 1100, e'c\\n ', '1')"); + gpdb.insertData("('row25_char_needs_padding', 's_17', 11, 37, 0.123456789012345679, " + + "'2013-07-23 21:00:05', 7.7, 23456789, false, 11, 'abcde', 1100, 'a ', '1')", writableExTable); + gpdb.insertData("('row26_char_with_tab', 's_17', 11, 37, 0.123456789012345679, " + + "'2013-07-23 21:00:05', 7.7, 23456789, false, 11, 'abcde', 1100, e'b\\t ', '1')", writableExTable); + gpdb.insertData("('row27_char_with_newline', 's_17', 11, 37, 0.123456789012345679, " + + "'2013-07-23 21:00:05', 7.7, 23456789, false, 11, 'abcde', 1100, e'c\\n ', '1')", writableExTable); if (protocol != ProtocolEnum.HDFS && protocol != ProtocolEnum.FILE) { // for HCFS on Cloud, wait a bit for async write in previous steps to finish @@ -195,7 +203,7 @@ public void parquetWriteUndefinedPrecisionNumeric() throws Exception { String writableExternalTableName = "pxf_parquet_write_undefined_precision_numeric"; prepareNumericWritableExtTable(filePathName, fileName, writableExternalTableName, false, false); - gpdb.runQuery("INSERT INTO " + exTable.getName() + " SELECT * FROM " + NUMERIC_UNDEFINED_PRECISION_TABLE); + gpdb.copyData(NUMERIC_UNDEFINED_PRECISION_TABLE, writableExTable); prepareReadableExternalTable("pxf_parquet_read_undefined_precision_numeric", UNDEFINED_PRECISION_NUMERIC, hdfsPath + fileName); @@ -220,7 +228,7 @@ public void parquetWriteNumericWithPrecisionAndScale() throws Exception { String writableExternalTableName = "pxf_parquet_write_numeric"; prepareNumericWritableExtTable(filePathName, fileName, writableExternalTableName, true, false); - gpdb.runQuery("INSERT INTO " + exTable.getName() + " SELECT * FROM " + NUMERIC_TABLE); + gpdb.copyData(NUMERIC_TABLE, writableExTable); prepareReadableExternalTable("pxf_parquet_read_numeric", PARQUET_TABLE_DECIMAL_COLUMNS, hdfsPath + fileName); @@ -257,7 +265,7 @@ public void parquetWriteUndefinedPrecisionNumericWithScaleOverflow() throws Exce String writableExternalTableName = "parquet_write_undefined_precision_numeric_large_scale"; prepareNumericWritableExtTable(filePathName, fileName, writableExternalTableName, false, false); - gpdb.runQuery("INSERT INTO " + exTable.getName() + " SELECT * FROM " + NUMERIC_UNDEFINED_PRECISION_TABLE); + gpdb.copyData(NUMERIC_UNDEFINED_PRECISION_TABLE, writableExTable); prepareReadableExternalTable("pxf_parquet_read_undefined_precision_numeric_large_scale", UNDEFINED_PRECISION_NUMERIC, hdfsPath + fileName); @@ -271,8 +279,7 @@ public void parquetWriteLists() throws Exception { String fullTestPath = hdfsPath + "parquet_write_list"; prepareWritableExternalTable(writeTableName, PARQUET_LIST_TABLE_COLUMNS, fullTestPath, null); - gpdb.runQuery("INSERT INTO " + exTable.getName() + " SELECT id, bool_arr, smallint_arr, int_arr, bigint_arr, real_arr, " + - "double_arr, text_arr, bytea_arr, char_arr, varchar_arr, numeric_arr, date_arr FROM " + PXF_PARQUET_LIST_TYPES); + gpdb.copyData(PXF_PARQUET_LIST_TYPES, writableExTable, PARQUET_ARRAY_COLUMN_NAMES); waitForAsyncWriteToSucceedOnHCFS("parquet_write_list"); @@ -290,7 +297,7 @@ public void parquetWriteTimestampLists() throws Exception { String fullTestPath = hdfsPath + "parquet_write_timestamp_list"; prepareWritableExternalTable(writeTableName, PARQUET_TIMESTAMP_LIST_TABLE_COLUMNS, fullTestPath, null); - gpdb.runQuery("INSERT INTO " + exTable.getName() + " SELECT id, tm_arr FROM " + PXF_PARQUET_TIMESTAMP_LIST_TYPES); + gpdb.copyData(PXF_PARQUET_TIMESTAMP_LIST_TYPES, writableExTable, new String[]{"id", "tm_arr"}); waitForAsyncWriteToSucceedOnHCFS("parquet_write_timestamp_list"); @@ -313,7 +320,7 @@ public void parquetWriteListsReadWithHive() throws Exception { String fullTestPath = hdfsPath + "parquet_write_list_read_with_hive"; prepareWritableExternalTable(writeTableName, PARQUET_LIST_TABLE_COLUMNS, fullTestPath, null); - insertArrayDataWithoutNulls(writeTableName, 33); + insertArrayDataWithoutNulls(writableExTable, 33); // load the data into hive to check that PXF-written Parquet files can be read by other data String hiveExternalTableName = writeTableName + "_external"; @@ -394,13 +401,11 @@ public void parquetWriteListsUserProvidedSchemaFile_ValidSchema() throws Excepti } hdfs.copyFromLocal(resourcePath + "parquet_list.schema", absoluteSchemaPath); - exTable.setExternalDataSchema(schemaPath); - // update the exTable with schema file provided - gpdb.createTableAndVerify(exTable); - - gpdb.runQuery("INSERT INTO " + exTable.getName() + " SELECT id, bool_arr, smallint_arr, int_arr, bigint_arr, real_arr, " + - "double_arr, text_arr, bytea_arr, char_arr, varchar_arr, numeric_arr, date_arr FROM " + PXF_PARQUET_LIST_TYPES); + writableExTable.setExternalDataSchema(schemaPath); + // update the writableExTable with schema file provided + gpdb.createTableAndVerify(writableExTable); + gpdb.copyData(PXF_PARQUET_LIST_TYPES, writableExTable, PARQUET_ARRAY_COLUMN_NAMES); waitForAsyncWriteToSucceedOnHCFS("parquet_write_lists_with_user_provided_schema_file_on_hcfs"); prepareReadableExternalTable(readTableName, PARQUET_LIST_TABLE_COLUMNS, fullTestPath); @@ -425,9 +430,9 @@ public void parquetWriteListsUserProvidedSchemaFile_InvalidSchema() throws Excep } hdfs.copyFromLocal(resourcePath + "invalid_parquet_list.schema", absoluteSchemaPath); - exTable.setExternalDataSchema(schemaPath); - // update the exTable with schema file provided - gpdb.createTableAndVerify(exTable); + writableExTable.setExternalDataSchema(schemaPath); + // update the writableExTable with schema file provided + gpdb.createTableAndVerify(writableExTable); runTincTest("pxf.features.parquet.write_list.write_with_invalid_schema_hcfs.runTest"); } @@ -436,9 +441,7 @@ private void runWritePrimitivesScenario(String writeTableName, String readTableN String filename, String[] userParameters) throws Exception { prepareWritableExternalTable(writeTableName, PARQUET_PRIMITIVE_TABLE_COLUMNS, hdfsPath + filename, userParameters); - gpdb.runQuery("INSERT INTO " + exTable.getName() + " SELECT s1, s2, n1, d1, dc1, tm, " + - "f, bg, b, tn, vc1, sml, c1, bin FROM " + PXF_PARQUET_PRIMITIVE_TABLE); - + gpdb.copyData(PXF_PARQUET_PRIMITIVE_TABLE, writableExTable, PARQUET_PRIMITIVE_COLUMN_NAMES); waitForAsyncWriteToSucceedOnHCFS(filename); prepareReadableExternalTable(readTableName, @@ -451,16 +454,16 @@ private void runWritePrimitivesScenario(String writeTableName, String readTableN } private void prepareReadableExternalTable(String name, String[] fields, String path) throws Exception { - exTable = TableFactory.getPxfHcfsReadableTable(name, fields, path, hdfs.getBasePath(), "parquet"); - createTable(exTable); + readableExTable = TableFactory.getPxfHcfsReadableTable(name, fields, path, hdfs.getBasePath(), "parquet"); + createTable(readableExTable); } private void prepareWritableExternalTable(String name, String[] fields, String path, String[] userParameters) throws Exception { - exTable = TableFactory.getPxfHcfsWritableTable(name, fields, path, hdfs.getBasePath(), "parquet"); + writableExTable = TableFactory.getPxfHcfsWritableTable(name, fields, path, hdfs.getBasePath(), "parquet"); if (userParameters != null) { - exTable.setUserParameters(userParameters); + writableExTable.setUserParameters(userParameters); } - createTable(exTable); + createTable(writableExTable); } private void waitForAsyncWriteToSucceedOnHCFS(String filename) throws Exception { @@ -478,9 +481,8 @@ private void waitForAsyncWriteToSucceedOnHCFS(String filename) throws Exception } } - private void insertArrayDataWithoutNulls(String exTable, int numRows) throws Exception { - StringBuilder insertStatement = new StringBuilder(); - insertStatement.append("INSERT INTO " + exTable + " VALUES "); + private void insertArrayDataWithoutNulls(Table exTable, int numRows) throws Exception { + StringBuilder values = new StringBuilder(); for (int i = 0; i < numRows; i++) { StringJoiner statementBuilder = new StringJoiner(",", "(", ")") .add(String.valueOf(i)) // always not-null row index, column index starts with 0 after it @@ -497,9 +499,9 @@ private void insertArrayDataWithoutNulls(String exTable, int numRows) throws Exc .add(String.format("'{12345678900000.00000%s}'", i)) // DataType.NUMERICARRAY .add(String.format("'{\"2010-01-%02d\"}'", (i % 30) + 1)) // DataType.DATEARRAY ; - insertStatement.append(statementBuilder.toString().concat((i < (numRows - 1)) ? "," : ";")); + values.append(statementBuilder.toString().concat((i < (numRows - 1)) ? "," : "")); } - gpdb.runQuery(insertStatement.toString()); + gpdb.insertData(values.toString(), exTable); } private void assertHiveByteaArrayData(List> queryResultData) { @@ -589,11 +591,11 @@ private void prepareNumericWritableExtTable(String filePathName, String fileName prepareWritableExternalTable(writableExternalTableName, numericTableColumns, hdfsPath + fileName, null); - exTable.setHost(pxfHost); - exTable.setPort(pxfPort); - exTable.setFormatter("pxfwritable_export"); - exTable.setProfile(ProtocolUtils.getProtocol().value() + ":parquet"); + writableExTable.setHost(pxfHost); + writableExTable.setPort(pxfPort); + writableExTable.setFormatter("pxfwritable_export"); + writableExTable.setProfile(ProtocolUtils.getProtocol().value() + ":parquet"); - gpdb.createTableAndVerify(exTable); + gpdb.createTableAndVerify(writableExTable); } } \ No newline at end of file diff --git a/automation/src/test/java/org/greenplum/pxf/automation/features/writable/HdfsWritableTextTest.java b/automation/src/test/java/org/greenplum/pxf/automation/features/writable/HdfsWritableTextTest.java index ec7b372c9a..92276f2c73 100755 --- a/automation/src/test/java/org/greenplum/pxf/automation/features/writable/HdfsWritableTextTest.java +++ b/automation/src/test/java/org/greenplum/pxf/automation/features/writable/HdfsWritableTextTest.java @@ -1,5 +1,7 @@ package org.greenplum.pxf.automation.features.writable; +import annotations.SkipForFDW; +import annotations.WorksWithFDW; import org.apache.commons.lang.StringUtils; import org.greenplum.pxf.automation.datapreparer.writable.WritableDataPreparer; import org.greenplum.pxf.automation.enums.EnumCompressionTypes; @@ -9,6 +11,7 @@ import org.greenplum.pxf.automation.structures.tables.pxf.WritableExternalTable; import org.greenplum.pxf.automation.structures.tables.utils.TableFactory; import org.greenplum.pxf.automation.utils.fileformats.FileFormatsUtils; +import org.greenplum.pxf.automation.utils.system.FDWUtils; import org.greenplum.pxf.automation.utils.system.ProtocolEnum; import org.greenplum.pxf.automation.utils.system.ProtocolUtils; import org.greenplum.pxf.automation.utils.tables.ComparisonUtils; @@ -26,6 +29,7 @@ /** * Testing cases for PXF Writable feature for Text formats (Text, CSV) and compressions. */ +@WorksWithFDW public class HdfsWritableTextTest extends BaseWritableFeature { private static final String COMPRESSION_CODEC = "org.apache.hadoop.io.compress.DefaultCodec"; @@ -73,7 +77,13 @@ public void textFormatInsertNoProfile() throws Exception { String hdfsPath = hdfsWritePath + "/text_format_no_profile"; writableExTable = prepareWritableTable("pxf_text_format_no_profile", hdfsPath, null); - writableExTable.setProfile(null); + if (FDWUtils.useFDW) { + // FDW does not really allow empty format, so we use special "test:text" profile that gets ignored + writableExTable.setFormat("text"); + writableExTable.setProfile("test:text"); + } else { + writableExTable.setProfile(null); + } writableExTable.setFragmenter("org.greenplum.pxf.plugins.hdfs.HdfsDataFragmenter"); writableExTable.setAccessor("org.greenplum.pxf.plugins.hdfs.LineBreakAccessor"); writableExTable.setResolver("org.greenplum.pxf.plugins.hdfs.StringPassResolver"); @@ -288,12 +298,7 @@ public void textFormatGZipInsert() throws Exception { public void textFormatGZipInsertShortname() throws Exception { String hdfsPath = hdfsWritePath + "/gzip_shortname_format_using_insert"; - writableExTable = new WritableExternalTable("pxf_gzip_shortname_format_using_insert", gpdbTableFields, - protocol.getExternalTablePath(hdfs.getBasePath(), hdfsPath), "Text"); - writableExTable.setProfile(ProtocolUtils.getProtocol().value() + ":text"); - writableExTable.setDelimiter(","); - writableExTable.setCompressionCodec("gzip"); - createTable(writableExTable); + writableExTable = prepareWritableGzipTable("pxf_gzip_shortname_format_using_insert", hdfsPath, "gzip"); insertData(dataTable, writableExTable, InsertionMethod.INSERT); verifyResult(hdfsPath, dataTable, EnumCompressionTypes.GZip); } @@ -341,7 +346,6 @@ public void textFormatBZip2Insert() throws Exception { String hdfsPath = hdfsWritePath + "/bzip2_format_using_insert"; writableExTable = prepareWritableBZip2Table("pxf_bzip2_format_using_insert", hdfsPath); - insertData(dataTable, writableExTable, InsertionMethod.INSERT); verifyResult(hdfsPath, dataTable, EnumCompressionTypes.BZip2); } @@ -356,7 +360,6 @@ public void textFormatBZip2CopyFromStdin() throws Exception { String hdfsPath = hdfsWritePath + "/copy_from_stdin_bzip2"; writableExTable = prepareWritableBZip2Table("pxf_copy_from_stdin_bzip2", hdfsPath); - insertData(dataTable, writableExTable, InsertionMethod.COPY); verifyResult(hdfsPath, dataTable, EnumCompressionTypes.BZip2); } @@ -370,12 +373,7 @@ public void textFormatBZip2CopyFromStdin() throws Exception { public void textFormatBZip2CopyFromStdinShortname() throws Exception { String hdfsPath = hdfsWritePath + "/copy_from_stdin_bzip2_shortname"; - writableExTable = new WritableExternalTable("pxf_copy_from_stdin_bzip2_shortname", gpdbTableFields, - protocol.getExternalTablePath(hdfs.getBasePath(), hdfsPath), "Text"); - writableExTable.setProfile(ProtocolUtils.getProtocol().value() + ":text"); - writableExTable.setDelimiter(","); - writableExTable.setCompressionCodec("bzip2"); - createTable(writableExTable); + writableExTable = prepareWritableBZip2Table("pxf_copy_from_stdin_bzip2_shortname", hdfsPath, "bzip2"); insertData(dataTable, writableExTable, InsertionMethod.COPY); verifyResult(hdfsPath, dataTable, EnumCompressionTypes.BZip2); } @@ -541,7 +539,9 @@ public void copyFromFileMultiBlockedDataBZip2() throws Exception { * The test creates a writable external table, copies the data into it, then * uses a readable external table to compare the data with the original. */ - @Test(groups = {"features"}) + // FDW does not yet support DISTRIBUTED BY on foreign tables, we can't guarantee only 1 segment will write data + @SkipForFDW + @Test(groups = {"features", "gpdb", "hcfs", "security"}) public void veryLongRecords() throws Exception { final String[][] data = new String[][]{ @@ -558,25 +558,24 @@ public void veryLongRecords() throws Exception { dataTable.addRows(data); String hdfsPath = hdfsWritePath + writableTableName + "_verylongrecord"; - writableExTable.setFields(fields); - writableExTable.setName("verylongrecordexport"); - writableExTable.setPath(hdfsPath); + writableExTable = TableFactory.getPxfWritableTextTable("pxf_text_very_long_records_w", fields, + protocol.getExternalTablePath(hdfs.getBasePath(), hdfsPath), ","); writableExTable.setFormat("CSV"); writableExTable.setDistributionFields(new String[]{"key"}); gpdb.createTableAndVerify(writableExTable); + insertData(dataTable, writableExTable, InsertionMethod.INSERT); - gpdb.insertData(dataTable, writableExTable); - Assert.assertEquals("More than one segment wrote to " + hdfsPath, - 1, hdfs.list(hdfsPath).size()); + if (protocol != ProtocolEnum.HDFS) { + // for HCFS on Cloud, wait a bit for async write in previous steps to finish + sleep(10000); + } + Assert.assertEquals("More than one segment wrote to " + hdfsPath,1, hdfs.list(hdfsPath).size()); - readableExTable.setFields(fields); - readableExTable.setPath(hdfsPath); - readableExTable.setName("verylongrecordimport"); - readableExTable.setFormat("csv"); + readableExTable = TableFactory.getPxfReadableCSVTable("pxf_text_very_long_records_r", fields, + protocol.getExternalTablePath(hdfs.getBasePath(), hdfsPath), ","); gpdb.createTableAndVerify(readableExTable); - gpdb.queryResults(readableExTable, - "SELECT * FROM " + readableExTable.getName() + " ORDER BY linenum"); + gpdb.queryResults(readableExTable,"SELECT * FROM " + readableExTable.getName() + " ORDER BY linenum"); ComparisonUtils.compareTables(readableExTable, dataTable, null); } @@ -594,7 +593,7 @@ private void verifyResult(String hdfsPath, Table data, EnumCompressionTypes comp String localResultFile = dataTempFolder + "/" + hdfsPath.replaceAll("/", "_"); // for HCFS on Cloud, wait a bit for async write in previous steps to finish - if (protocol != ProtocolEnum.HDFS && protocol != ProtocolEnum.FILE) { + if (protocol != ProtocolEnum.HDFS) { sleep(10000); } List files = hdfs.list(hdfsPath); @@ -602,11 +601,6 @@ private void verifyResult(String hdfsPath, Table data, EnumCompressionTypes comp int index = 0; for (String file : files) { String pathToLocalFile = localResultFile + "/_" + index; - // make sure the file is available, saw flakes on Cloud that listed files were not available - int attempts = 0; - while (!hdfs.doesFileExist(file) && attempts++ < 20) { - sleep(1000); - } hdfs.copyToLocal(file, pathToLocalFile); sleep(250); resultTable.loadDataFromFile(pathToLocalFile, ",", 1, "UTF-8", @@ -648,7 +642,7 @@ private void insertData(Table data, WritableExternalTable table, InsertionMethod gpdb.copyFromStdin(data, table, ",", false); break; case INSERT_FROM_TABLE: - gpdb.runQuery("INSERT INTO " + table.getName() + " SELECT * FROM " + data.getName()); + gpdb.copyData(data, table); break; } } @@ -685,15 +679,28 @@ private WritableExternalTable prepareWritableTable(String name, String path, Str } private WritableExternalTable prepareWritableBZip2Table(String name, String path) throws Exception { + return prepareWritableBZip2Table(name, path, null); + } + + private WritableExternalTable prepareWritableBZip2Table(String name, String path, String customCodecName) throws Exception { WritableExternalTable table = TableFactory.getPxfWritableBZip2Table(name, gpdbTableFields, protocol.getExternalTablePath(hdfs.getBasePath(), path), ","); + if (customCodecName != null) { + table.setCompressionCodec(customCodecName); + } createTable(table); return table; } private WritableExternalTable prepareWritableGzipTable(String name, String path) throws Exception { + return prepareWritableGzipTable(name, path, null); + } + private WritableExternalTable prepareWritableGzipTable(String name, String path, String customCodecName) throws Exception { WritableExternalTable table = TableFactory.getPxfWritableGzipTable(name, gpdbTableFields, protocol.getExternalTablePath(hdfs.getBasePath(), path), ","); + if (customCodecName != null) { + table.setCompressionCodec(customCodecName); + } createTable(table); return table; } diff --git a/automation/src/test/java/org/greenplum/pxf/automation/smoke/WritableSmokeTest.java b/automation/src/test/java/org/greenplum/pxf/automation/smoke/WritableSmokeTest.java index 7cd6728709..dac2b0725b 100755 --- a/automation/src/test/java/org/greenplum/pxf/automation/smoke/WritableSmokeTest.java +++ b/automation/src/test/java/org/greenplum/pxf/automation/smoke/WritableSmokeTest.java @@ -1,16 +1,25 @@ package org.greenplum.pxf.automation.smoke; -import java.io.File; - +import annotations.WorksWithFDW; import org.greenplum.pxf.automation.structures.tables.basic.Table; import org.greenplum.pxf.automation.structures.tables.pxf.WritableExternalTable; import org.greenplum.pxf.automation.structures.tables.utils.TableFactory; import org.greenplum.pxf.automation.utils.files.FileUtils; import org.testng.annotations.Test; +import java.io.File; + /** Write data to HDFS using Writable External table. Read it using PXF. */ +@WorksWithFDW public class WritableSmokeTest extends BaseSmoke { WritableExternalTable writableExTable; + private final static String[] FIELDS = new String[]{ + "name text", + "num integer", + "dub double precision", + "longNum bigint", + "bool boolean" + }; @Override protected void prepareData() throws Exception { @@ -23,30 +32,18 @@ protected void prepareData() throws Exception { @Override protected void createTables() throws Exception { // Create Writable external table - writableExTable = new WritableExternalTable("hdfs_writable_table", new String[] { - "name text", - "num integer", - "dub double precision", - "longNum bigint", - "bool boolean" - }, hdfs.getWorkingDirectory() + "/bzip", "Text"); + writableExTable = TableFactory.getPxfWritableTextTable("hdfs_writable_table", FIELDS, + hdfs.getWorkingDirectory() + "/bzip", "|"); - writableExTable.setAccessor("org.greenplum.pxf.plugins.hdfs.LineBreakAccessor"); - writableExTable.setResolver("org.greenplum.pxf.plugins.hdfs.StringPassResolver"); writableExTable.setCompressionCodec("org.apache.hadoop.io.compress.BZip2Codec"); - writableExTable.setDelimiter("|"); writableExTable.setHost(pxfHost); writableExTable.setPort(pxfPort); gpdb.createTableAndVerify(writableExTable); gpdb.copyFromFile(writableExTable, new File(dataTempFolder + "/" + fileName), "|", false); + // Create Readable External Table - exTable = TableFactory.getPxfReadableTextTable("pxf_smoke_small_data", new String[] { - "name text", - "num integer", - "dub double precision", - "longNum bigint", - "bool boolean" - }, hdfs.getWorkingDirectory() + "/bzip", "|"); + exTable = TableFactory.getPxfReadableTextTable("pxf_smoke_small_data", FIELDS, + hdfs.getWorkingDirectory() + "/bzip", "|"); exTable.setHost(pxfHost); exTable.setPort(pxfPort); gpdb.createTableAndVerify(exTable); diff --git a/automation/src/test/resources/data/json/array_types.csv b/automation/src/test/resources/data/json/array_types.csv new file mode 100644 index 0000000000..ead33f3f8f --- /dev/null +++ b/automation/src/test/resources/data/json/array_types.csv @@ -0,0 +1,16 @@ +0,"{row-00,""esc1\"""",""esc2\\"",""esc3}"",NULL}","{0,1,NULL}","{1000000,-1000000,NULL}","{5555500000,-5555500000,NULL}","{0.0001,-0.0001,NULL}","{3.14159265358979,-3.14159265358979,NULL}","{12345678900000.000000,-12345678900000.000000,NULL}","{t,f,NULL}","{2010-01-01,2010-01-01,NULL}","{10:11:00,11:11:00,NULL}","{2013-07-13 21:00:05.000456,2014-07-13 21:00:05.000456,NULL}","{2013-07-13 21:00:05.000123-07,2014-07-13 21:00:05.000123-07,NULL}","{abc,cba,NULL}","{defhi,jklmn,NULL}","{b-0,c-0,NULL}" +1,,"{}","{1000001}","{NULL}","{1.0001,-1.0001}","{3.14159265358979,-3.14159265358979}","{12345678900000.000001,-12345678900000.000001}","{t,f}","{2010-01-02,2010-02-02}","{10:11:01,11:11:01}","{2013-07-13 21:00:05.001456,2014-07-13 21:00:05.001456}","{2013-07-13 21:00:05.001123-07,2014-07-13 21:00:05.001123-07}","{abc,cba,NULL}","{defhi,NULL,jklmn}","{NULL,b-1,c-1}" +2,"{NULL,""esc1\"""",""esc2\\"",""esc3}""}",,"{}","{5555500002}","{NULL}","{3.14159265358979,-3.14159265358979}","{12345678900000.000002,-12345678900000.000002}","{t,f}","{2010-01-03,2010-02-03}","{10:11:02,11:11:02}","{2013-07-13 21:00:05.002456,2014-07-13 21:00:05.002456}","{2013-07-13 21:00:05.002123-07,2014-07-13 21:00:05.002123-07}","{abc,cba}","{defhi,jklmn,NULL}","{b-2,NULL,c-2}" +3,"{row-03,NULL,""esc2\\"",""esc3}""}","{NULL,3,4}",,"{}","{-3.0001}","{NULL}","{12345678900000.000003,-12345678900000.000003}","{t,f}","{2010-01-04,2010-02-04}","{10:11:03,11:11:03}","{2013-07-13 21:00:05.003456,2014-07-13 21:00:05.003456}","{2013-07-13 21:00:05.003123-07,2014-07-13 21:00:05.003123-07}","{abc,cba}","{defhi,jklmn}","{b-3,c-3,NULL}" +4,"{row-04,""esc1\"""",NULL,""esc3}""}","{4,NULL,5}","{NULL,1000004,-1000004}",,"{}","{3.14159265358979}","{NULL}","{t,f}","{2010-01-05,2010-02-05}","{10:11:04,11:11:04}","{2013-07-13 21:00:05.004456,2014-07-13 21:00:05.004456}","{2013-07-13 21:00:05.004123-07,2014-07-13 21:00:05.004123-07}","{abc,cba}","{defhi,jklmn}","{b-4,c-4}" +5,"{row-05,""esc1\"""",""esc2\\"",NULL}","{5,6,NULL}","{1000005,NULL,-1000005}","{NULL,5555500005,-5555500005}",,"{}","{-12345678900000.000005}","{NULL}","{2010-01-06,2010-02-06}","{10:11:05,11:11:05}","{2013-07-13 21:00:05.005456,2014-07-13 21:00:05.005456}","{2013-07-13 21:00:05.005123-07,2014-07-13 21:00:05.005123-07}","{abc,cba}","{defhi,jklmn}","{b-5,c-5}" +6,"{row-06,""esc1\"""",""esc2\\"",""esc3}""}","{6,7}","{1000006,-1000006,NULL}","{5555500006,NULL,-5555500006}","{NULL,6.0001,-6.0001}",,"{}","{t}","{NULL}","{10:11:06,11:11:06}","{2013-07-13 21:00:05.006456,2014-07-13 21:00:05.006456}","{2013-07-13 21:00:05.006123-07,2014-07-13 21:00:05.006123-07}","{abc,cba}","{defhi,jklmn}","{b-6,c-6}" +7,"{row-07,""esc1\"""",""esc2\\"",""esc3}""}","{7,8}","{1000007,-1000007}","{5555500007,-5555500007,NULL}","{7.0001,NULL,-7.0001}","{NULL,3.14159265358979,-3.14159265358979}",,"{}","{2010-01-08}","{NULL}","{2013-07-13 21:00:05.007456,2014-07-13 21:00:05.007456}","{2013-07-13 21:00:05.007123-07,2014-07-13 21:00:05.007123-07}","{abc,cba}","{defhi,jklmn}","{b-7,c-7}" +8,"{row-08,""esc1\"""",""esc2\\"",""esc3}""}","{8,9}","{1000008,-1000008}","{5555500008,-5555500008}","{8.0001,-8.0001,NULL}","{3.14159265358979,NULL,-3.14159265358979}","{NULL,12345678900000.000008,-12345678900000.000008}",,"{}","{10:11:08}","{NULL}","{2013-07-13 21:00:05.008123-07,2014-07-13 21:00:05.008123-07}","{abc,cba}","{defhi,jklmn}","{b-8,c-8}" +9,"{row-09,""esc1\"""",""esc2\\"",""esc3}""}","{9,10}","{1000009,-1000009}","{5555500009,-5555500009}","{9.0001,-9.0001}","{3.14159265358979,-3.14159265358979,NULL}","{12345678900000.000009,NULL,-12345678900000.000009}","{NULL,t,f}",,"{}","{2013-07-13 21:00:05.009456}","{NULL}","{abc,cba}","{defhi,jklmn}","{b-9,c-9}" +10,"{row-10,""esc1\"""",""esc2\\"",""esc3}""}","{10,11}","{1000010,-1000010}","{5555500010,-5555500010}","{10.0001,-10.0001}","{3.14159265358979,-3.14159265358979}","{12345678900000.000010,-12345678900000.000010,NULL}","{t,NULL,f}","{NULL,2010-01-11,2010-02-11}",,"{}","{2013-07-13 21:00:05.010123-07}","{NULL}","{defhi,jklmn}","{b-10,c-10}" +11,"{row-11,""esc1\"""",""esc2\\"",""esc3}""}","{11,12}","{1000011,-1000011}","{5555500011,-5555500011}","{11.0001,-11.0001}","{3.14159265358979,-3.14159265358979}","{12345678900000.000011,-12345678900000.000011}","{t,f,NULL}","{2010-01-12,NULL,2010-02-12}","{NULL,10:11:11,11:11:11}",,"{}","{abc}","{NULL}","{b-11,c-11}" +12,"{row-12,""esc1\"""",""esc2\\"",""esc3}""}","{12,13}","{1000012,-1000012}","{5555500012,-5555500012}","{12.0001,-12.0001}","{3.14159265358979,-3.14159265358979}","{12345678900000.000012,-12345678900000.000012}","{t,f}","{2010-01-13,2010-02-13,NULL}","{10:11:12,NULL,11:11:12}","{NULL,2013-07-13 21:00:05.012456,2014-07-13 21:00:05.012456}",,"{}","{defhi}","{NULL}" +13,"{NULL}","{13,14}","{1000013,-1000013}","{5555500013,-5555500013}","{13.0001,-13.0001}","{3.14159265358979,-3.14159265358979}","{12345678900000.000013,-12345678900000.000013}","{t,f}","{2010-01-14,2010-02-14}","{10:11:13,11:11:13,NULL}","{2013-07-13 21:00:05.013456,NULL,2014-07-13 21:00:05.013456}","{NULL,2013-07-13 21:00:05.013123-07,2014-07-13 21:00:05.013123-07}",,"{}","{b-13}" +14,"{row-14}","{NULL}","{1000014,-1000014}","{5555500014,-5555500014}","{14.0001,-14.0001}","{3.14159265358979,-3.14159265358979}","{12345678900000.000014,-12345678900000.000014}","{t,f}","{2010-01-15,2010-02-15}","{10:11:14,11:11:14}","{2013-07-13 21:00:05.014456,2014-07-13 21:00:05.014456,NULL}","{2013-07-13 21:00:05.014123-07,NULL,2014-07-13 21:00:05.014123-07}","{NULL,abc,cba}",,"{}" +15,"{}","{15}","{NULL}","{5555500015,-5555500015}","{15.0001,-15.0001}","{3.14159265358979,-3.14159265358979}","{12345678900000.000015,-12345678900000.000015}","{t,f}","{2010-01-16,2010-02-16}","{10:11:15,11:11:15}","{2013-07-13 21:00:05.015456,2014-07-13 21:00:05.015456}","{2013-07-13 21:00:05.015123-07,2014-07-13 21:00:05.015123-07,NULL}","{abc,NULL,cba}","{NULL,defhi,jklmn}", diff --git a/automation/src/test/resources/data/json/array_types_aligned.csv b/automation/src/test/resources/data/json/array_types_aligned.csv new file mode 100644 index 0000000000..4e3895e632 --- /dev/null +++ b/automation/src/test/resources/data/json/array_types_aligned.csv @@ -0,0 +1,16 @@ +0,"{row-00,""esc1\"""",""esc2\\"",""esc3}"",NULL}","{0,1,NULL}" ,"{1000000,-1000000,NULL}","{5555500000,-5555500000,NULL}","{0.0001,-0.0001,NULL}","{3.14159265358979,-3.14159265358979,NULL}","{12345678900000.000000,-12345678900000.000000,NULL}","{t,f,NULL}","{2010-01-01,2010-01-01,NULL}","{10:11:00,11:11:00,NULL}","{2013-07-13 21:00:05.000456,2014-07-13 21:00:05.000456,NULL}","{2013-07-13 21:00:05.000123-07,2014-07-13 21:00:05.000123-07,NULL}","{abc,cba,NULL}","{defhi,jklmn,NULL}","{b-0,c-0,NULL}" +1, ,"{}" ,"{1000001}" ,"{NULL}" ,"{1.0001,-1.0001}" ,"{3.14159265358979,-3.14159265358979}" ,"{12345678900000.000001,-12345678900000.000001}" ,"{t,f}" ,"{2010-01-02,2010-02-02}" ,"{10:11:01,11:11:01}" ,"{2013-07-13 21:00:05.001456,2014-07-13 21:00:05.001456}" ,"{2013-07-13 21:00:05.001123-07,2014-07-13 21:00:05.001123-07}" ,"{abc,cba,NULL}","{defhi,NULL,jklmn}","{NULL,b-1,c-1}" +2,"{NULL,""esc1\"""",""esc2\\"",""esc3}""}" , ,"{}" ,"{5555500002}" ,"{NULL}" ,"{3.14159265358979,-3.14159265358979}" ,"{12345678900000.000002,-12345678900000.000002}" ,"{t,f}" ,"{2010-01-03,2010-02-03}" ,"{10:11:02,11:11:02}" ,"{2013-07-13 21:00:05.002456,2014-07-13 21:00:05.002456}" ,"{2013-07-13 21:00:05.002123-07,2014-07-13 21:00:05.002123-07}" ,"{abc,cba}" ,"{defhi,jklmn,NULL}","{b-2,NULL,c-2}" +3,"{row-03,NULL,""esc2\\"",""esc3}""}" ,"{NULL,3,4}" , ,"{}" ,"{-3.0001}" ,"{NULL}" ,"{12345678900000.000003,-12345678900000.000003}" ,"{t,f}" ,"{2010-01-04,2010-02-04}" ,"{10:11:03,11:11:03}" ,"{2013-07-13 21:00:05.003456,2014-07-13 21:00:05.003456}" ,"{2013-07-13 21:00:05.003123-07,2014-07-13 21:00:05.003123-07}" ,"{abc,cba}" ,"{defhi,jklmn}" ,"{b-3,c-3,NULL}" +4,"{row-04,""esc1\"""",NULL,""esc3}""}" ,"{4,NULL,5}" ,"{NULL,1000004,-1000004}", ,"{}" ,"{3.14159265358979}" ,"{NULL}" ,"{t,f}" ,"{2010-01-05,2010-02-05}" ,"{10:11:04,11:11:04}" ,"{2013-07-13 21:00:05.004456,2014-07-13 21:00:05.004456}" ,"{2013-07-13 21:00:05.004123-07,2014-07-13 21:00:05.004123-07}" ,"{abc,cba}" ,"{defhi,jklmn}" ,"{b-4,c-4}" +5,"{row-05,""esc1\"""",""esc2\\"",NULL}" ,"{5,6,NULL}" ,"{1000005,NULL,-1000005}","{NULL,5555500005,-5555500005}", ,"{}" ,"{-12345678900000.000005}" ,"{NULL}" ,"{2010-01-06,2010-02-06}" ,"{10:11:05,11:11:05}" ,"{2013-07-13 21:00:05.005456,2014-07-13 21:00:05.005456}" ,"{2013-07-13 21:00:05.005123-07,2014-07-13 21:00:05.005123-07}" ,"{abc,cba}" ,"{defhi,jklmn}" ,"{b-5,c-5}" +6,"{row-06,""esc1\"""",""esc2\\"",""esc3}""}" ,"{6,7}" ,"{1000006,-1000006,NULL}","{5555500006,NULL,-5555500006}","{NULL,6.0001,-6.0001}", ,"{}" ,"{t}" ,"{NULL}" ,"{10:11:06,11:11:06}" ,"{2013-07-13 21:00:05.006456,2014-07-13 21:00:05.006456}" ,"{2013-07-13 21:00:05.006123-07,2014-07-13 21:00:05.006123-07}" ,"{abc,cba}" ,"{defhi,jklmn}" ,"{b-6,c-6}" +7,"{row-07,""esc1\"""",""esc2\\"",""esc3}""}" ,"{7,8}" ,"{1000007,-1000007}" ,"{5555500007,-5555500007,NULL}","{7.0001,NULL,-7.0001}","{NULL,3.14159265358979,-3.14159265358979}", ,"{}" ,"{2010-01-08}" ,"{NULL}" ,"{2013-07-13 21:00:05.007456,2014-07-13 21:00:05.007456}" ,"{2013-07-13 21:00:05.007123-07,2014-07-13 21:00:05.007123-07}" ,"{abc,cba}" ,"{defhi,jklmn}" ,"{b-7,c-7}" +8,"{row-08,""esc1\"""",""esc2\\"",""esc3}""}" ,"{8,9}" ,"{1000008,-1000008}" ,"{5555500008,-5555500008}" ,"{8.0001,-8.0001,NULL}","{3.14159265358979,NULL,-3.14159265358979}","{NULL,12345678900000.000008,-12345678900000.000008}", ,"{}" ,"{10:11:08}" ,"{NULL}" ,"{2013-07-13 21:00:05.008123-07,2014-07-13 21:00:05.008123-07}" ,"{abc,cba}" ,"{defhi,jklmn}" ,"{b-8,c-8}" +9,"{row-09,""esc1\"""",""esc2\\"",""esc3}""}" ,"{9,10}" ,"{1000009,-1000009}" ,"{5555500009,-5555500009}" ,"{9.0001,-9.0001}" ,"{3.14159265358979,-3.14159265358979,NULL}","{12345678900000.000009,NULL,-12345678900000.000009}","{NULL,t,f}", ,"{}" ,"{2013-07-13 21:00:05.009456}" ,"{NULL}" ,"{abc,cba}" ,"{defhi,jklmn}" ,"{b-9,c-9}" +10,"{row-10,""esc1\"""",""esc2\\"",""esc3}""}" ,"{10,11}" ,"{1000010,-1000010}" ,"{5555500010,-5555500010}" ,"{10.0001,-10.0001}" ,"{3.14159265358979,-3.14159265358979}" ,"{12345678900000.000010,-12345678900000.000010,NULL}","{t,NULL,f}","{NULL,2010-01-11,2010-02-11}", ,"{}" ,"{2013-07-13 21:00:05.010123-07}" ,"{NULL}" ,"{defhi,jklmn}" ,"{b-10,c-10}" +11,"{row-11,""esc1\"""",""esc2\\"",""esc3}""}" ,"{11,12}" ,"{1000011,-1000011}" ,"{5555500011,-5555500011}" ,"{11.0001,-11.0001}" ,"{3.14159265358979,-3.14159265358979}" ,"{12345678900000.000011,-12345678900000.000011}" ,"{t,f,NULL}","{2010-01-12,NULL,2010-02-12}","{NULL,10:11:11,11:11:11}", ,"{}" ,"{abc}" ,"{NULL}" ,"{b-11,c-11}" +12,"{row-12,""esc1\"""",""esc2\\"",""esc3}""}" ,"{12,13}" ,"{1000012,-1000012}" ,"{5555500012,-5555500012}" ,"{12.0001,-12.0001}" ,"{3.14159265358979,-3.14159265358979}" ,"{12345678900000.000012,-12345678900000.000012}" ,"{t,f}" ,"{2010-01-13,2010-02-13,NULL}","{10:11:12,NULL,11:11:12}","{NULL,2013-07-13 21:00:05.012456,2014-07-13 21:00:05.012456}", ,"{}" ,"{defhi}" ,"{NULL}" +13,"{NULL}" ,"{13,14}" ,"{1000013,-1000013}" ,"{5555500013,-5555500013}" ,"{13.0001,-13.0001}" ,"{3.14159265358979,-3.14159265358979}" ,"{12345678900000.000013,-12345678900000.000013}" ,"{t,f}" ,"{2010-01-14,2010-02-14}" ,"{10:11:13,11:11:13,NULL}","{2013-07-13 21:00:05.013456,NULL,2014-07-13 21:00:05.013456}","{NULL,2013-07-13 21:00:05.013123-07,2014-07-13 21:00:05.013123-07}", ,"{}" ,"{b-13}" +14,"{row-14}" ,"{NULL}" ,"{1000014,-1000014}" ,"{5555500014,-5555500014}" ,"{14.0001,-14.0001}" ,"{3.14159265358979,-3.14159265358979}" ,"{12345678900000.000014,-12345678900000.000014}" ,"{t,f}" ,"{2010-01-15,2010-02-15}" ,"{10:11:14,11:11:14}" ,"{2013-07-13 21:00:05.014456,2014-07-13 21:00:05.014456,NULL}","{2013-07-13 21:00:05.014123-07,NULL,2014-07-13 21:00:05.014123-07}","{NULL,abc,cba}", ,"{}" +15,"{}" ,"{15}" ,"{NULL}" ,"{5555500015,-5555500015}" ,"{15.0001,-15.0001}" ,"{3.14159265358979,-3.14159265358979}" ,"{12345678900000.000015,-12345678900000.000015}" ,"{t,f}" ,"{2010-01-16,2010-02-16}" ,"{10:11:15,11:11:15}" ,"{2013-07-13 21:00:05.015456,2014-07-13 21:00:05.015456}" ,"{2013-07-13 21:00:05.015123-07,2014-07-13 21:00:05.015123-07,NULL}","{abc,NULL,cba}","{NULL,defhi,jklmn}", diff --git a/automation/src/test/resources/data/json/primitive_types.csv b/automation/src/test/resources/data/json/primitive_types.csv new file mode 100644 index 0000000000..d8cf6a3883 --- /dev/null +++ b/automation/src/test/resources/data/json/primitive_types.csv @@ -0,0 +1,16 @@ +0,row-00,0,1000000,5555500000,0.0001,3.14159265358979,12345678900000.000000,f,2010-01-01,10:11:00,2013-07-13 21:00:05.000456,2013-07-13 21:00:05.000123-07,abc, def ,b-0 +1,,1,1000001,5555500001,1.0001,3.14159265358979,12345678900000.000001,t,2010-01-02,10:11:01,2013-07-13 21:00:05.001456,2013-07-13 21:00:05.001123-07,abc, def ,b-1 +2,row-02,,1000002,5555500002,2.0001,3.14159265358979,12345678900000.000002,f,2010-01-03,10:11:02,2013-07-13 21:00:05.002456,2013-07-13 21:00:05.002123-07,abc, def ,b-2 +3,row-03,3,,5555500003,3.0001,3.14159265358979,12345678900000.000003,t,2010-01-04,10:11:03,2013-07-13 21:00:05.003456,2013-07-13 21:00:05.003123-07,abc, def ,b-3 +4,row-04,4,1000004,,4.0001,3.14159265358979,12345678900000.000004,f,2010-01-05,10:11:04,2013-07-13 21:00:05.004456,2013-07-13 21:00:05.004123-07,abc, def ,b-4 +5,row-05,5,1000005,5555500005,,3.14159265358979,12345678900000.000005,t,2010-01-06,10:11:05,2013-07-13 21:00:05.005456,2013-07-13 21:00:05.005123-07,abc, def ,b-5 +6,row-06,6,1000006,5555500006,6.0001,,12345678900000.000006,f,2010-01-07,10:11:06,2013-07-13 21:00:05.006456,2013-07-13 21:00:05.006123-07,abc, def ,b-6 +7,row-07,7,1000007,5555500007,7.0001,3.14159265358979,,t,2010-01-08,10:11:07,2013-07-13 21:00:05.007456,2013-07-13 21:00:05.007123-07,abc, def ,b-7 +8,row-08,8,1000008,5555500008,8.0001,3.14159265358979,12345678900000.000008,,2010-01-09,10:11:08,2013-07-13 21:00:05.008456,2013-07-13 21:00:05.008123-07,abc, def ,b-8 +9,row-09,9,1000009,5555500009,9.0001,3.14159265358979,12345678900000.000009,t,,10:11:09,2013-07-13 21:00:05.009456,2013-07-13 21:00:05.009123-07,abc, def ,b-9 +10,row-10,10,1000010,5555500010,10.0001,3.14159265358979,12345678900000.0000010,f,2010-01-11,,2013-07-13 21:00:05.010456,2013-07-13 21:00:05.010123-07,abc, def ,b-10 +11,row-11,11,1000011,5555500011,11.0001,3.14159265358979,12345678900000.0000011,t,2010-01-12,10:11:11,,2013-07-13 21:00:05.011123-07,abc, def ,b-11 +12,row-12,12,1000012,5555500012,12.0001,3.14159265358979,12345678900000.0000012,f,2010-01-13,10:11:12,2013-07-13 21:00:05.012456,,abc, def ,b-12 +13,row-13,13,1000013,5555500013,13.0001,3.14159265358979,12345678900000.0000013,t,2010-01-14,10:11:13,2013-07-13 21:00:05.013456,2013-07-13 21:00:05.013123-07,, def ,b-13 +14,row-14,14,1000014,5555500014,14.0001,3.14159265358979,12345678900000.0000014,f,2010-01-15,10:11:14,2013-07-13 21:00:05.014456,2013-07-13 21:00:05.014123-07,abc,,b-14 +15,row-15,15,1000015,5555500015,15.0001,3.14159265358979,12345678900000.0000015,t,2010-01-16,10:11:15,2013-07-13 21:00:05.015456,2013-07-13 21:00:05.015123-07,abc, def , diff --git a/automation/tinc/main/tinctest/lib/global_init_file b/automation/tinc/main/tinctest/lib/global_init_file index dd5f73311b..d193dd5338 100755 --- a/automation/tinc/main/tinctest/lib/global_init_file +++ b/automation/tinc/main/tinctest/lib/global_init_file @@ -67,6 +67,10 @@ s/FOREIGN TABLE/EXTERNAL TABLE/ m/, record \d+ of/ s/, record/, line/ +# rename resource (FDW) to file (exttable) +m/, resource/ +s/, resource/, file/ + # syntax mismatch error is different between exttable with GP6 and fdw with GP7 m/invalid input syntax for type/ s/invalid input syntax for type/invalid input syntax for/ diff --git a/automation/tincrepo/main/pxf/features/cloud_access/no_server_credentials_no_config_with_hdfs/expected/query01.ans b/automation/tincrepo/main/pxf/features/cloud_access/no_server_credentials_no_config_with_hdfs/expected/query01.ans index 389e183787..f0f8fcc390 100755 --- a/automation/tincrepo/main/pxf/features/cloud_access/no_server_credentials_no_config_with_hdfs/expected/query01.ans +++ b/automation/tincrepo/main/pxf/features/cloud_access/no_server_credentials_no_config_with_hdfs/expected/query01.ans @@ -18,6 +18,12 @@ -- m/pxf:\/\/(.*)\/pxf_automation_data/ -- s/pxf:\/\/.*PROFILE=s3:text&.*/pxf:\/\/pxf_automation_data?PROFILE=s3:text&ACCESS_AND_SECRET_KEY/ -- +-- m/default_s3/ +-- s/default_s3/default/ +-- +-- m/, file.*pxf_automation_data/ +-- s/, file.*pxf_automation_data.*/pxf_automation_data/ +-- -- end_matchsubs SELECT * FROM cloudaccess_no_server_credentials_no_config_with_hdfs; ERROR: PXF server error : profile 's3a' is not compatible with server's 'default' configuration ('hdfs') diff --git a/automation/tincrepo/main/pxf/features/cloud_access/no_server_credentials_no_config_with_hdfs/sql/query01.sql b/automation/tincrepo/main/pxf/features/cloud_access/no_server_credentials_no_config_with_hdfs/sql/query01.sql index 1aba89b577..ffa82957c9 100755 --- a/automation/tincrepo/main/pxf/features/cloud_access/no_server_credentials_no_config_with_hdfs/sql/query01.sql +++ b/automation/tincrepo/main/pxf/features/cloud_access/no_server_credentials_no_config_with_hdfs/sql/query01.sql @@ -18,6 +18,12 @@ -- m/pxf:\/\/(.*)\/pxf_automation_data/ -- s/pxf:\/\/.*PROFILE=s3:text&.*/pxf:\/\/pxf_automation_data?PROFILE=s3:text&ACCESS_AND_SECRET_KEY/ -- +-- m/default_s3/ +-- s/default_s3/default/ +-- +-- m/, file.*pxf_automation_data/ +-- s/, file.*pxf_automation_data.*/pxf_automation_data/ +-- -- end_matchsubs SELECT * FROM cloudaccess_no_server_credentials_no_config_with_hdfs; diff --git a/automation/tincrepo/main/pxf/features/cloud_access/no_server_no_credentials/expected/query01.ans b/automation/tincrepo/main/pxf/features/cloud_access/no_server_no_credentials/expected/query01.ans index 3f1eff778e..561c662366 100755 --- a/automation/tincrepo/main/pxf/features/cloud_access/no_server_no_credentials/expected/query01.ans +++ b/automation/tincrepo/main/pxf/features/cloud_access/no_server_no_credentials/expected/query01.ans @@ -12,16 +12,16 @@ -- m/DETAIL/ -- s/DETAIL/CONTEXT/ -- --- m/pxf:\/\/(.*)\/pxf_automation_data/ --- s/pxf:\/\/.*PROFILE/pxf:\/\/pxf_automation_data?PROFILE/ --- -- m/CONTEXT:.*line.*/ -- s/line \d* of //g -- +-- m/, file.*pxf_automation_data/ +-- s/, file.*pxf_automation_data.*/pxf_automation_data/ +-- -- end_matchsubs SELECT * FROM cloudaccess_no_server_no_credentials; ERROR: PXF server error : com.amazonaws.AmazonClientException: No AWS Credentials provided by BasicAWSCredentialsProvider -- start_ignore HINT: Check the PXF logs located in the 'logs-dir' directory on host 'mdw' or 'set client_min_messages=LOG' for additional details. -- end_ignore -DETAIL: External table cloudaccess_no_server_no_credentials, file pxf://pxf_automation_data?PROFILE=s3:text +DETAIL: External table cloudaccess_no_server_no_credentials, file pxf_automation_data diff --git a/automation/tincrepo/main/pxf/features/cloud_access/no_server_no_credentials/sql/query01.sql b/automation/tincrepo/main/pxf/features/cloud_access/no_server_no_credentials/sql/query01.sql index b720de7dd7..039c85b519 100755 --- a/automation/tincrepo/main/pxf/features/cloud_access/no_server_no_credentials/sql/query01.sql +++ b/automation/tincrepo/main/pxf/features/cloud_access/no_server_no_credentials/sql/query01.sql @@ -12,12 +12,12 @@ -- m/DETAIL/ -- s/DETAIL/CONTEXT/ -- --- m/pxf:\/\/(.*)\/pxf_automation_data/ --- s/pxf:\/\/.*PROFILE/pxf:\/\/pxf_automation_data?PROFILE/ --- -- m/CONTEXT:.*line.*/ -- s/line \d* of //g -- +-- m/, file.*pxf_automation_data/ +-- s/, file.*pxf_automation_data.*/pxf_automation_data/ +-- -- end_matchsubs SELECT * FROM cloudaccess_no_server_no_credentials; diff --git a/automation/tincrepo/main/pxf/features/cloud_access/no_server_no_credentials_with_hdfs/expected/query01.ans b/automation/tincrepo/main/pxf/features/cloud_access/no_server_no_credentials_with_hdfs/expected/query01.ans index 27a1b220f3..60420c3240 100755 --- a/automation/tincrepo/main/pxf/features/cloud_access/no_server_no_credentials_with_hdfs/expected/query01.ans +++ b/automation/tincrepo/main/pxf/features/cloud_access/no_server_no_credentials_with_hdfs/expected/query01.ans @@ -18,10 +18,16 @@ -- m/CONTEXT:.*line.*/ -- s/line \d* of //g -- +-- m/default_s3/ +-- s/default_s3/default/ +-- +-- m/, file.*pxf_automation_data/ +-- s/, file.*pxf_automation_data.*/pxf_automation_data/ +-- -- end_matchsubs SELECT * FROM cloudaccess_no_server_no_credentials_with_hdfs; ERROR: PXF server error : profile 's3a' is not compatible with server's 'default' configuration ('hdfs') -- start_ignore HINT: Ensure that 'server-dir' includes only the configuration files for profile 's3a'. Check the PXF logs located in the 'logs-dir' directory on host 'mdw' or 'set client_min_messages=LOG' for additional details. -- end_ignore -DETAIL: External table cloudaccess_no_server_no_credentials_with_hdfs, file pxf://pxf_automation_data?PROFILE=s3:text +DETAIL: External table cloudaccess_no_server_no_credentials_with_hdfs, file pxf_automation_data diff --git a/automation/tincrepo/main/pxf/features/cloud_access/no_server_no_credentials_with_hdfs/sql/query01.sql b/automation/tincrepo/main/pxf/features/cloud_access/no_server_no_credentials_with_hdfs/sql/query01.sql index 84ae63bfca..0934723946 100755 --- a/automation/tincrepo/main/pxf/features/cloud_access/no_server_no_credentials_with_hdfs/sql/query01.sql +++ b/automation/tincrepo/main/pxf/features/cloud_access/no_server_no_credentials_with_hdfs/sql/query01.sql @@ -18,6 +18,12 @@ -- m/CONTEXT:.*line.*/ -- s/line \d* of //g -- +-- m/default_s3/ +-- s/default_s3/default/ +-- +-- m/, file.*pxf_automation_data/ +-- s/, file.*pxf_automation_data.*/pxf_automation_data/ +-- -- end_matchsubs SELECT * FROM cloudaccess_no_server_no_credentials_with_hdfs; diff --git a/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_invalid_config/expected/query01.ans b/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_invalid_config/expected/query01.ans index 489828a064..4d5dcfb2b3 100755 --- a/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_invalid_config/expected/query01.ans +++ b/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_invalid_config/expected/query01.ans @@ -18,10 +18,13 @@ -- m/CONTEXT:.*line.*/ -- s/line \d* of //g -- +-- m/, file.*pxf_automation_data/ +-- s/, file.*pxf_automation_data.*/pxf_automation_data/ +-- -- end_matchsubs SELECT * FROM cloudaccess_server_no_credentials_invalid_config; ERROR: PXF server error : com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden -- start_ignore HINT: Check the PXF logs located in the 'logs-dir' directory on host 'mdw' or 'set client_min_messages=LOG' for additional details. -- end_ignore -DETAIL: External table cloudaccess_server_no_credentials_invalid_config, file pxf://pxf_automation_data?PROFILE=s3:text&server=s3-invalid +DETAIL: External table cloudaccess_server_no_credentials_invalid_config, file pxf_automation_data diff --git a/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_invalid_config/sql/query01.sql b/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_invalid_config/sql/query01.sql index d0980b7fea..1ff81a6793 100755 --- a/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_invalid_config/sql/query01.sql +++ b/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_invalid_config/sql/query01.sql @@ -18,6 +18,9 @@ -- m/CONTEXT:.*line.*/ -- s/line \d* of //g -- +-- m/, file.*pxf_automation_data/ +-- s/, file.*pxf_automation_data.*/pxf_automation_data/ +-- -- end_matchsubs SELECT * FROM cloudaccess_server_no_credentials_invalid_config; diff --git a/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_invalid_config_with_hdfs/expected/query01.ans b/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_invalid_config_with_hdfs/expected/query01.ans index 5b680248fb..92428d9a06 100755 --- a/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_invalid_config_with_hdfs/expected/query01.ans +++ b/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_invalid_config_with_hdfs/expected/query01.ans @@ -18,10 +18,13 @@ -- m/CONTEXT:.*line.*/ -- s/line \d* of //g -- +-- m/, file.*pxf_automation_data/ +-- s/, file.*pxf_automation_data.*/pxf_automation_data/ +-- -- end_matchsubs SELECT * FROM cloudaccess_server_no_credentials_invalid_config_with_hdfs; ERROR: PXF server error : com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden -- start_ignore HINT: Check the PXF logs located in the 'logs-dir' directory on host 'mdw' or 'set client_min_messages=LOG' for additional details. -- end_ignore -DETAIL: External table cloudaccess_server_no_credentials_invalid_config_with_hdfs, file pxf://pxf_automation_data?PROFILE=s3:text&server=s3-invalid +DETAIL: External table cloudaccess_server_no_credentials_invalid_config_with_hdfs, file pxf_automation_data diff --git a/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_invalid_config_with_hdfs/sql/query01.sql b/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_invalid_config_with_hdfs/sql/query01.sql index 17fa7b76ab..f28f4e1838 100755 --- a/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_invalid_config_with_hdfs/sql/query01.sql +++ b/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_invalid_config_with_hdfs/sql/query01.sql @@ -18,6 +18,9 @@ -- m/CONTEXT:.*line.*/ -- s/line \d* of //g -- +-- m/, file.*pxf_automation_data/ +-- s/, file.*pxf_automation_data.*/pxf_automation_data/ +-- -- end_matchsubs SELECT * FROM cloudaccess_server_no_credentials_invalid_config_with_hdfs; diff --git a/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_no_config/expected/query01.ans b/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_no_config/expected/query01.ans index 13f8472d68..2c3c2ac1b6 100755 --- a/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_no_config/expected/query01.ans +++ b/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_no_config/expected/query01.ans @@ -18,10 +18,13 @@ -- m/CONTEXT:.*line.*/ -- s/line \d* of //g -- +-- m/, file.*pxf_automation_data/ +-- s/, file.*pxf_automation_data.*/pxf_automation_data/ +-- -- end_matchsubs SELECT * FROM cloudaccess_server_no_credentials_no_config; ERROR: PXF server error : com.amazonaws.AmazonClientException: No AWS Credentials provided by BasicAWSCredentialsProvider -- start_ignore HINT: Check the PXF logs located in the 'logs-dir' directory on host 'mdw' or 'set client_min_messages=LOG' for additional details. -- end_ignore -DETAIL: External table cloudaccess_server_no_credentials_no_config, file pxf://pxf_automation_data?PROFILE=s3:text&server=s3-non-existent +DETAIL: External table cloudaccess_server_no_credentials_no_config, file pxf_automation_data diff --git a/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_no_config/sql/query01.sql b/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_no_config/sql/query01.sql index 7ee6ffa84e..fe31a10e67 100755 --- a/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_no_config/sql/query01.sql +++ b/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_no_config/sql/query01.sql @@ -18,6 +18,9 @@ -- m/CONTEXT:.*line.*/ -- s/line \d* of //g -- +-- m/, file.*pxf_automation_data/ +-- s/, file.*pxf_automation_data.*/pxf_automation_data/ +-- -- end_matchsubs SELECT * FROM cloudaccess_server_no_credentials_no_config; diff --git a/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_no_config_with_hdfs/expected/query01.ans b/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_no_config_with_hdfs/expected/query01.ans index 99f16b75ea..0af50070b0 100755 --- a/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_no_config_with_hdfs/expected/query01.ans +++ b/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_no_config_with_hdfs/expected/query01.ans @@ -21,10 +21,13 @@ -- m/CONTEXT:.*line.*/ -- s/line \d* of //g -- +-- m/, file.*pxf_automation_data/ +-- s/, file.*pxf_automation_data.*/pxf_automation_data/ +-- -- end_matchsubs SELECT * FROM cloudaccess_server_no_credentials_no_config_with_hdfs; ERROR: PXF server error : com.amazonaws.AmazonClientException: No AWS Credentials provided by BasicAWSCredentialsProvider -- start_ignore HINT: Check the PXF logs located in the 'logs-dir' directory on host 'mdw' or 'set client_min_messages=LOG' for additional details. -- end_ignore -DETAIL: External table cloudaccess_server_no_credentials_no_config_with_hdfs, file pxf://pxf_automation_data?PROFILE=s3:text&server=s3-non-existent +DETAIL: External table cloudaccess_server_no_credentials_no_config_with_hdfs, file pxf_automation_data diff --git a/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_no_config_with_hdfs/sql/query01.sql b/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_no_config_with_hdfs/sql/query01.sql index c2d2a92b7a..9d09b48626 100755 --- a/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_no_config_with_hdfs/sql/query01.sql +++ b/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_no_config_with_hdfs/sql/query01.sql @@ -21,6 +21,9 @@ -- m/CONTEXT:.*line.*/ -- s/line \d* of //g -- +-- m/, file.*pxf_automation_data/ +-- s/, file.*pxf_automation_data.*/pxf_automation_data/ +-- -- end_matchsubs SELECT * FROM cloudaccess_server_no_credentials_no_config_with_hdfs; diff --git a/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_valid_config_with_hdfs_write/expected/query01.ans b/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_valid_config_with_hdfs_write/expected/query01.ans index 44f4bfce41..1694a0a6c1 100755 --- a/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_valid_config_with_hdfs_write/expected/query01.ans +++ b/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_valid_config_with_hdfs_write/expected/query01.ans @@ -2,7 +2,14 @@ -- end_ignore -- @description query01 for PXF test for cloud write where server is specified, no credentials are specified, and configuration file exists running alongside an HDFS setup -- - +-- start_matchsubs +-- +-- # create a match/subs +-- +-- m/WARNING.*cannot analyze this foreign table/ +-- s/.*// +-- +-- end_matchsubs INSERT INTO cloudwrite_server_no_credentials_valid_config_with_hdfs_write SELECT md5(random()::text), round(random()*100) from generate_series(1,10); INSERT 0 10 diff --git a/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_valid_config_with_hdfs_write/sql/query01.sql b/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_valid_config_with_hdfs_write/sql/query01.sql index 31c7b1c111..8a734efdbf 100755 --- a/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_valid_config_with_hdfs_write/sql/query01.sql +++ b/automation/tincrepo/main/pxf/features/cloud_access/server_no_credentials_valid_config_with_hdfs_write/sql/query01.sql @@ -2,7 +2,14 @@ -- end_ignore -- @description query01 for PXF test for cloud write where server is specified, no credentials are specified, and configuration file exists running alongside an HDFS setup -- - +-- start_matchsubs +-- +-- # create a match/subs +-- +-- m/WARNING.*cannot analyze this foreign table/ +-- s/.*// +-- +-- end_matchsubs INSERT INTO cloudwrite_server_no_credentials_valid_config_with_hdfs_write SELECT md5(random()::text), round(random()*100) from generate_series(1,10); SELECT count(*) FROM cloudaccess_server_no_credentials_valid_config_with_hdfs_write; diff --git a/automation/tincrepo/main/pxf/features/gpupgrade/step_1_before_running_pxf_pre_gpupgrade/__init__.py b/automation/tincrepo/main/pxf/features/columnprojection/checkColumnProjection_fdw/__init__.py old mode 100644 new mode 100755 similarity index 100% rename from automation/tincrepo/main/pxf/features/gpupgrade/step_1_before_running_pxf_pre_gpupgrade/__init__.py rename to automation/tincrepo/main/pxf/features/columnprojection/checkColumnProjection_fdw/__init__.py diff --git a/automation/tincrepo/main/pxf/features/columnprojection/checkColumnProjection_fdw/expected/query01.ans b/automation/tincrepo/main/pxf/features/columnprojection/checkColumnProjection_fdw/expected/query01.ans new file mode 100755 index 0000000000..2c6b313141 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/columnprojection/checkColumnProjection_fdw/expected/query01.ans @@ -0,0 +1,358 @@ +-- start_ignore +-- end_ignore +-- start_ignore +DROP TABLE IF EXISTS t0_values; +DROP TABLE +CREATE TABLE t0_values(key char(1), value int) DISTRIBUTED BY (key); +CREATE TABLE +INSERT INTO t0_values VALUES('A', 50); +INSERT 0 1 +-- end_ignore +-- @description query01 for PXF Column Projection Support +SET optimizer = off; +SET +SELECT * FROM test_column_projection ORDER BY t0; + t0 | a1 | b2 | colprojvalue +----+----+----+---------------------- + A | 0 | t | No Column Projection + B | 1 | f | No Column Projection + C | 2 | t | No Column Projection + D | 3 | f | No Column Projection + E | 4 | t | No Column Projection + F | 5 | f | No Column Projection + G | 6 | t | No Column Projection + H | 7 | f | No Column Projection + I | 8 | t | No Column Projection + J | 9 | f | No Column Projection +(10 rows) + +SELECT t0, colprojvalue FROM test_column_projection ORDER BY t0; + t0 | colprojvalue +----+----------------- + A | t0|colprojvalue + B | t0|colprojvalue + C | t0|colprojvalue + D | t0|colprojvalue + E | t0|colprojvalue + F | t0|colprojvalue + G | t0|colprojvalue + H | t0|colprojvalue + I | t0|colprojvalue + J | t0|colprojvalue +(10 rows) + +SELECT colprojvalue FROM test_column_projection ORDER BY t0; + colprojvalue +----------------- + t0|colprojvalue + t0|colprojvalue + t0|colprojvalue + t0|colprojvalue + t0|colprojvalue + t0|colprojvalue + t0|colprojvalue + t0|colprojvalue + t0|colprojvalue + t0|colprojvalue +(10 rows) + +SELECT t0, colprojvalue FROM test_column_projection WHERE b2 ORDER BY t0; + t0 | colprojvalue +----+-------------------- + A | t0|b2|colprojvalue + C | t0|b2|colprojvalue + E | t0|b2|colprojvalue + G | t0|b2|colprojvalue + I | t0|b2|colprojvalue +(5 rows) + +SELECT t0, a1, colprojvalue FROM test_column_projection WHERE a1 < 5 AND b2 = false ORDER BY t0; + t0 | a1 | colprojvalue +----+----+----------------------- + B | 1 | t0|a1|b2|colprojvalue + D | 3 | t0|a1|b2|colprojvalue +(2 rows) + +SELECT round(sqrt(a1)::numeric, 5), colprojvalue FROM test_column_projection WHERE a1 < 5 AND b2 = false ORDER BY t0; + round | colprojvalue +---------+----------------------- + 1.00000 | t0|a1|b2|colprojvalue + 1.73205 | t0|a1|b2|colprojvalue +(2 rows) + +SELECT round(sqrt(a1)::numeric, 5), colprojvalue FROM test_column_projection WHERE b2 = false ORDER BY t0; + round | colprojvalue +---------+----------------------- + 1.00000 | t0|a1|b2|colprojvalue + 1.73205 | t0|a1|b2|colprojvalue + 2.23607 | t0|a1|b2|colprojvalue + 2.64575 | t0|a1|b2|colprojvalue + 3.00000 | t0|a1|b2|colprojvalue +(5 rows) + +SELECT t0, colprojvalue FROM test_column_projection WHERE a1 < 5 ORDER BY t0; + t0 | colprojvalue +----+-------------------- + A | t0|a1|colprojvalue + B | t0|a1|colprojvalue + C | t0|a1|colprojvalue + D | t0|a1|colprojvalue + E | t0|a1|colprojvalue +(5 rows) + +SELECT t0, colprojvalue FROM test_column_projection WHERE a1 <= 5 ORDER BY t0; + t0 | colprojvalue +----+-------------------- + A | t0|a1|colprojvalue + B | t0|a1|colprojvalue + C | t0|a1|colprojvalue + D | t0|a1|colprojvalue + E | t0|a1|colprojvalue + F | t0|a1|colprojvalue +(6 rows) + +SELECT t0, colprojvalue FROM test_column_projection GROUP BY t0, colprojvalue HAVING AVG(a1) < 5 ORDER BY t0; + t0 | colprojvalue +----+---------------------- + A | No Column Projection + B | No Column Projection + C | No Column Projection + D | No Column Projection + E | No Column Projection +(5 rows) + +SELECT b.value, a.colprojvalue FROM test_column_projection a JOIN t0_values b ON a.t0 = b.key; + value | colprojvalue +-------+----------------- + 50 | t0|colprojvalue +(1 row) + +SELECT t0, colprojvalue FROM test_column_projection WHERE a1 < 2 OR a1 >= 8 ORDER BY t0; + t0 | colprojvalue +----+-------------------- + A | t0|a1|colprojvalue + B | t0|a1|colprojvalue + I | t0|a1|colprojvalue + J | t0|a1|colprojvalue +(4 rows) + +SELECT t0, colprojvalue FROM test_column_projection WHERE sqrt(a1) > 1 ORDER BY t0; + t0 | colprojvalue +----+-------------------- + C | t0|a1|colprojvalue + D | t0|a1|colprojvalue + E | t0|a1|colprojvalue + F | t0|a1|colprojvalue + G | t0|a1|colprojvalue + H | t0|a1|colprojvalue + I | t0|a1|colprojvalue + J | t0|a1|colprojvalue +(8 rows) + +SELECT t0, colprojvalue, round(sqrt(a1)::numeric, 5) FROM test_column_projection ORDER BY t0; + t0 | colprojvalue | round +----+--------------------+--------- + A | t0|a1|colprojvalue | 0.00000 + B | t0|a1|colprojvalue | 1.00000 + C | t0|a1|colprojvalue | 1.41421 + D | t0|a1|colprojvalue | 1.73205 + E | t0|a1|colprojvalue | 2.00000 + F | t0|a1|colprojvalue | 2.23607 + G | t0|a1|colprojvalue | 2.44949 + H | t0|a1|colprojvalue | 2.64575 + I | t0|a1|colprojvalue | 2.82843 + J | t0|a1|colprojvalue | 3.00000 +(10 rows) + +-- Casting boolean column to int +SELECT t0, colprojvalue, sqrt(b2::int) FROM test_column_projection ORDER BY t0; + t0 | colprojvalue | sqrt +----+--------------------+------ + A | t0|b2|colprojvalue | 1 + B | t0|b2|colprojvalue | 0 + C | t0|b2|colprojvalue | 1 + D | t0|b2|colprojvalue | 0 + E | t0|b2|colprojvalue | 1 + F | t0|b2|colprojvalue | 0 + G | t0|b2|colprojvalue | 1 + H | t0|b2|colprojvalue | 0 + I | t0|b2|colprojvalue | 1 + J | t0|b2|colprojvalue | 0 +(10 rows) + +SET optimizer = on; +SET +SELECT * FROM test_column_projection ORDER BY t0; + t0 | a1 | b2 | colprojvalue +----+----+----+---------------------- + A | 0 | t | No Column Projection + B | 1 | f | No Column Projection + C | 2 | t | No Column Projection + D | 3 | f | No Column Projection + E | 4 | t | No Column Projection + F | 5 | f | No Column Projection + G | 6 | t | No Column Projection + H | 7 | f | No Column Projection + I | 8 | t | No Column Projection + J | 9 | f | No Column Projection +(10 rows) + +SELECT t0, colprojvalue FROM test_column_projection ORDER BY t0; + t0 | colprojvalue +----+----------------- + A | t0|colprojvalue + B | t0|colprojvalue + C | t0|colprojvalue + D | t0|colprojvalue + E | t0|colprojvalue + F | t0|colprojvalue + G | t0|colprojvalue + H | t0|colprojvalue + I | t0|colprojvalue + J | t0|colprojvalue +(10 rows) + +SELECT colprojvalue FROM test_column_projection ORDER BY t0; + colprojvalue +----------------- + t0|colprojvalue + t0|colprojvalue + t0|colprojvalue + t0|colprojvalue + t0|colprojvalue + t0|colprojvalue + t0|colprojvalue + t0|colprojvalue + t0|colprojvalue + t0|colprojvalue +(10 rows) + +SELECT t0, colprojvalue FROM test_column_projection WHERE b2 ORDER BY t0; + t0 | colprojvalue +----+-------------------- + A | t0|b2|colprojvalue + C | t0|b2|colprojvalue + E | t0|b2|colprojvalue + G | t0|b2|colprojvalue + I | t0|b2|colprojvalue +(5 rows) + +SELECT t0, a1, colprojvalue FROM test_column_projection WHERE a1 < 5 AND b2 = false ORDER BY t0; + t0 | a1 | colprojvalue +----+----+----------------------- + B | 1 | t0|a1|b2|colprojvalue + D | 3 | t0|a1|b2|colprojvalue +(2 rows) + +SELECT round(sqrt(a1)::numeric, 5), colprojvalue FROM test_column_projection WHERE a1 < 5 AND b2 = false ORDER BY t0; + round | colprojvalue +---------+----------------------- + 1.00000 | t0|a1|b2|colprojvalue + 1.73205 | t0|a1|b2|colprojvalue +(2 rows) + +SELECT round(sqrt(a1)::numeric, 5), colprojvalue FROM test_column_projection WHERE b2 = false ORDER BY t0; + round | colprojvalue +---------+----------------------- + 1.00000 | t0|a1|b2|colprojvalue + 1.73205 | t0|a1|b2|colprojvalue + 2.23607 | t0|a1|b2|colprojvalue + 2.64575 | t0|a1|b2|colprojvalue + 3.00000 | t0|a1|b2|colprojvalue +(5 rows) + +SELECT t0, colprojvalue FROM test_column_projection WHERE a1 < 5 ORDER BY t0; + t0 | colprojvalue +----+-------------------- + A | t0|a1|colprojvalue + B | t0|a1|colprojvalue + C | t0|a1|colprojvalue + D | t0|a1|colprojvalue + E | t0|a1|colprojvalue +(5 rows) + +SELECT t0, colprojvalue FROM test_column_projection WHERE a1 <= 5 ORDER BY t0; + t0 | colprojvalue +----+-------------------- + A | t0|a1|colprojvalue + B | t0|a1|colprojvalue + C | t0|a1|colprojvalue + D | t0|a1|colprojvalue + E | t0|a1|colprojvalue + F | t0|a1|colprojvalue +(6 rows) + +SELECT t0, colprojvalue FROM test_column_projection GROUP BY t0, colprojvalue HAVING AVG(a1) < 5 ORDER BY t0; + t0 | colprojvalue +----+-------------------- + A | t0|a1|colprojvalue + B | t0|a1|colprojvalue + C | t0|a1|colprojvalue + D | t0|a1|colprojvalue + E | t0|a1|colprojvalue +(5 rows) + +SELECT b.value, a.colprojvalue FROM test_column_projection a JOIN t0_values b ON a.t0 = b.key; + value | colprojvalue +-------+----------------- + 50 | t0|colprojvalue +(1 row) + +SELECT t0, colprojvalue FROM test_column_projection WHERE a1 < 2 OR a1 >= 8 ORDER BY t0; + t0 | colprojvalue +----+-------------------- + A | t0|a1|colprojvalue + B | t0|a1|colprojvalue + I | t0|a1|colprojvalue + J | t0|a1|colprojvalue +(4 rows) + +SELECT t0, colprojvalue FROM test_column_projection WHERE sqrt(a1) > 1 ORDER BY t0; + t0 | colprojvalue +----+-------------------- + C | t0|a1|colprojvalue + D | t0|a1|colprojvalue + E | t0|a1|colprojvalue + F | t0|a1|colprojvalue + G | t0|a1|colprojvalue + H | t0|a1|colprojvalue + I | t0|a1|colprojvalue + J | t0|a1|colprojvalue +(8 rows) + +SELECT t0, colprojvalue, round(sqrt(a1)::numeric, 5) FROM test_column_projection ORDER BY t0; + t0 | colprojvalue | round +----+--------------------+--------- + A | t0|a1|colprojvalue | 0.00000 + B | t0|a1|colprojvalue | 1.00000 + C | t0|a1|colprojvalue | 1.41421 + D | t0|a1|colprojvalue | 1.73205 + E | t0|a1|colprojvalue | 2.00000 + F | t0|a1|colprojvalue | 2.23607 + G | t0|a1|colprojvalue | 2.44949 + H | t0|a1|colprojvalue | 2.64575 + I | t0|a1|colprojvalue | 2.82843 + J | t0|a1|colprojvalue | 3.00000 +(10 rows) + +-- Casting boolean column to int +SELECT t0, colprojvalue, sqrt(b2::int) FROM test_column_projection ORDER BY t0; + t0 | colprojvalue | sqrt +----+--------------------+------ + A | t0|b2|colprojvalue | 1 + B | t0|b2|colprojvalue | 0 + C | t0|b2|colprojvalue | 1 + D | t0|b2|colprojvalue | 0 + E | t0|b2|colprojvalue | 1 + F | t0|b2|colprojvalue | 0 + G | t0|b2|colprojvalue | 1 + H | t0|b2|colprojvalue | 0 + I | t0|b2|colprojvalue | 1 + J | t0|b2|colprojvalue | 0 +(10 rows) + +-- cleanup +-- start_ignore +DROP TABLE IF EXISTS t0_values; +DROP TABLE +-- end_ignore diff --git a/automation/tincrepo/main/pxf/features/columnprojection/checkColumnProjection_fdw/runTest.py b/automation/tincrepo/main/pxf/features/columnprojection/checkColumnProjection_fdw/runTest.py new file mode 100755 index 0000000000..310814b6ea --- /dev/null +++ b/automation/tincrepo/main/pxf/features/columnprojection/checkColumnProjection_fdw/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfColumnProjection(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/columnprojection/checkColumnProjection_fdw/sql/query01.sql b/automation/tincrepo/main/pxf/features/columnprojection/checkColumnProjection_fdw/sql/query01.sql new file mode 100755 index 0000000000..eb833658a6 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/columnprojection/checkColumnProjection_fdw/sql/query01.sql @@ -0,0 +1,77 @@ +-- start_ignore +DROP TABLE IF EXISTS t0_values; +CREATE TABLE t0_values(key char(1), value int) DISTRIBUTED BY (key); +INSERT INTO t0_values VALUES('A', 50); +-- end_ignore +-- @description query01 for PXF Column Projection Support + +SET optimizer = off; + +SELECT * FROM test_column_projection ORDER BY t0; + +SELECT t0, colprojvalue FROM test_column_projection ORDER BY t0; + +SELECT colprojvalue FROM test_column_projection ORDER BY t0; + +SELECT t0, colprojvalue FROM test_column_projection WHERE b2 ORDER BY t0; + +SELECT t0, a1, colprojvalue FROM test_column_projection WHERE a1 < 5 AND b2 = false ORDER BY t0; + +SELECT round(sqrt(a1)::numeric, 5), colprojvalue FROM test_column_projection WHERE a1 < 5 AND b2 = false ORDER BY t0; + +SELECT round(sqrt(a1)::numeric, 5), colprojvalue FROM test_column_projection WHERE b2 = false ORDER BY t0; + +SELECT t0, colprojvalue FROM test_column_projection WHERE a1 < 5 ORDER BY t0; + +SELECT t0, colprojvalue FROM test_column_projection WHERE a1 <= 5 ORDER BY t0; + +SELECT t0, colprojvalue FROM test_column_projection GROUP BY t0, colprojvalue HAVING AVG(a1) < 5 ORDER BY t0; + +SELECT b.value, a.colprojvalue FROM test_column_projection a JOIN t0_values b ON a.t0 = b.key; + +SELECT t0, colprojvalue FROM test_column_projection WHERE a1 < 2 OR a1 >= 8 ORDER BY t0; + +SELECT t0, colprojvalue FROM test_column_projection WHERE sqrt(a1) > 1 ORDER BY t0; + +SELECT t0, colprojvalue, round(sqrt(a1)::numeric, 5) FROM test_column_projection ORDER BY t0; + +-- Casting boolean column to int +SELECT t0, colprojvalue, sqrt(b2::int) FROM test_column_projection ORDER BY t0; + +SET optimizer = on; + +SELECT * FROM test_column_projection ORDER BY t0; + +SELECT t0, colprojvalue FROM test_column_projection ORDER BY t0; + +SELECT colprojvalue FROM test_column_projection ORDER BY t0; + +SELECT t0, colprojvalue FROM test_column_projection WHERE b2 ORDER BY t0; + +SELECT t0, a1, colprojvalue FROM test_column_projection WHERE a1 < 5 AND b2 = false ORDER BY t0; + +SELECT round(sqrt(a1)::numeric, 5), colprojvalue FROM test_column_projection WHERE a1 < 5 AND b2 = false ORDER BY t0; + +SELECT round(sqrt(a1)::numeric, 5), colprojvalue FROM test_column_projection WHERE b2 = false ORDER BY t0; + +SELECT t0, colprojvalue FROM test_column_projection WHERE a1 < 5 ORDER BY t0; + +SELECT t0, colprojvalue FROM test_column_projection WHERE a1 <= 5 ORDER BY t0; + +SELECT t0, colprojvalue FROM test_column_projection GROUP BY t0, colprojvalue HAVING AVG(a1) < 5 ORDER BY t0; + +SELECT b.value, a.colprojvalue FROM test_column_projection a JOIN t0_values b ON a.t0 = b.key; + +SELECT t0, colprojvalue FROM test_column_projection WHERE a1 < 2 OR a1 >= 8 ORDER BY t0; + +SELECT t0, colprojvalue FROM test_column_projection WHERE sqrt(a1) > 1 ORDER BY t0; + +SELECT t0, colprojvalue, round(sqrt(a1)::numeric, 5) FROM test_column_projection ORDER BY t0; + +-- Casting boolean column to int +SELECT t0, colprojvalue, sqrt(b2::int) FROM test_column_projection ORDER BY t0; + +-- cleanup +-- start_ignore +DROP TABLE IF EXISTS t0_values; +-- end_ignore diff --git a/automation/tincrepo/main/pxf/features/gpupgrade/step_2_after_running_pxf_pre_gpupgrade/__init__.py b/automation/tincrepo/main/pxf/features/extension_tests/__init__.py similarity index 100% rename from automation/tincrepo/main/pxf/features/gpupgrade/step_2_after_running_pxf_pre_gpupgrade/__init__.py rename to automation/tincrepo/main/pxf/features/extension_tests/__init__.py diff --git a/automation/tincrepo/main/pxf/features/gpupgrade/step_3_after_running_pxf_post_gpupgrade/__init__.py b/automation/tincrepo/main/pxf/features/extension_tests/create_extension/__init__.py similarity index 100% rename from automation/tincrepo/main/pxf/features/gpupgrade/step_3_after_running_pxf_post_gpupgrade/__init__.py rename to automation/tincrepo/main/pxf/features/extension_tests/create_extension/__init__.py diff --git a/automation/tincrepo/main/pxf/features/extension_tests/create_extension/expected/query01.ans b/automation/tincrepo/main/pxf/features/extension_tests/create_extension/expected/query01.ans new file mode 100644 index 0000000000..c6553ee2e3 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/extension_tests/create_extension/expected/query01.ans @@ -0,0 +1,72 @@ +-- start_ignore +-- end_ignore +-- @description query01 for PXF install test on small data +-- start_matchsubs +-- +-- m{.*/usr/local/pxf-(dev|gp\d).*} +-- s{/usr/local/pxf-(dev|gp\d)}{\$PXF_HOME} +-- +-- m{.*\$libdir/pxf.*} +-- s{\$libdir}{\$PXF_HOME/gpextable} +-- +-- end_matchsubs +-- start_ignore +\c pxfautomation_extension +-- end_ignore + +SELECT extversion FROM pg_extension WHERE extname = 'pxf'; + extversion +------------ + 2.1 +(1 row) + +SHOW dynamic_library_path; + dynamic_library_path +---------------------- + $libdir +(1 row) +SELECT p.proname, p.prosrc, p.probin +FROM pg_catalog.pg_extension AS e + INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) + INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) +WHERE d.deptype = 'e' AND e.extname = 'pxf' +ORDER BY 1; + proname | prosrc | probin +---------------------+------------------------------+---------------------------------- + pxf_read | pxfprotocol_import | $PXF_HOME/gpextable/pxf + pxf_validate | pxfprotocol_validate_urls | $PXF_HOME/gpextable/pxf + pxf_write | pxfprotocol_export | $PXF_HOME/gpextable/pxf + pxfdelimited_import | pxfdelimited_import | $PXF_HOME/gpextable/pxf + pxfwritable_export | gpdbwritableformatter_export | $PXF_HOME/gpextable/pxf + pxfwritable_import | gpdbwritableformatter_import | $PXF_HOME/gpextable/pxf +(6 rows) + +SELECT * FROM pxf_upgrade_test ORDER BY num; + name | num | dub | longnum | bool +--------+-----+-----+---------------+------ + row_1 | 1 | 1 | 100000000000 | f + row_2 | 2 | 2 | 200000000000 | t + row_3 | 3 | 3 | 300000000000 | f + row_4 | 4 | 4 | 400000000000 | t + row_5 | 5 | 5 | 500000000000 | f + row_6 | 6 | 6 | 600000000000 | t + row_7 | 7 | 7 | 700000000000 | f + row_8 | 8 | 8 | 800000000000 | t + row_9 | 9 | 9 | 900000000000 | f + row_10 | 10 | 10 | 1000000000000 | t +(10 rows) + +SELECT * FROM pxf_upgrade_test_multibyte ORDER BY num; + name | num | dub | longnum | bool +--------+-----+-----+---------------+------ + row_1 | 1 | 1 | 100000000000 | f + row_2 | 2 | 2 | 200000000000 | t + row_3 | 3 | 3 | 300000000000 | f + row_4 | 4 | 4 | 400000000000 | t + row_5 | 5 | 5 | 500000000000 | f + row_6 | 6 | 6 | 600000000000 | t + row_7 | 7 | 7 | 700000000000 | f + row_8 | 8 | 8 | 800000000000 | t + row_9 | 9 | 9 | 900000000000 | f + row_10 | 10 | 10 | 1000000000000 | t +(10 rows) diff --git a/automation/tincrepo/main/pxf/features/extension_tests/create_extension/runTest.py b/automation/tincrepo/main/pxf/features/extension_tests/create_extension/runTest.py new file mode 100644 index 0000000000..2a6aba5073 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/extension_tests/create_extension/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfCreateExtension(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/extension_tests/create_extension/sql/query01.sql b/automation/tincrepo/main/pxf/features/extension_tests/create_extension/sql/query01.sql new file mode 100644 index 0000000000..c9962dffec --- /dev/null +++ b/automation/tincrepo/main/pxf/features/extension_tests/create_extension/sql/query01.sql @@ -0,0 +1,28 @@ +-- @description query01 for PXF install test on small data +-- start_matchsubs +-- +-- m{.*/usr/local/pxf-(dev|gp\d).*} +-- s{/usr/local/pxf-(dev|gp\d)}{\$PXF_HOME} +-- +-- m{.*\$libdir/pxf.*} +-- s{\$libdir}{\$PXF_HOME/gpextable} +-- +-- end_matchsubs +-- start_ignore +\c pxfautomation_extension +-- end_ignore + +SELECT extversion FROM pg_extension WHERE extname = 'pxf'; + +SHOW dynamic_library_path; + +SELECT p.proname, p.prosrc, p.probin +FROM pg_catalog.pg_extension AS e + INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) + INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) +WHERE d.deptype = 'e' AND e.extname = 'pxf' +ORDER BY 1; + +SELECT * FROM pxf_upgrade_test ORDER BY num; + +SELECT * FROM pxf_upgrade_test_multibyte ORDER BY num; diff --git a/automation/tincrepo/main/pxf/features/extension_tests/create_extension_rpm/__init__.py b/automation/tincrepo/main/pxf/features/extension_tests/create_extension_rpm/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/extension_tests/create_extension_rpm/expected/query01.ans b/automation/tincrepo/main/pxf/features/extension_tests/create_extension_rpm/expected/query01.ans new file mode 100644 index 0000000000..417c714a8b --- /dev/null +++ b/automation/tincrepo/main/pxf/features/extension_tests/create_extension_rpm/expected/query01.ans @@ -0,0 +1,60 @@ +-- start_ignore +-- end_ignore +-- @description query01 for PXF install test on small data +-- start_matchsubs +-- +-- m{.*/usr/local/pxf-(dev|gp\d).*} +-- s{/usr/local/pxf-(dev|gp\d)}{\$PXF_HOME} +-- +-- m{.*\$libdir/pxf.*} +-- s{\$libdir}{\$PXF_HOME/gpextable} +-- +-- end_matchsubs +-- start_ignore +\c pxfautomation_extension +-- end_ignore + +SELECT extversion FROM pg_extension WHERE extname = 'pxf'; + extversion +------------ + 2.0 +(1 row) + +SHOW dynamic_library_path; + dynamic_library_path +---------------------- + $libdir +(1 row) +SELECT p.proname, p.prosrc, p.probin +FROM pg_catalog.pg_extension AS e + INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) + INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) +WHERE d.deptype = 'e' AND e.extname = 'pxf' +ORDER BY 1; + proname | prosrc | probin +--------------------+------------------------------+---------------------------------- + pxf_read | pxfprotocol_import | $PXF_HOME/gpextable/pxf + pxf_validate | pxfprotocol_validate_urls | $PXF_HOME/gpextable/pxf + pxf_write | pxfprotocol_export | $PXF_HOME/gpextable/pxf + pxfwritable_export | gpdbwritableformatter_export | $PXF_HOME/gpextable/pxf + pxfwritable_import | gpdbwritableformatter_import | $PXF_HOME/gpextable/pxf +(5 rows) + +SELECT * FROM pxf_upgrade_test ORDER BY num; + name | num | dub | longnum | bool +--------+-----+-----+---------------+------ + row_1 | 1 | 1 | 100000000000 | f + row_2 | 2 | 2 | 200000000000 | t + row_3 | 3 | 3 | 300000000000 | f + row_4 | 4 | 4 | 400000000000 | t + row_5 | 5 | 5 | 500000000000 | f + row_6 | 6 | 6 | 600000000000 | t + row_7 | 7 | 7 | 700000000000 | f + row_8 | 8 | 8 | 800000000000 | t + row_9 | 9 | 9 | 900000000000 | f + row_10 | 10 | 10 | 1000000000000 | t +(10 rows) + +SELECT * FROM pxf_upgrade_test_multibyte ORDER BY num; +ERROR: formatter function "pxfdelimited_import" of type readable was not found +HINT: Create it with CREATE FUNCTION. diff --git a/automation/tincrepo/main/pxf/features/extension_tests/create_extension_rpm/runTest.py b/automation/tincrepo/main/pxf/features/extension_tests/create_extension_rpm/runTest.py new file mode 100644 index 0000000000..e58fbc474f --- /dev/null +++ b/automation/tincrepo/main/pxf/features/extension_tests/create_extension_rpm/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfCreateExtensionRpm(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/extension_tests/create_extension_rpm/sql/query01.sql b/automation/tincrepo/main/pxf/features/extension_tests/create_extension_rpm/sql/query01.sql new file mode 100644 index 0000000000..c9962dffec --- /dev/null +++ b/automation/tincrepo/main/pxf/features/extension_tests/create_extension_rpm/sql/query01.sql @@ -0,0 +1,28 @@ +-- @description query01 for PXF install test on small data +-- start_matchsubs +-- +-- m{.*/usr/local/pxf-(dev|gp\d).*} +-- s{/usr/local/pxf-(dev|gp\d)}{\$PXF_HOME} +-- +-- m{.*\$libdir/pxf.*} +-- s{\$libdir}{\$PXF_HOME/gpextable} +-- +-- end_matchsubs +-- start_ignore +\c pxfautomation_extension +-- end_ignore + +SELECT extversion FROM pg_extension WHERE extname = 'pxf'; + +SHOW dynamic_library_path; + +SELECT p.proname, p.prosrc, p.probin +FROM pg_catalog.pg_extension AS e + INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) + INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) +WHERE d.deptype = 'e' AND e.extname = 'pxf' +ORDER BY 1; + +SELECT * FROM pxf_upgrade_test ORDER BY num; + +SELECT * FROM pxf_upgrade_test_multibyte ORDER BY num; diff --git a/automation/tincrepo/main/pxf/features/extension_tests/downgrade/__init__.py b/automation/tincrepo/main/pxf/features/extension_tests/downgrade/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/extension_tests/downgrade/step_1_create_extension/__init__.py b/automation/tincrepo/main/pxf/features/extension_tests/downgrade/step_1_create_extension/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/extension_tests/downgrade/step_1_create_extension/expected/query01.ans b/automation/tincrepo/main/pxf/features/extension_tests/downgrade/step_1_create_extension/expected/query01.ans new file mode 100644 index 0000000000..e142b6eade --- /dev/null +++ b/automation/tincrepo/main/pxf/features/extension_tests/downgrade/step_1_create_extension/expected/query01.ans @@ -0,0 +1,73 @@ +-- start_ignore +-- end_ignore +-- @description query01 for PXF downgrade test on small data +-- start_matchsubs +-- +-- m{.*/usr/local/pxf-(dev|gp\d).*} +-- s{/usr/local/pxf-(dev|gp\d)}{\$PXF_HOME} +-- +-- m{.*\$libdir/pxf.*} +-- s{\$libdir}{\$PXF_HOME/gpextable} +-- +-- end_matchsubs +-- start_ignore +\c pxfautomation_extension +-- end_ignore + +SELECT extversion FROM pg_extension WHERE extname = 'pxf'; + extversion +------------ + 2.1 +(1 row) + +SHOW dynamic_library_path; + dynamic_library_path +---------------------- + $libdir +(1 row) + +SELECT p.proname, p.prosrc, p.probin +FROM pg_catalog.pg_extension AS e + INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) + INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) +WHERE d.deptype = 'e' AND e.extname = 'pxf' +ORDER BY 1; + proname | prosrc | probin +---------------------+------------------------------+---------------------------------- + pxf_read | pxfprotocol_import | $PXF_HOME/gpextable/pxf + pxf_validate | pxfprotocol_validate_urls | $PXF_HOME/gpextable/pxf + pxf_write | pxfprotocol_export | $PXF_HOME/gpextable/pxf + pxfdelimited_import | pxfdelimited_import | $PXF_HOME/gpextable/pxf + pxfwritable_export | gpdbwritableformatter_export | $PXF_HOME/gpextable/pxf + pxfwritable_import | gpdbwritableformatter_import | $PXF_HOME/gpextable/pxf +(6 rows) + +SELECT * FROM pxf_upgrade_test ORDER BY num; + name | num | dub | longnum | bool +--------+-----+-----+---------------+------ + row_1 | 1 | 1 | 100000000000 | f + row_2 | 2 | 2 | 200000000000 | t + row_3 | 3 | 3 | 300000000000 | f + row_4 | 4 | 4 | 400000000000 | t + row_5 | 5 | 5 | 500000000000 | f + row_6 | 6 | 6 | 600000000000 | t + row_7 | 7 | 7 | 700000000000 | f + row_8 | 8 | 8 | 800000000000 | t + row_9 | 9 | 9 | 900000000000 | f + row_10 | 10 | 10 | 1000000000000 | t +(10 rows) + +SELECT * FROM pxf_upgrade_test_multibyte ORDER BY num; + name | num | dub | longnum | bool +--------+-----+-----+---------------+------ + row_1 | 1 | 1 | 100000000000 | f + row_2 | 2 | 2 | 200000000000 | t + row_3 | 3 | 3 | 300000000000 | f + row_4 | 4 | 4 | 400000000000 | t + row_5 | 5 | 5 | 500000000000 | f + row_6 | 6 | 6 | 600000000000 | t + row_7 | 7 | 7 | 700000000000 | f + row_8 | 8 | 8 | 800000000000 | t + row_9 | 9 | 9 | 900000000000 | f + row_10 | 10 | 10 | 1000000000000 | t +(10 rows) diff --git a/automation/tincrepo/main/pxf/features/extension_tests/downgrade/step_1_create_extension/runTest.py b/automation/tincrepo/main/pxf/features/extension_tests/downgrade/step_1_create_extension/runTest.py new file mode 100644 index 0000000000..d3ea0c067f --- /dev/null +++ b/automation/tincrepo/main/pxf/features/extension_tests/downgrade/step_1_create_extension/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfDowngradeStep1CreateExtension(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/extension_tests/downgrade/step_1_create_extension/sql/query01.sql b/automation/tincrepo/main/pxf/features/extension_tests/downgrade/step_1_create_extension/sql/query01.sql new file mode 100644 index 0000000000..76edbb8211 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/extension_tests/downgrade/step_1_create_extension/sql/query01.sql @@ -0,0 +1,28 @@ +-- @description query01 for PXF downgrade test on small data +-- start_matchsubs +-- +-- m{.*/usr/local/pxf-(dev|gp\d).*} +-- s{/usr/local/pxf-(dev|gp\d)}{\$PXF_HOME} +-- +-- m{.*\$libdir/pxf.*} +-- s{\$libdir}{\$PXF_HOME/gpextable} +-- +-- end_matchsubs +-- start_ignore +\c pxfautomation_extension +-- end_ignore + +SELECT extversion FROM pg_extension WHERE extname = 'pxf'; + +SHOW dynamic_library_path; + +SELECT p.proname, p.prosrc, p.probin +FROM pg_catalog.pg_extension AS e + INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) + INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) +WHERE d.deptype = 'e' AND e.extname = 'pxf' +ORDER BY 1; + +SELECT * FROM pxf_upgrade_test ORDER BY num; + +SELECT * FROM pxf_upgrade_test_multibyte ORDER BY num; diff --git a/automation/tincrepo/main/pxf/features/extension_tests/downgrade/step_2_after_alter_extension_downgrade/__init__.py b/automation/tincrepo/main/pxf/features/extension_tests/downgrade/step_2_after_alter_extension_downgrade/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/extension_tests/downgrade/step_2_after_alter_extension_downgrade/expected/query01.ans b/automation/tincrepo/main/pxf/features/extension_tests/downgrade/step_2_after_alter_extension_downgrade/expected/query01.ans new file mode 100644 index 0000000000..c51d284396 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/extension_tests/downgrade/step_2_after_alter_extension_downgrade/expected/query01.ans @@ -0,0 +1,67 @@ +-- start_ignore +-- end_ignore +-- @description query01 for PXF downgrade test on small data +-- start_matchsubs +-- +-- m{.*/usr/local/pxf-(dev|gp\d).*} +-- s{/usr/local/pxf-(dev|gp\d)}{\$PXF_HOME} +-- +-- m{.*\$libdir/pxf.*} +-- s{\$libdir}{\$PXF_HOME/gpextable} +-- +-- m{.*\"pxfdelimited_import\".*} +-- s{\"pxfdelimited_import\"}{pxfdelimited_import} +-- +-- m{.*found\.*} +-- s{found\.}{found} +-- +-- end_matchsubs +-- start_ignore +\c pxfautomation_eextnsion +-- end_ignore + +SELECT extversion FROM pg_extension WHERE extname = 'pxf'; + extversion +------------ + 2.0 +(1 row) + +SHOW dynamic_library_path; + dynamic_library_path +---------------------- + $libdir +(1 row) + +SELECT p.proname, p.prosrc, p.probin +FROM pg_catalog.pg_extension AS e + INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) + INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) +WHERE d.deptype = 'e' AND e.extname = 'pxf' +ORDER BY 1; + proname | prosrc | probin +--------------------+------------------------------+---------------------------------- + pxf_read | pxfprotocol_import | $PXF_HOME/gpextable/pxf + pxf_validate | pxfprotocol_validate_urls | $PXF_HOME/gpextable/pxf + pxf_write | pxfprotocol_export | $PXF_HOME/gpextable/pxf + pxfwritable_export | gpdbwritableformatter_export | $PXF_HOME/gpextable/pxf + pxfwritable_import | gpdbwritableformatter_import | $PXF_HOME/gpextable/pxf +(5 rows) + +SELECT * FROM pxf_upgrade_test ORDER BY num; + name | num | dub | longnum | bool +--------+-----+-----+---------------+------ + row_1 | 1 | 1 | 100000000000 | f + row_2 | 2 | 2 | 200000000000 | t + row_3 | 3 | 3 | 300000000000 | f + row_4 | 4 | 4 | 400000000000 | t + row_5 | 5 | 5 | 500000000000 | f + row_6 | 6 | 6 | 600000000000 | t + row_7 | 7 | 7 | 700000000000 | f + row_8 | 8 | 8 | 800000000000 | t + row_9 | 9 | 9 | 900000000000 | f + row_10 | 10 | 10 | 1000000000000 | t +(10 rows) + +SELECT * FROM pxf_upgrade_test_multibyte ORDER BY num; +ERROR: formatter function "pxfdelimited_import" of type readable was not found +HINT: Create it with CREATE FUNCTION. diff --git a/automation/tincrepo/main/pxf/features/extension_tests/downgrade/step_2_after_alter_extension_downgrade/runTest.py b/automation/tincrepo/main/pxf/features/extension_tests/downgrade/step_2_after_alter_extension_downgrade/runTest.py new file mode 100644 index 0000000000..074c7f1e20 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/extension_tests/downgrade/step_2_after_alter_extension_downgrade/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfDowngradeStep2AlterExtensionPxf(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/extension_tests/downgrade/step_2_after_alter_extension_downgrade/sql/query01.sql b/automation/tincrepo/main/pxf/features/extension_tests/downgrade/step_2_after_alter_extension_downgrade/sql/query01.sql new file mode 100644 index 0000000000..8a87c225e2 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/extension_tests/downgrade/step_2_after_alter_extension_downgrade/sql/query01.sql @@ -0,0 +1,34 @@ +-- @description query01 for PXF downgrade test on small data +-- start_matchsubs +-- +-- m{.*/usr/local/pxf-(dev|gp\d).*} +-- s{/usr/local/pxf-(dev|gp\d)}{\$PXF_HOME} +-- +-- m{.*\$libdir/pxf.*} +-- s{\$libdir}{\$PXF_HOME/gpextable} +-- +-- m{.*\"pxfdelimited_import\".*} +-- s{\"pxfdelimited_import\"}{pxfdelimited_import} +-- +-- m{.*found\.*} +-- s{found\.}{found} +-- +-- end_matchsubs +-- start_ignore +\c pxfautomation_extension +-- end_ignore + +SELECT extversion FROM pg_extension WHERE extname = 'pxf'; + +SHOW dynamic_library_path; + +SELECT p.proname, p.prosrc, p.probin +FROM pg_catalog.pg_extension AS e + INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) + INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) +WHERE d.deptype = 'e' AND e.extname = 'pxf' +ORDER BY 1; + +SELECT * FROM pxf_upgrade_test ORDER BY num; + +SELECT * FROM pxf_upgrade_test_multibyte ORDER BY num; diff --git a/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/__init__.py b/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/step_1_check_extension/__init__.py b/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/step_1_check_extension/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/step_1_check_extension/expected/query01.ans b/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/step_1_check_extension/expected/query01.ans new file mode 100644 index 0000000000..e142b6eade --- /dev/null +++ b/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/step_1_check_extension/expected/query01.ans @@ -0,0 +1,73 @@ +-- start_ignore +-- end_ignore +-- @description query01 for PXF downgrade test on small data +-- start_matchsubs +-- +-- m{.*/usr/local/pxf-(dev|gp\d).*} +-- s{/usr/local/pxf-(dev|gp\d)}{\$PXF_HOME} +-- +-- m{.*\$libdir/pxf.*} +-- s{\$libdir}{\$PXF_HOME/gpextable} +-- +-- end_matchsubs +-- start_ignore +\c pxfautomation_extension +-- end_ignore + +SELECT extversion FROM pg_extension WHERE extname = 'pxf'; + extversion +------------ + 2.1 +(1 row) + +SHOW dynamic_library_path; + dynamic_library_path +---------------------- + $libdir +(1 row) + +SELECT p.proname, p.prosrc, p.probin +FROM pg_catalog.pg_extension AS e + INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) + INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) +WHERE d.deptype = 'e' AND e.extname = 'pxf' +ORDER BY 1; + proname | prosrc | probin +---------------------+------------------------------+---------------------------------- + pxf_read | pxfprotocol_import | $PXF_HOME/gpextable/pxf + pxf_validate | pxfprotocol_validate_urls | $PXF_HOME/gpextable/pxf + pxf_write | pxfprotocol_export | $PXF_HOME/gpextable/pxf + pxfdelimited_import | pxfdelimited_import | $PXF_HOME/gpextable/pxf + pxfwritable_export | gpdbwritableformatter_export | $PXF_HOME/gpextable/pxf + pxfwritable_import | gpdbwritableformatter_import | $PXF_HOME/gpextable/pxf +(6 rows) + +SELECT * FROM pxf_upgrade_test ORDER BY num; + name | num | dub | longnum | bool +--------+-----+-----+---------------+------ + row_1 | 1 | 1 | 100000000000 | f + row_2 | 2 | 2 | 200000000000 | t + row_3 | 3 | 3 | 300000000000 | f + row_4 | 4 | 4 | 400000000000 | t + row_5 | 5 | 5 | 500000000000 | f + row_6 | 6 | 6 | 600000000000 | t + row_7 | 7 | 7 | 700000000000 | f + row_8 | 8 | 8 | 800000000000 | t + row_9 | 9 | 9 | 900000000000 | f + row_10 | 10 | 10 | 1000000000000 | t +(10 rows) + +SELECT * FROM pxf_upgrade_test_multibyte ORDER BY num; + name | num | dub | longnum | bool +--------+-----+-----+---------------+------ + row_1 | 1 | 1 | 100000000000 | f + row_2 | 2 | 2 | 200000000000 | t + row_3 | 3 | 3 | 300000000000 | f + row_4 | 4 | 4 | 400000000000 | t + row_5 | 5 | 5 | 500000000000 | f + row_6 | 6 | 6 | 600000000000 | t + row_7 | 7 | 7 | 700000000000 | f + row_8 | 8 | 8 | 800000000000 | t + row_9 | 9 | 9 | 900000000000 | f + row_10 | 10 | 10 | 1000000000000 | t +(10 rows) diff --git a/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/step_1_check_extension/runTest.py b/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/step_1_check_extension/runTest.py new file mode 100644 index 0000000000..018a9aac88 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/step_1_check_extension/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfDowngradeStep1CheckExtension(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/step_1_check_extension/sql/query01.sql b/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/step_1_check_extension/sql/query01.sql new file mode 100644 index 0000000000..76edbb8211 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/step_1_check_extension/sql/query01.sql @@ -0,0 +1,28 @@ +-- @description query01 for PXF downgrade test on small data +-- start_matchsubs +-- +-- m{.*/usr/local/pxf-(dev|gp\d).*} +-- s{/usr/local/pxf-(dev|gp\d)}{\$PXF_HOME} +-- +-- m{.*\$libdir/pxf.*} +-- s{\$libdir}{\$PXF_HOME/gpextable} +-- +-- end_matchsubs +-- start_ignore +\c pxfautomation_extension +-- end_ignore + +SELECT extversion FROM pg_extension WHERE extname = 'pxf'; + +SHOW dynamic_library_path; + +SELECT p.proname, p.prosrc, p.probin +FROM pg_catalog.pg_extension AS e + INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) + INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) +WHERE d.deptype = 'e' AND e.extname = 'pxf' +ORDER BY 1; + +SELECT * FROM pxf_upgrade_test ORDER BY num; + +SELECT * FROM pxf_upgrade_test_multibyte ORDER BY num; diff --git a/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/step_2_after_alter_extension_downgrade/__init__.py b/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/step_2_after_alter_extension_downgrade/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/step_2_after_alter_extension_downgrade/expected/query01.ans b/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/step_2_after_alter_extension_downgrade/expected/query01.ans new file mode 100644 index 0000000000..c2b26b82bb --- /dev/null +++ b/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/step_2_after_alter_extension_downgrade/expected/query01.ans @@ -0,0 +1,67 @@ +-- start_ignore +-- end_ignore +-- @description query01 for PXF downgrade test on small data +-- start_matchsubs +-- +-- m{.*/usr/local/pxf-(dev|gp\d).*} +-- s{/usr/local/pxf-(dev|gp\d)}{\$PXF_HOME} +-- +-- m{.*\$libdir/pxf.*} +-- s{\$libdir}{\$PXF_HOME/gpextable} +-- +-- m{.*\"pxfdelimited_import\".*} +-- s{\"pxfdelimited_import\"}{pxfdelimited_import} +-- +-- m{.*found\.*} +-- s{found\.}{found} +-- +-- end_matchsubs +-- start_ignore +\c pxfautomation_extension +-- end_ignore + +SELECT extversion FROM pg_extension WHERE extname = 'pxf'; + extversion +------------ + 2.0 +(1 row) + +SHOW dynamic_library_path; + dynamic_library_path +---------------------- + $libdir +(1 row) + +SELECT p.proname, p.prosrc, p.probin +FROM pg_catalog.pg_extension AS e + INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) + INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) +WHERE d.deptype = 'e' AND e.extname = 'pxf' +ORDER BY 1; + proname | prosrc | probin +--------------------+------------------------------+---------------------------------- + pxf_read | pxfprotocol_import | $PXF_HOME/gpextable/pxf + pxf_validate | pxfprotocol_validate_urls | $PXF_HOME/gpextable/pxf + pxf_write | pxfprotocol_export | $PXF_HOME/gpextable/pxf + pxfwritable_export | gpdbwritableformatter_export | $PXF_HOME/gpextable/pxf + pxfwritable_import | gpdbwritableformatter_import | $PXF_HOME/gpextable/pxf +(5 rows) + +SELECT * FROM pxf_upgrade_test ORDER BY num; + name | num | dub | longnum | bool +--------+-----+-----+---------------+------ + row_1 | 1 | 1 | 100000000000 | f + row_2 | 2 | 2 | 200000000000 | t + row_3 | 3 | 3 | 300000000000 | f + row_4 | 4 | 4 | 400000000000 | t + row_5 | 5 | 5 | 500000000000 | f + row_6 | 6 | 6 | 600000000000 | t + row_7 | 7 | 7 | 700000000000 | f + row_8 | 8 | 8 | 800000000000 | t + row_9 | 9 | 9 | 900000000000 | f + row_10 | 10 | 10 | 1000000000000 | t +(10 rows) + +SELECT * FROM pxf_upgrade_test_multibyte ORDER BY num; +ERROR: formatter function "pxfdelimited_import" of type readable was not found +HINT: Create it with CREATE FUNCTION. diff --git a/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/step_2_after_alter_extension_downgrade/runTest.py b/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/step_2_after_alter_extension_downgrade/runTest.py new file mode 100644 index 0000000000..074c7f1e20 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/step_2_after_alter_extension_downgrade/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfDowngradeStep2AlterExtensionPxf(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/step_2_after_alter_extension_downgrade/sql/query01.sql b/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/step_2_after_alter_extension_downgrade/sql/query01.sql new file mode 100644 index 0000000000..8a87c225e2 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/step_2_after_alter_extension_downgrade/sql/query01.sql @@ -0,0 +1,34 @@ +-- @description query01 for PXF downgrade test on small data +-- start_matchsubs +-- +-- m{.*/usr/local/pxf-(dev|gp\d).*} +-- s{/usr/local/pxf-(dev|gp\d)}{\$PXF_HOME} +-- +-- m{.*\$libdir/pxf.*} +-- s{\$libdir}{\$PXF_HOME/gpextable} +-- +-- m{.*\"pxfdelimited_import\".*} +-- s{\"pxfdelimited_import\"}{pxfdelimited_import} +-- +-- m{.*found\.*} +-- s{found\.}{found} +-- +-- end_matchsubs +-- start_ignore +\c pxfautomation_extension +-- end_ignore + +SELECT extversion FROM pg_extension WHERE extname = 'pxf'; + +SHOW dynamic_library_path; + +SELECT p.proname, p.prosrc, p.probin +FROM pg_catalog.pg_extension AS e + INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) + INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) +WHERE d.deptype = 'e' AND e.extname = 'pxf' +ORDER BY 1; + +SELECT * FROM pxf_upgrade_test ORDER BY num; + +SELECT * FROM pxf_upgrade_test_multibyte ORDER BY num; diff --git a/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/step_3_after_alter_extension_upgrade/__init__.py b/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/step_3_after_alter_extension_upgrade/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/step_3_after_alter_extension_upgrade/expected/query01.ans b/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/step_3_after_alter_extension_upgrade/expected/query01.ans new file mode 100644 index 0000000000..e142b6eade --- /dev/null +++ b/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/step_3_after_alter_extension_upgrade/expected/query01.ans @@ -0,0 +1,73 @@ +-- start_ignore +-- end_ignore +-- @description query01 for PXF downgrade test on small data +-- start_matchsubs +-- +-- m{.*/usr/local/pxf-(dev|gp\d).*} +-- s{/usr/local/pxf-(dev|gp\d)}{\$PXF_HOME} +-- +-- m{.*\$libdir/pxf.*} +-- s{\$libdir}{\$PXF_HOME/gpextable} +-- +-- end_matchsubs +-- start_ignore +\c pxfautomation_extension +-- end_ignore + +SELECT extversion FROM pg_extension WHERE extname = 'pxf'; + extversion +------------ + 2.1 +(1 row) + +SHOW dynamic_library_path; + dynamic_library_path +---------------------- + $libdir +(1 row) + +SELECT p.proname, p.prosrc, p.probin +FROM pg_catalog.pg_extension AS e + INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) + INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) +WHERE d.deptype = 'e' AND e.extname = 'pxf' +ORDER BY 1; + proname | prosrc | probin +---------------------+------------------------------+---------------------------------- + pxf_read | pxfprotocol_import | $PXF_HOME/gpextable/pxf + pxf_validate | pxfprotocol_validate_urls | $PXF_HOME/gpextable/pxf + pxf_write | pxfprotocol_export | $PXF_HOME/gpextable/pxf + pxfdelimited_import | pxfdelimited_import | $PXF_HOME/gpextable/pxf + pxfwritable_export | gpdbwritableformatter_export | $PXF_HOME/gpextable/pxf + pxfwritable_import | gpdbwritableformatter_import | $PXF_HOME/gpextable/pxf +(6 rows) + +SELECT * FROM pxf_upgrade_test ORDER BY num; + name | num | dub | longnum | bool +--------+-----+-----+---------------+------ + row_1 | 1 | 1 | 100000000000 | f + row_2 | 2 | 2 | 200000000000 | t + row_3 | 3 | 3 | 300000000000 | f + row_4 | 4 | 4 | 400000000000 | t + row_5 | 5 | 5 | 500000000000 | f + row_6 | 6 | 6 | 600000000000 | t + row_7 | 7 | 7 | 700000000000 | f + row_8 | 8 | 8 | 800000000000 | t + row_9 | 9 | 9 | 900000000000 | f + row_10 | 10 | 10 | 1000000000000 | t +(10 rows) + +SELECT * FROM pxf_upgrade_test_multibyte ORDER BY num; + name | num | dub | longnum | bool +--------+-----+-----+---------------+------ + row_1 | 1 | 1 | 100000000000 | f + row_2 | 2 | 2 | 200000000000 | t + row_3 | 3 | 3 | 300000000000 | f + row_4 | 4 | 4 | 400000000000 | t + row_5 | 5 | 5 | 500000000000 | f + row_6 | 6 | 6 | 600000000000 | t + row_7 | 7 | 7 | 700000000000 | f + row_8 | 8 | 8 | 800000000000 | t + row_9 | 9 | 9 | 900000000000 | f + row_10 | 10 | 10 | 1000000000000 | t +(10 rows) diff --git a/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/step_3_after_alter_extension_upgrade/runTest.py b/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/step_3_after_alter_extension_upgrade/runTest.py new file mode 100644 index 0000000000..c04be95838 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/step_3_after_alter_extension_upgrade/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfDowngradeStep3UpgradeExtension(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/step_3_after_alter_extension_upgrade/sql/query01.sql b/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/step_3_after_alter_extension_upgrade/sql/query01.sql new file mode 100644 index 0000000000..76edbb8211 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/extension_tests/downgrade_then_upgrade/step_3_after_alter_extension_upgrade/sql/query01.sql @@ -0,0 +1,28 @@ +-- @description query01 for PXF downgrade test on small data +-- start_matchsubs +-- +-- m{.*/usr/local/pxf-(dev|gp\d).*} +-- s{/usr/local/pxf-(dev|gp\d)}{\$PXF_HOME} +-- +-- m{.*\$libdir/pxf.*} +-- s{\$libdir}{\$PXF_HOME/gpextable} +-- +-- end_matchsubs +-- start_ignore +\c pxfautomation_extension +-- end_ignore + +SELECT extversion FROM pg_extension WHERE extname = 'pxf'; + +SHOW dynamic_library_path; + +SELECT p.proname, p.prosrc, p.probin +FROM pg_catalog.pg_extension AS e + INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) + INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) +WHERE d.deptype = 'e' AND e.extname = 'pxf' +ORDER BY 1; + +SELECT * FROM pxf_upgrade_test ORDER BY num; + +SELECT * FROM pxf_upgrade_test_multibyte ORDER BY num; diff --git a/automation/tincrepo/main/pxf/features/extension_tests/explicit_upgrade/__init__.py b/automation/tincrepo/main/pxf/features/extension_tests/explicit_upgrade/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/extension_tests/explicit_upgrade/step_1_create_extension_with_older_pxf_version/__init__.py b/automation/tincrepo/main/pxf/features/extension_tests/explicit_upgrade/step_1_create_extension_with_older_pxf_version/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/extension_tests/explicit_upgrade/step_1_create_extension_with_older_pxf_version/expected/query01.ans b/automation/tincrepo/main/pxf/features/extension_tests/explicit_upgrade/step_1_create_extension_with_older_pxf_version/expected/query01.ans new file mode 100644 index 0000000000..406e013d71 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/extension_tests/explicit_upgrade/step_1_create_extension_with_older_pxf_version/expected/query01.ans @@ -0,0 +1,57 @@ +-- start_ignore +-- end_ignore +-- @description query01 for PXF upgrade with explicit version test on small data +-- start_matchsubs +-- +-- m{.*/usr/local/pxf-(dev|gp\d).*} +-- s{/usr/local/pxf-(dev|gp\d)}{\$PXF_HOME} +-- +-- m{.*\$libdir/pxf.*} +-- s{\$libdir}{\$PXF_HOME/gpextable} +-- +-- end_matchsubs +-- start_ignore +\c pxfautomation_extension +-- end_ignore + +SELECT extversion FROM pg_extension WHERE extname = 'pxf'; + extversion +------------ + 2.0 +(1 row) + +SHOW dynamic_library_path; + dynamic_library_path +---------------------- + $libdir +(1 row) + +SELECT p.proname, p.prosrc, p.probin +FROM pg_catalog.pg_extension AS e + INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) + INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) +WHERE d.deptype = 'e' AND e.extname = 'pxf' +ORDER BY 1; + proname | prosrc | probin +--------------------+------------------------------+---------------------------------- + pxf_read | pxfprotocol_import | $PXF_HOME/gpextable/pxf + pxf_validate | pxfprotocol_validate_urls | $PXF_HOME/gpextable/pxf + pxf_write | pxfprotocol_export | $PXF_HOME/gpextable/pxf + pxfwritable_export | gpdbwritableformatter_export | $PXF_HOME/gpextable/pxf + pxfwritable_import | gpdbwritableformatter_import | $PXF_HOME/gpextable/pxf +(5 rows) + +SELECT * FROM pxf_upgrade_test ORDER BY num; + name | num | dub | longnum | bool +--------+-----+-----+---------------+------ + row_1 | 1 | 1 | 100000000000 | f + row_2 | 2 | 2 | 200000000000 | t + row_3 | 3 | 3 | 300000000000 | f + row_4 | 4 | 4 | 400000000000 | t + row_5 | 5 | 5 | 500000000000 | f + row_6 | 6 | 6 | 600000000000 | t + row_7 | 7 | 7 | 700000000000 | f + row_8 | 8 | 8 | 800000000000 | t + row_9 | 9 | 9 | 900000000000 | f + row_10 | 10 | 10 | 1000000000000 | t +(10 rows) diff --git a/automation/tincrepo/main/pxf/features/extension_tests/explicit_upgrade/step_1_create_extension_with_older_pxf_version/runTest.py b/automation/tincrepo/main/pxf/features/extension_tests/explicit_upgrade/step_1_create_extension_with_older_pxf_version/runTest.py new file mode 100644 index 0000000000..b5326cdcbe --- /dev/null +++ b/automation/tincrepo/main/pxf/features/extension_tests/explicit_upgrade/step_1_create_extension_with_older_pxf_version/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfUpgradeStep1CreateExtensionExplicitVersion(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/extension_tests/explicit_upgrade/step_1_create_extension_with_older_pxf_version/sql/query01.sql b/automation/tincrepo/main/pxf/features/extension_tests/explicit_upgrade/step_1_create_extension_with_older_pxf_version/sql/query01.sql new file mode 100644 index 0000000000..48ba18a339 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/extension_tests/explicit_upgrade/step_1_create_extension_with_older_pxf_version/sql/query01.sql @@ -0,0 +1,26 @@ +-- @description query01 for PXF upgrade with explicit version test on small data +-- start_matchsubs +-- +-- m{.*/usr/local/pxf-(dev|gp\d).*} +-- s{/usr/local/pxf-(dev|gp\d)}{\$PXF_HOME} +-- +-- m{.*\$libdir/pxf.*} +-- s{\$libdir}{\$PXF_HOME/gpextable} +-- +-- end_matchsubs +-- start_ignore +\c pxfautomation_extension +-- end_ignore + +SELECT extversion FROM pg_extension WHERE extname = 'pxf'; + +SHOW dynamic_library_path; + +SELECT p.proname, p.prosrc, p.probin +FROM pg_catalog.pg_extension AS e + INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) + INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) +WHERE d.deptype = 'e' AND e.extname = 'pxf' +ORDER BY 1; + +SELECT * FROM pxf_upgrade_test ORDER BY num; diff --git a/automation/tincrepo/main/pxf/features/extension_tests/explicit_upgrade/step_2_after_alter_extension/__init__.py b/automation/tincrepo/main/pxf/features/extension_tests/explicit_upgrade/step_2_after_alter_extension/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/extension_tests/explicit_upgrade/step_2_after_alter_extension/expected/query01.ans b/automation/tincrepo/main/pxf/features/extension_tests/explicit_upgrade/step_2_after_alter_extension/expected/query01.ans new file mode 100644 index 0000000000..b70f65c76d --- /dev/null +++ b/automation/tincrepo/main/pxf/features/extension_tests/explicit_upgrade/step_2_after_alter_extension/expected/query01.ans @@ -0,0 +1,73 @@ +-- start_ignore +-- end_ignore +-- @description query01 for PXF upgrade with explicit version test on small data +-- start_matchsubs +-- +-- m{.*/usr/local/pxf-(dev|gp\d).*} +-- s{/usr/local/pxf-(dev|gp\d)}{\$PXF_HOME} +-- +-- m{.*\$libdir/pxf.*} +-- s{\$libdir}{\$PXF_HOME/gpextable} +-- +-- end_matchsubs +-- start_ignore +\c pxfautomation_extension +-- end_ignore + +SELECT extversion FROM pg_extension WHERE extname = 'pxf'; + extversion +------------ + 2.1 +(1 row) + +SHOW dynamic_library_path; + dynamic_library_path +---------------------- + $libdir +(1 row) + +SELECT p.proname, p.prosrc, p.probin +FROM pg_catalog.pg_extension AS e + INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) + INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) +WHERE d.deptype = 'e' AND e.extname = 'pxf' +ORDER BY 1; + proname | prosrc | probin +---------------------+------------------------------+---------------------------------- + pxf_read | pxfprotocol_import | $PXF_HOME/gpextable/pxf + pxf_validate | pxfprotocol_validate_urls | $PXF_HOME/gpextable/pxf + pxf_write | pxfprotocol_export | $PXF_HOME/gpextable/pxf + pxfdelimited_import | pxfdelimited_import | $PXF_HOME/gpextable/pxf + pxfwritable_export | gpdbwritableformatter_export | $PXF_HOME/gpextable/pxf + pxfwritable_import | gpdbwritableformatter_import | $PXF_HOME/gpextable/pxf +(6 rows) + +SELECT * FROM pxf_upgrade_test ORDER BY num; + name | num | dub | longnum | bool +--------+-----+-----+---------------+------ + row_1 | 1 | 1 | 100000000000 | f + row_2 | 2 | 2 | 200000000000 | t + row_3 | 3 | 3 | 300000000000 | f + row_4 | 4 | 4 | 400000000000 | t + row_5 | 5 | 5 | 500000000000 | f + row_6 | 6 | 6 | 600000000000 | t + row_7 | 7 | 7 | 700000000000 | f + row_8 | 8 | 8 | 800000000000 | t + row_9 | 9 | 9 | 900000000000 | f + row_10 | 10 | 10 | 1000000000000 | t +(10 rows) + +SELECT * FROM pxf_upgrade_test_multibyte ORDER BY num; + name | num | dub | longnum | bool +--------+-----+-----+---------------+------ + row_1 | 1 | 1 | 100000000000 | f + row_2 | 2 | 2 | 200000000000 | t + row_3 | 3 | 3 | 300000000000 | f + row_4 | 4 | 4 | 400000000000 | t + row_5 | 5 | 5 | 500000000000 | f + row_6 | 6 | 6 | 600000000000 | t + row_7 | 7 | 7 | 700000000000 | f + row_8 | 8 | 8 | 800000000000 | t + row_9 | 9 | 9 | 900000000000 | f + row_10 | 10 | 10 | 1000000000000 | t +(10 rows) diff --git a/automation/tincrepo/main/pxf/features/extension_tests/explicit_upgrade/step_2_after_alter_extension/runTest.py b/automation/tincrepo/main/pxf/features/extension_tests/explicit_upgrade/step_2_after_alter_extension/runTest.py new file mode 100644 index 0000000000..a045020533 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/extension_tests/explicit_upgrade/step_2_after_alter_extension/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfUpgradeStep2AfterAlterExtensionExplicitVersion(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/extension_tests/explicit_upgrade/step_2_after_alter_extension/sql/query01.sql b/automation/tincrepo/main/pxf/features/extension_tests/explicit_upgrade/step_2_after_alter_extension/sql/query01.sql new file mode 100644 index 0000000000..c3075e9b9c --- /dev/null +++ b/automation/tincrepo/main/pxf/features/extension_tests/explicit_upgrade/step_2_after_alter_extension/sql/query01.sql @@ -0,0 +1,28 @@ +-- @description query01 for PXF upgrade with explicit version test on small data +-- start_matchsubs +-- +-- m{.*/usr/local/pxf-(dev|gp\d).*} +-- s{/usr/local/pxf-(dev|gp\d)}{\$PXF_HOME} +-- +-- m{.*\$libdir/pxf.*} +-- s{\$libdir}{\$PXF_HOME/gpextable} +-- +-- end_matchsubs +-- start_ignore +\c pxfautomation_extension +-- end_ignore + +SELECT extversion FROM pg_extension WHERE extname = 'pxf'; + +SHOW dynamic_library_path; + +SELECT p.proname, p.prosrc, p.probin +FROM pg_catalog.pg_extension AS e + INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) + INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) +WHERE d.deptype = 'e' AND e.extname = 'pxf' +ORDER BY 1; + +SELECT * FROM pxf_upgrade_test ORDER BY num; + +SELECT * FROM pxf_upgrade_test_multibyte ORDER BY num; diff --git a/automation/tincrepo/main/pxf/features/extension_tests/upgrade/__init__.py b/automation/tincrepo/main/pxf/features/extension_tests/upgrade/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/extension_tests/upgrade/step_1_create_extension_with_older_pxf_version/__init__.py b/automation/tincrepo/main/pxf/features/extension_tests/upgrade/step_1_create_extension_with_older_pxf_version/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/extension_tests/upgrade/step_1_create_extension_with_older_pxf_version/expected/query01.ans b/automation/tincrepo/main/pxf/features/extension_tests/upgrade/step_1_create_extension_with_older_pxf_version/expected/query01.ans new file mode 100644 index 0000000000..02dfffac39 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/extension_tests/upgrade/step_1_create_extension_with_older_pxf_version/expected/query01.ans @@ -0,0 +1,58 @@ +-- start_ignore +-- end_ignore +-- @description query01 for PXF gpupgrade test on small data +-- start_matchsubs +-- +-- m{.*/usr/local/pxf-(dev|gp\d).*} +-- s{/usr/local/pxf-(dev|gp\d)}{\$PXF_HOME} +-- +-- m{.*\$libdir/pxf.*} +-- s{\$libdir}{\$PXF_HOME/gpextable} +-- +-- end_matchsubs +-- @description query01 for PXF upgrade test on small data +-- start_ignore +\c pxfautomation_extension +-- end_ignore + +SELECT extversion FROM pg_extension WHERE extname = 'pxf'; + extversion +------------ + 2.0 +(1 row) + +SHOW dynamic_library_path; + dynamic_library_path +---------------------- + $libdir +(1 row) + +SELECT p.proname, p.prosrc, p.probin +FROM pg_catalog.pg_extension AS e + INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) + INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) +WHERE d.deptype = 'e' AND e.extname = 'pxf' +ORDER BY 1; + proname | prosrc | probin +--------------------+------------------------------+---------------------------------- + pxf_read | pxfprotocol_import | $PXF_HOME/gpextable/pxf + pxf_validate | pxfprotocol_validate_urls | $PXF_HOME/gpextable/pxf + pxf_write | pxfprotocol_export | $PXF_HOME/gpextable/pxf + pxfwritable_export | gpdbwritableformatter_export | $PXF_HOME/gpextable/pxf + pxfwritable_import | gpdbwritableformatter_import | $PXF_HOME/gpextable/pxf +(5 rows) + +SELECT * FROM pxf_upgrade_test ORDER BY num; + name | num | dub | longnum | bool +--------+-----+-----+---------------+------ + row_1 | 1 | 1 | 100000000000 | f + row_2 | 2 | 2 | 200000000000 | t + row_3 | 3 | 3 | 300000000000 | f + row_4 | 4 | 4 | 400000000000 | t + row_5 | 5 | 5 | 500000000000 | f + row_6 | 6 | 6 | 600000000000 | t + row_7 | 7 | 7 | 700000000000 | f + row_8 | 8 | 8 | 800000000000 | t + row_9 | 9 | 9 | 900000000000 | f + row_10 | 10 | 10 | 1000000000000 | t +(10 rows) diff --git a/automation/tincrepo/main/pxf/features/extension_tests/upgrade/step_1_create_extension_with_older_pxf_version/runTest.py b/automation/tincrepo/main/pxf/features/extension_tests/upgrade/step_1_create_extension_with_older_pxf_version/runTest.py new file mode 100644 index 0000000000..c51b01210f --- /dev/null +++ b/automation/tincrepo/main/pxf/features/extension_tests/upgrade/step_1_create_extension_with_older_pxf_version/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfUpgradeStep1CreateExtension(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/extension_tests/upgrade/step_1_create_extension_with_older_pxf_version/sql/query01.sql b/automation/tincrepo/main/pxf/features/extension_tests/upgrade/step_1_create_extension_with_older_pxf_version/sql/query01.sql new file mode 100644 index 0000000000..8c31dcf16f --- /dev/null +++ b/automation/tincrepo/main/pxf/features/extension_tests/upgrade/step_1_create_extension_with_older_pxf_version/sql/query01.sql @@ -0,0 +1,26 @@ +-- @description query01 for PXF gpupgrade test on small data +-- start_matchsubs +-- +-- m{.*/usr/local/pxf-(dev|gp\d).*} +-- s{/usr/local/pxf-(dev|gp\d)}{\$PXF_HOME} +-- +-- m{.*\$libdir/pxf.*} +-- s{\$libdir}{\$PXF_HOME/gpextable} +-- +-- end_matchsubs +-- start_ignore +\c pxfautomation_extension +-- end_ignore + +SELECT extversion FROM pg_extension WHERE extname = 'pxf'; + +SHOW dynamic_library_path; + +SELECT p.proname, p.prosrc, p.probin +FROM pg_catalog.pg_extension AS e + INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) + INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) +WHERE d.deptype = 'e' AND e.extname = 'pxf' +ORDER BY 1; + +SELECT * FROM pxf_upgrade_test ORDER BY num; diff --git a/automation/tincrepo/main/pxf/features/extension_tests/upgrade/step_2_after_alter_extension/__init__.py b/automation/tincrepo/main/pxf/features/extension_tests/upgrade/step_2_after_alter_extension/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/extension_tests/upgrade/step_2_after_alter_extension/expected/query01.ans b/automation/tincrepo/main/pxf/features/extension_tests/upgrade/step_2_after_alter_extension/expected/query01.ans new file mode 100644 index 0000000000..9f87b291b6 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/extension_tests/upgrade/step_2_after_alter_extension/expected/query01.ans @@ -0,0 +1,73 @@ +-- start_ignore +-- end_ignore +-- @description query01 for PXF upgrade test on small data +-- start_matchsubs +-- +-- m{.*/usr/local/pxf-(dev|gp\d).*} +-- s{/usr/local/pxf-(dev|gp\d)}{\$PXF_HOME} +-- +-- m{.*\$libdir/pxf.*} +-- s{\$libdir}{\$PXF_HOME/gpextable} +-- +-- end_matchsubs +-- start_ignore +\c pxfautomation_extension +-- end_ignore + +SELECT extversion FROM pg_extension WHERE extname = 'pxf'; + extversion +------------ + 2.1 +(1 row) + +SHOW dynamic_library_path; + dynamic_library_path +---------------------- + $libdir +(1 row) + +SELECT p.proname, p.prosrc, p.probin +FROM pg_catalog.pg_extension AS e + INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) + INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) +WHERE d.deptype = 'e' AND e.extname = 'pxf' +ORDER BY 1; + proname | prosrc | probin +---------------------+------------------------------+---------------------------------- + pxf_read | pxfprotocol_import | $PXF_HOME/gpextable/pxf + pxf_validate | pxfprotocol_validate_urls | $PXF_HOME/gpextable/pxf + pxf_write | pxfprotocol_export | $PXF_HOME/gpextable/pxf + pxfdelimited_import | pxfdelimited_import | $PXF_HOME/gpextable/pxf + pxfwritable_export | gpdbwritableformatter_export | $PXF_HOME/gpextable/pxf + pxfwritable_import | gpdbwritableformatter_import | $PXF_HOME/gpextable/pxf +(6 rows) + +SELECT * FROM pxf_upgrade_test ORDER BY num; + name | num | dub | longnum | bool +--------+-----+-----+---------------+------ + row_1 | 1 | 1 | 100000000000 | f + row_2 | 2 | 2 | 200000000000 | t + row_3 | 3 | 3 | 300000000000 | f + row_4 | 4 | 4 | 400000000000 | t + row_5 | 5 | 5 | 500000000000 | f + row_6 | 6 | 6 | 600000000000 | t + row_7 | 7 | 7 | 700000000000 | f + row_8 | 8 | 8 | 800000000000 | t + row_9 | 9 | 9 | 900000000000 | f + row_10 | 10 | 10 | 1000000000000 | t +(10 rows) + +SELECT * FROM pxf_upgrade_test_multibyte ORDER BY num; + name | num | dub | longnum | bool +--------+-----+-----+---------------+------ + row_1 | 1 | 1 | 100000000000 | f + row_2 | 2 | 2 | 200000000000 | t + row_3 | 3 | 3 | 300000000000 | f + row_4 | 4 | 4 | 400000000000 | t + row_5 | 5 | 5 | 500000000000 | f + row_6 | 6 | 6 | 600000000000 | t + row_7 | 7 | 7 | 700000000000 | f + row_8 | 8 | 8 | 800000000000 | t + row_9 | 9 | 9 | 900000000000 | f + row_10 | 10 | 10 | 1000000000000 | t +(10 rows) diff --git a/automation/tincrepo/main/pxf/features/extension_tests/upgrade/step_2_after_alter_extension/runTest.py b/automation/tincrepo/main/pxf/features/extension_tests/upgrade/step_2_after_alter_extension/runTest.py new file mode 100644 index 0000000000..b5c292ab01 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/extension_tests/upgrade/step_2_after_alter_extension/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfUpgradeStep2AfterAlterExtension(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/extension_tests/upgrade/step_2_after_alter_extension/sql/query01.sql b/automation/tincrepo/main/pxf/features/extension_tests/upgrade/step_2_after_alter_extension/sql/query01.sql new file mode 100644 index 0000000000..5edb69e59b --- /dev/null +++ b/automation/tincrepo/main/pxf/features/extension_tests/upgrade/step_2_after_alter_extension/sql/query01.sql @@ -0,0 +1,28 @@ +-- @description query01 for PXF upgrade test on small data +-- start_matchsubs +-- +-- m{.*/usr/local/pxf-(dev|gp\d).*} +-- s{/usr/local/pxf-(dev|gp\d)}{\$PXF_HOME} +-- +-- m{.*\$libdir/pxf.*} +-- s{\$libdir}{\$PXF_HOME/gpextable} +-- +-- end_matchsubs +-- start_ignore +\c pxfautomation_extension +-- end_ignore + +SELECT extversion FROM pg_extension WHERE extname = 'pxf'; + +SHOW dynamic_library_path; + +SELECT p.proname, p.prosrc, p.probin +FROM pg_catalog.pg_extension AS e + INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) + INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) +WHERE d.deptype = 'e' AND e.extname = 'pxf' +ORDER BY 1; + +SELECT * FROM pxf_upgrade_test ORDER BY num; + +SELECT * FROM pxf_upgrade_test_multibyte ORDER BY num; diff --git a/automation/tincrepo/main/pxf/features/general/alter/csv/expected/query01.ans b/automation/tincrepo/main/pxf/features/general/alter/csv/expected/query01.ans index e0fa1f3e04..387649e390 100755 --- a/automation/tincrepo/main/pxf/features/general/alter/csv/expected/query01.ans +++ b/automation/tincrepo/main/pxf/features/general/alter/csv/expected/query01.ans @@ -18,6 +18,9 @@ -- m/CONTEXT:.*line.*/ -- s/line \d* of //g -- +-- m/tmp\/pxf_automation_data\/(.*)/ +-- s/tmp\/pxf_automation_data\/(.*)/pxf:\/\/pxf_automation_data?PROFILE=*:text/g +-- -- end_matchsubs -- This query should error out with invalid input syntax for integer diff --git a/automation/tincrepo/main/pxf/features/general/alter/csv/sql/query01.sql b/automation/tincrepo/main/pxf/features/general/alter/csv/sql/query01.sql index 71fa13953f..55b4390644 100755 --- a/automation/tincrepo/main/pxf/features/general/alter/csv/sql/query01.sql +++ b/automation/tincrepo/main/pxf/features/general/alter/csv/sql/query01.sql @@ -16,6 +16,9 @@ -- m/CONTEXT:.*line.*/ -- s/line \d* of //g -- +-- m/tmp\/pxf_automation_data\/(.*)/ +-- s/tmp\/pxf_automation_data\/(.*)/pxf:\/\/pxf_automation_data?PROFILE=*:text/g +-- -- end_matchsubs -- This query should error out with invalid input syntax for integer diff --git a/automation/tincrepo/main/pxf/features/general/alter/pxfwritable_import/without_column_projection/expected/query01.ans b/automation/tincrepo/main/pxf/features/general/alter/pxfwritable_import/without_column_projection/expected/query01.ans index 434e1cd97b..270788417e 100755 --- a/automation/tincrepo/main/pxf/features/general/alter/pxfwritable_import/without_column_projection/expected/query01.ans +++ b/automation/tincrepo/main/pxf/features/general/alter/pxfwritable_import/without_column_projection/expected/query01.ans @@ -8,6 +8,9 @@ -- m/DETAIL/ -- s/DETAIL/CONTEXT/ -- +-- m/,.*line (.*)/ +-- s/,.*line (.*)//g +-- -- end_matchsubs -- sets the bytea output to the expected by the tests SET bytea_output='escape'; diff --git a/automation/tincrepo/main/pxf/features/general/alter/pxfwritable_import/without_column_projection/sql/query01.sql b/automation/tincrepo/main/pxf/features/general/alter/pxfwritable_import/without_column_projection/sql/query01.sql index 9faf58a82f..d8230707c8 100755 --- a/automation/tincrepo/main/pxf/features/general/alter/pxfwritable_import/without_column_projection/sql/query01.sql +++ b/automation/tincrepo/main/pxf/features/general/alter/pxfwritable_import/without_column_projection/sql/query01.sql @@ -7,6 +7,9 @@ -- m/DETAIL/ -- s/DETAIL/CONTEXT/ -- +-- m/,.*line (.*)/ +-- s/,.*line (.*)//g +-- -- end_matchsubs -- sets the bytea output to the expected by the tests diff --git a/automation/tincrepo/main/pxf/features/gpupgrade/extension2_0/__init__.py b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_0/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/gpupgrade/extension2_0/step_1_before_running_pxf_pre_gpupgrade/__init__.py b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_0/step_1_before_running_pxf_pre_gpupgrade/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/gpupgrade/step_1_before_running_pxf_pre_gpupgrade/expected/query01.ans b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_0/step_1_before_running_pxf_pre_gpupgrade/expected/query01.ans similarity index 87% rename from automation/tincrepo/main/pxf/features/gpupgrade/step_1_before_running_pxf_pre_gpupgrade/expected/query01.ans rename to automation/tincrepo/main/pxf/features/gpupgrade/extension2_0/step_1_before_running_pxf_pre_gpupgrade/expected/query01.ans index 9963429d73..b5720ece84 100644 --- a/automation/tincrepo/main/pxf/features/gpupgrade/step_1_before_running_pxf_pre_gpupgrade/expected/query01.ans +++ b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_0/step_1_before_running_pxf_pre_gpupgrade/expected/query01.ans @@ -7,6 +7,12 @@ -- s{/usr/local/pxf-(dev|gp\d)}{\$PXF_HOME} -- -- end_matchsubs +SELECT extname, extversion FROM pg_catalog.pg_extension WHERE extname = 'pxf'; + extname | extversion +---------+------------ + pxf | 2.0 +(1 row) + SELECT * FROM pxf_gpupgrade_test; name | num | dub | longnum | bool --------+-----+-----+---------------+------ @@ -34,7 +40,7 @@ FROM pg_catalog.pg_extension AS e INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) WHERE d.deptype = 'e' AND e.extname = 'pxf' ORDER BY 1; - proname | prosrc | probin + proname | prosrc | probin --------------------+------------------------------+---------------------------------- pxf_read | pxfprotocol_import | $PXF_HOME/gpextable/pxf pxf_validate | pxfprotocol_validate_urls | $PXF_HOME/gpextable/pxf @@ -42,4 +48,3 @@ ORDER BY 1; pxfwritable_export | gpdbwritableformatter_export | $PXF_HOME/gpextable/pxf pxfwritable_import | gpdbwritableformatter_import | $PXF_HOME/gpextable/pxf (5 rows) - diff --git a/automation/tincrepo/main/pxf/features/gpupgrade/step_1_before_running_pxf_pre_gpupgrade/runTest.py b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_0/step_1_before_running_pxf_pre_gpupgrade/runTest.py similarity index 100% rename from automation/tincrepo/main/pxf/features/gpupgrade/step_1_before_running_pxf_pre_gpupgrade/runTest.py rename to automation/tincrepo/main/pxf/features/gpupgrade/extension2_0/step_1_before_running_pxf_pre_gpupgrade/runTest.py diff --git a/automation/tincrepo/main/pxf/features/gpupgrade/step_2_after_running_pxf_pre_gpupgrade/sql/query01.sql b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_0/step_1_before_running_pxf_pre_gpupgrade/sql/query01.sql similarity index 86% rename from automation/tincrepo/main/pxf/features/gpupgrade/step_2_after_running_pxf_pre_gpupgrade/sql/query01.sql rename to automation/tincrepo/main/pxf/features/gpupgrade/extension2_0/step_1_before_running_pxf_pre_gpupgrade/sql/query01.sql index 3ec652f336..2b5ba5c029 100644 --- a/automation/tincrepo/main/pxf/features/gpupgrade/step_2_after_running_pxf_pre_gpupgrade/sql/query01.sql +++ b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_0/step_1_before_running_pxf_pre_gpupgrade/sql/query01.sql @@ -6,6 +6,8 @@ -- -- end_matchsubs +SELECT extname, extversion FROM pg_catalog.pg_extension WHERE extname = 'pxf'; + SELECT * FROM pxf_gpupgrade_test; SHOW dynamic_library_path; diff --git a/automation/tincrepo/main/pxf/features/gpupgrade/extension2_0/step_2_after_running_pxf_pre_gpupgrade/__init__.py b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_0/step_2_after_running_pxf_pre_gpupgrade/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/gpupgrade/step_2_after_running_pxf_pre_gpupgrade/expected/query01.ans b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_0/step_2_after_running_pxf_pre_gpupgrade/expected/query01.ans similarity index 87% rename from automation/tincrepo/main/pxf/features/gpupgrade/step_2_after_running_pxf_pre_gpupgrade/expected/query01.ans rename to automation/tincrepo/main/pxf/features/gpupgrade/extension2_0/step_2_after_running_pxf_pre_gpupgrade/expected/query01.ans index dcbf6882d7..1dc81e3553 100644 --- a/automation/tincrepo/main/pxf/features/gpupgrade/step_2_after_running_pxf_pre_gpupgrade/expected/query01.ans +++ b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_0/step_2_after_running_pxf_pre_gpupgrade/expected/query01.ans @@ -7,6 +7,12 @@ -- s{/usr/local/pxf-(dev|gp\d)}{\$PXF_HOME} -- -- end_matchsubs +SELECT extname, extversion FROM pg_catalog.pg_extension WHERE extname = 'pxf'; + extname | extversion +---------+------------ + pxf | 2.0 +(1 row) + SELECT * FROM pxf_gpupgrade_test; name | num | dub | longnum | bool --------+-----+-----+---------------+------ @@ -34,7 +40,7 @@ FROM pg_catalog.pg_extension AS e INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) WHERE d.deptype = 'e' AND e.extname = 'pxf' ORDER BY 1; - proname | prosrc | probin + proname | prosrc | probin --------------------+------------------------------+-------- pxf_read | pxfprotocol_import | pxf pxf_validate | pxfprotocol_validate_urls | pxf @@ -42,4 +48,3 @@ ORDER BY 1; pxfwritable_export | gpdbwritableformatter_export | pxf pxfwritable_import | gpdbwritableformatter_import | pxf (5 rows) - diff --git a/automation/tincrepo/main/pxf/features/gpupgrade/step_2_after_running_pxf_pre_gpupgrade/runTest.py b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_0/step_2_after_running_pxf_pre_gpupgrade/runTest.py similarity index 100% rename from automation/tincrepo/main/pxf/features/gpupgrade/step_2_after_running_pxf_pre_gpupgrade/runTest.py rename to automation/tincrepo/main/pxf/features/gpupgrade/extension2_0/step_2_after_running_pxf_pre_gpupgrade/runTest.py diff --git a/automation/tincrepo/main/pxf/features/gpupgrade/step_3_after_running_pxf_post_gpupgrade/sql/query01.sql b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_0/step_2_after_running_pxf_pre_gpupgrade/sql/query01.sql similarity index 86% rename from automation/tincrepo/main/pxf/features/gpupgrade/step_3_after_running_pxf_post_gpupgrade/sql/query01.sql rename to automation/tincrepo/main/pxf/features/gpupgrade/extension2_0/step_2_after_running_pxf_pre_gpupgrade/sql/query01.sql index 3ec652f336..2b5ba5c029 100644 --- a/automation/tincrepo/main/pxf/features/gpupgrade/step_3_after_running_pxf_post_gpupgrade/sql/query01.sql +++ b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_0/step_2_after_running_pxf_pre_gpupgrade/sql/query01.sql @@ -6,6 +6,8 @@ -- -- end_matchsubs +SELECT extname, extversion FROM pg_catalog.pg_extension WHERE extname = 'pxf'; + SELECT * FROM pxf_gpupgrade_test; SHOW dynamic_library_path; diff --git a/automation/tincrepo/main/pxf/features/gpupgrade/extension2_0/step_3_after_running_pxf_post_gpupgrade/__init__.py b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_0/step_3_after_running_pxf_post_gpupgrade/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/gpupgrade/step_3_after_running_pxf_post_gpupgrade/expected/query01.ans b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_0/step_3_after_running_pxf_post_gpupgrade/expected/query01.ans similarity index 87% rename from automation/tincrepo/main/pxf/features/gpupgrade/step_3_after_running_pxf_post_gpupgrade/expected/query01.ans rename to automation/tincrepo/main/pxf/features/gpupgrade/extension2_0/step_3_after_running_pxf_post_gpupgrade/expected/query01.ans index fe392964f8..7b7484635e 100644 --- a/automation/tincrepo/main/pxf/features/gpupgrade/step_3_after_running_pxf_post_gpupgrade/expected/query01.ans +++ b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_0/step_3_after_running_pxf_post_gpupgrade/expected/query01.ans @@ -5,6 +5,12 @@ -- s{/usr/local/pxf-(dev|gp\d)}{\$PXF_HOME} -- -- end_matchsubs +SELECT extname, extversion FROM pg_catalog.pg_extension WHERE extname = 'pxf'; + extname | extversion +---------+------------ + pxf | 2.0 +(1 row) + SELECT * FROM pxf_gpupgrade_test; name | num | dub | longnum | bool --------+-----+-----+---------------+------ @@ -32,7 +38,7 @@ FROM pg_catalog.pg_extension AS e INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) WHERE d.deptype = 'e' AND e.extname = 'pxf' ORDER BY 1; - proname | prosrc | probin + proname | prosrc | probin --------------------+------------------------------+---------------------------------- pxf_read | pxfprotocol_import | $PXF_HOME/gpextable/pxf pxf_validate | pxfprotocol_validate_urls | $PXF_HOME/gpextable/pxf @@ -40,4 +46,3 @@ ORDER BY 1; pxfwritable_export | gpdbwritableformatter_export | $PXF_HOME/gpextable/pxf pxfwritable_import | gpdbwritableformatter_import | $PXF_HOME/gpextable/pxf (5 rows) - diff --git a/automation/tincrepo/main/pxf/features/gpupgrade/step_3_after_running_pxf_post_gpupgrade/runTest.py b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_0/step_3_after_running_pxf_post_gpupgrade/runTest.py similarity index 100% rename from automation/tincrepo/main/pxf/features/gpupgrade/step_3_after_running_pxf_post_gpupgrade/runTest.py rename to automation/tincrepo/main/pxf/features/gpupgrade/extension2_0/step_3_after_running_pxf_post_gpupgrade/runTest.py diff --git a/automation/tincrepo/main/pxf/features/gpupgrade/step_1_before_running_pxf_pre_gpupgrade/sql/query01.sql b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_0/step_3_after_running_pxf_post_gpupgrade/sql/query01.sql similarity index 86% rename from automation/tincrepo/main/pxf/features/gpupgrade/step_1_before_running_pxf_pre_gpupgrade/sql/query01.sql rename to automation/tincrepo/main/pxf/features/gpupgrade/extension2_0/step_3_after_running_pxf_post_gpupgrade/sql/query01.sql index 3ec652f336..2b5ba5c029 100644 --- a/automation/tincrepo/main/pxf/features/gpupgrade/step_1_before_running_pxf_pre_gpupgrade/sql/query01.sql +++ b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_0/step_3_after_running_pxf_post_gpupgrade/sql/query01.sql @@ -6,6 +6,8 @@ -- -- end_matchsubs +SELECT extname, extversion FROM pg_catalog.pg_extension WHERE extname = 'pxf'; + SELECT * FROM pxf_gpupgrade_test; SHOW dynamic_library_path; diff --git a/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/__init__.py b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/step_1_before_running_pxf_pre_gpupgrade/__init__.py b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/step_1_before_running_pxf_pre_gpupgrade/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/step_1_before_running_pxf_pre_gpupgrade/expected/query01.ans b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/step_1_before_running_pxf_pre_gpupgrade/expected/query01.ans new file mode 100644 index 0000000000..36314ef52c --- /dev/null +++ b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/step_1_before_running_pxf_pre_gpupgrade/expected/query01.ans @@ -0,0 +1,52 @@ +-- start_ignore +-- end_ignore +-- @description query01 for PXF gpupgrade test on small data +-- start_matchsubs +-- +-- m{.*/usr/local/pxf-(dev|gp\d).*} +-- s{/usr/local/pxf-(dev|gp\d)}{\$PXF_HOME} +-- +-- end_matchsubs +SELECT extname, extversion FROM pg_catalog.pg_extension WHERE extname = 'pxf'; + extname | extversion +---------+------------ + pxf | 2.1 +(1 row) + +SELECT * FROM pxf_gpupgrade_test; + name | num | dub | longnum | bool +--------+-----+-----+---------------+------ + row_1 | 1 | 1 | 100000000000 | f + row_2 | 2 | 2 | 200000000000 | t + row_3 | 3 | 3 | 300000000000 | f + row_4 | 4 | 4 | 400000000000 | t + row_5 | 5 | 5 | 500000000000 | f + row_6 | 6 | 6 | 600000000000 | t + row_7 | 7 | 7 | 700000000000 | f + row_8 | 8 | 8 | 800000000000 | t + row_9 | 9 | 9 | 900000000000 | f + row_10 | 10 | 10 | 1000000000000 | t +(10 rows) + +SHOW dynamic_library_path; + dynamic_library_path +---------------------- + $libdir +(1 row) + +SELECT p.proname, p.prosrc, p.probin +FROM pg_catalog.pg_extension AS e + INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) + INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) +WHERE d.deptype = 'e' AND e.extname = 'pxf' +ORDER BY 1; + proname | prosrc | probin +---------------------+------------------------------+---------------------------------- + pxf_read | pxfprotocol_import | $PXF_HOME/gpextable/pxf + pxf_validate | pxfprotocol_validate_urls | $PXF_HOME/gpextable/pxf + pxf_write | pxfprotocol_export | $PXF_HOME/gpextable/pxf + pxfdelimited_import | pxfdelimited_import | $PXF_HOME/gpextable/pxf + pxfwritable_export | gpdbwritableformatter_export | $PXF_HOME/gpextable/pxf + pxfwritable_import | gpdbwritableformatter_import | $PXF_HOME/gpextable/pxf +(6 rows) + diff --git a/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/step_1_before_running_pxf_pre_gpupgrade/runTest.py b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/step_1_before_running_pxf_pre_gpupgrade/runTest.py new file mode 100644 index 0000000000..e30838ab3d --- /dev/null +++ b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/step_1_before_running_pxf_pre_gpupgrade/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfGpupgradeStep1BeforeRunningPxfPreGpupgrade(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/step_1_before_running_pxf_pre_gpupgrade/sql/query01.sql b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/step_1_before_running_pxf_pre_gpupgrade/sql/query01.sql new file mode 100644 index 0000000000..2b5ba5c029 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/step_1_before_running_pxf_pre_gpupgrade/sql/query01.sql @@ -0,0 +1,20 @@ +-- @description query01 for PXF gpupgrade test on small data +-- start_matchsubs +-- +-- m{.*/usr/local/pxf-(dev|gp\d).*} +-- s{/usr/local/pxf-(dev|gp\d)}{\$PXF_HOME} +-- +-- end_matchsubs + +SELECT extname, extversion FROM pg_catalog.pg_extension WHERE extname = 'pxf'; + +SELECT * FROM pxf_gpupgrade_test; + +SHOW dynamic_library_path; + +SELECT p.proname, p.prosrc, p.probin +FROM pg_catalog.pg_extension AS e + INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) + INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) +WHERE d.deptype = 'e' AND e.extname = 'pxf' +ORDER BY 1; \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/step_2_after_running_pxf_pre_gpupgrade/__init__.py b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/step_2_after_running_pxf_pre_gpupgrade/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/step_2_after_running_pxf_pre_gpupgrade/expected/query01.ans b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/step_2_after_running_pxf_pre_gpupgrade/expected/query01.ans new file mode 100644 index 0000000000..57fb6851b1 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/step_2_after_running_pxf_pre_gpupgrade/expected/query01.ans @@ -0,0 +1,52 @@ +-- start_ignore +-- end_ignore +-- @description query01 for PXF gpupgrade test on small data +-- start_matchsubs +-- +-- m{.*/usr/local/pxf-(dev|gp\d).*} +-- s{/usr/local/pxf-(dev|gp\d)}{\$PXF_HOME} +-- +-- end_matchsubs +SELECT extname, extversion FROM pg_catalog.pg_extension WHERE extname = 'pxf'; + extname | extversion +---------+------------ + pxf | 2.1 +(1 row) + +SELECT * FROM pxf_gpupgrade_test; + name | num | dub | longnum | bool +--------+-----+-----+---------------+------ + row_1 | 1 | 1 | 100000000000 | f + row_2 | 2 | 2 | 200000000000 | t + row_3 | 3 | 3 | 300000000000 | f + row_4 | 4 | 4 | 400000000000 | t + row_5 | 5 | 5 | 500000000000 | f + row_6 | 6 | 6 | 600000000000 | t + row_7 | 7 | 7 | 700000000000 | f + row_8 | 8 | 8 | 800000000000 | t + row_9 | 9 | 9 | 900000000000 | f + row_10 | 10 | 10 | 1000000000000 | t +(10 rows) + +SHOW dynamic_library_path; + dynamic_library_path +---------------------- + $PXF_HOME/gpextable:$libdir +(1 row) + +SELECT p.proname, p.prosrc, p.probin +FROM pg_catalog.pg_extension AS e + INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) + INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) +WHERE d.deptype = 'e' AND e.extname = 'pxf' +ORDER BY 1; + proname | prosrc | probin +---------------------+------------------------------+-------- + pxf_read | pxfprotocol_import | pxf + pxf_validate | pxfprotocol_validate_urls | pxf + pxf_write | pxfprotocol_export | pxf + pxfdelimited_import | pxfdelimited_import | pxf + pxfwritable_export | gpdbwritableformatter_export | pxf + pxfwritable_import | gpdbwritableformatter_import | pxf +(6 rows) + diff --git a/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/step_2_after_running_pxf_pre_gpupgrade/runTest.py b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/step_2_after_running_pxf_pre_gpupgrade/runTest.py new file mode 100644 index 0000000000..c95831f419 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/step_2_after_running_pxf_pre_gpupgrade/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfGpupgradeStep2AfterRunningPxfPreGpupgrade(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/step_2_after_running_pxf_pre_gpupgrade/sql/query01.sql b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/step_2_after_running_pxf_pre_gpupgrade/sql/query01.sql new file mode 100644 index 0000000000..2b5ba5c029 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/step_2_after_running_pxf_pre_gpupgrade/sql/query01.sql @@ -0,0 +1,20 @@ +-- @description query01 for PXF gpupgrade test on small data +-- start_matchsubs +-- +-- m{.*/usr/local/pxf-(dev|gp\d).*} +-- s{/usr/local/pxf-(dev|gp\d)}{\$PXF_HOME} +-- +-- end_matchsubs + +SELECT extname, extversion FROM pg_catalog.pg_extension WHERE extname = 'pxf'; + +SELECT * FROM pxf_gpupgrade_test; + +SHOW dynamic_library_path; + +SELECT p.proname, p.prosrc, p.probin +FROM pg_catalog.pg_extension AS e + INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) + INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) +WHERE d.deptype = 'e' AND e.extname = 'pxf' +ORDER BY 1; \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/step_3_after_running_pxf_post_gpupgrade/__init__.py b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/step_3_after_running_pxf_post_gpupgrade/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/step_3_after_running_pxf_post_gpupgrade/expected/query01.ans b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/step_3_after_running_pxf_post_gpupgrade/expected/query01.ans new file mode 100644 index 0000000000..88833d6376 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/step_3_after_running_pxf_post_gpupgrade/expected/query01.ans @@ -0,0 +1,50 @@ +-- @description query01 for PXF gpupgrade test on small data +-- start_matchsubs +-- +-- m{.*/usr/local/pxf-(dev|gp\d).*} +-- s{/usr/local/pxf-(dev|gp\d)}{\$PXF_HOME} +-- +-- end_matchsubs +SELECT extname, extversion FROM pg_catalog.pg_extension WHERE extname = 'pxf'; + extname | extversion +---------+------------ + pxf | 2.1 +(1 row) + +SELECT * FROM pxf_gpupgrade_test; + name | num | dub | longnum | bool +--------+-----+-----+---------------+------ + row_1 | 1 | 1 | 100000000000 | f + row_2 | 2 | 2 | 200000000000 | t + row_3 | 3 | 3 | 300000000000 | f + row_4 | 4 | 4 | 400000000000 | t + row_5 | 5 | 5 | 500000000000 | f + row_6 | 6 | 6 | 600000000000 | t + row_7 | 7 | 7 | 700000000000 | f + row_8 | 8 | 8 | 800000000000 | t + row_9 | 9 | 9 | 900000000000 | f + row_10 | 10 | 10 | 1000000000000 | t +(10 rows) + +SHOW dynamic_library_path; + dynamic_library_path +---------------------- + $libdir +(1 row) + +SELECT p.proname, p.prosrc, p.probin +FROM pg_catalog.pg_extension AS e + INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) + INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) +WHERE d.deptype = 'e' AND e.extname = 'pxf' +ORDER BY 1; + proname | prosrc | probin +---------------------+------------------------------+---------------------------------- + pxf_read | pxfprotocol_import | $PXF_HOME/gpextable/pxf + pxf_validate | pxfprotocol_validate_urls | $PXF_HOME/gpextable/pxf + pxf_write | pxfprotocol_export | $PXF_HOME/gpextable/pxf + pxfdelimited_import | pxfdelimited_import | $PXF_HOME/gpextable/pxf + pxfwritable_export | gpdbwritableformatter_export | $PXF_HOME/gpextable/pxf + pxfwritable_import | gpdbwritableformatter_import | $PXF_HOME/gpextable/pxf +(6 rows) + diff --git a/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/step_3_after_running_pxf_post_gpupgrade/runTest.py b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/step_3_after_running_pxf_post_gpupgrade/runTest.py new file mode 100644 index 0000000000..6742df6f97 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/step_3_after_running_pxf_post_gpupgrade/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfGpupgradeStep3AfterRunningPxfPostGpupgrade(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/step_3_after_running_pxf_post_gpupgrade/sql/query01.sql b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/step_3_after_running_pxf_post_gpupgrade/sql/query01.sql new file mode 100644 index 0000000000..2b5ba5c029 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/gpupgrade/extension2_1/step_3_after_running_pxf_post_gpupgrade/sql/query01.sql @@ -0,0 +1,20 @@ +-- @description query01 for PXF gpupgrade test on small data +-- start_matchsubs +-- +-- m{.*/usr/local/pxf-(dev|gp\d).*} +-- s{/usr/local/pxf-(dev|gp\d)}{\$PXF_HOME} +-- +-- end_matchsubs + +SELECT extname, extversion FROM pg_catalog.pg_extension WHERE extname = 'pxf'; + +SELECT * FROM pxf_gpupgrade_test; + +SHOW dynamic_library_path; + +SELECT p.proname, p.prosrc, p.probin +FROM pg_catalog.pg_extension AS e + INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) + INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) +WHERE d.deptype = 'e' AND e.extname = 'pxf' +ORDER BY 1; \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/hcfs/globbing/match_string_from_string_set/expected/query04.ans b/automation/tincrepo/main/pxf/features/hcfs/globbing/match_string_from_string_set/expected/query04.ans index 4db9b3394d..aa7d90ac41 100755 --- a/automation/tincrepo/main/pxf/features/hcfs/globbing/match_string_from_string_set/expected/query04.ans +++ b/automation/tincrepo/main/pxf/features/hcfs/globbing/match_string_from_string_set/expected/query04.ans @@ -244,6 +244,9 @@ select * from hcfs_glob_match_string_from_string_set_10 order by name, num; -- m/CONTEXT:.*line.*/ -- s/line \d* of //g -- +-- m/CONTEXT:.*file.*/ +-- s/, file.*//g +-- -- end_matchsubs select * from hcfs_glob_match_string_from_string_set_11 order by name, num; ERROR: PXF server error : Illegal file pattern: Unclosed group near index xxx diff --git a/automation/tincrepo/main/pxf/features/hcfs/globbing/match_string_from_string_set/sql/query04.sql b/automation/tincrepo/main/pxf/features/hcfs/globbing/match_string_from_string_set/sql/query04.sql index 0039dd0f91..98d2aed3dd 100755 --- a/automation/tincrepo/main/pxf/features/hcfs/globbing/match_string_from_string_set/sql/query04.sql +++ b/automation/tincrepo/main/pxf/features/hcfs/globbing/match_string_from_string_set/sql/query04.sql @@ -49,6 +49,9 @@ select * from hcfs_glob_match_string_from_string_set_10 order by name, num; -- m/CONTEXT:.*line.*/ -- s/line \d* of //g -- +-- m/CONTEXT:.*file.*/ +-- s/, file.*//g +-- -- end_matchsubs select * from hcfs_glob_match_string_from_string_set_11 order by name, num; diff --git a/automation/tincrepo/main/pxf/features/hdfs/readable/avro/errors/no_schema_file/expected/query01.ans b/automation/tincrepo/main/pxf/features/hdfs/readable/avro/errors/no_schema_file/expected/query01.ans index 4ef64b6c1f..2ce2248dfd 100755 --- a/automation/tincrepo/main/pxf/features/hdfs/readable/avro/errors/no_schema_file/expected/query01.ans +++ b/automation/tincrepo/main/pxf/features/hdfs/readable/avro/errors/no_schema_file/expected/query01.ans @@ -13,6 +13,9 @@ -- m/(E|e)xception (r|R)eport +(m|M)essage/ -- s/(E|e)xception (r|R)eport +(m|M)essage/exception report message/ -- +-- m/, file .*/ +-- s/, file .*// +-- -- end_matchsubs SELECT * from avro_in_seq_no_schema; ERROR: PXF server error : Failed to obtain Avro schema from 'i_do_not_exist' diff --git a/automation/tincrepo/main/pxf/features/hdfs/readable/avro/errors/no_schema_file/sql/query01.sql b/automation/tincrepo/main/pxf/features/hdfs/readable/avro/errors/no_schema_file/sql/query01.sql index 74e9af38fa..0f3269a3ed 100755 --- a/automation/tincrepo/main/pxf/features/hdfs/readable/avro/errors/no_schema_file/sql/query01.sql +++ b/automation/tincrepo/main/pxf/features/hdfs/readable/avro/errors/no_schema_file/sql/query01.sql @@ -13,5 +13,8 @@ -- m/(E|e)xception (r|R)eport +(m|M)essage/ -- s/(E|e)xception (r|R)eport +(m|M)essage/exception report message/ -- +-- m/, file .*/ +-- s/, file .*// +-- -- end_matchsubs SELECT * from avro_in_seq_no_schema; diff --git a/automation/tincrepo/main/pxf/features/hdfs/readable/json/array_as_text/expected/query01.ans b/automation/tincrepo/main/pxf/features/hdfs/readable/json/array_as_text/expected/query01.ans index 0beb690c7d..9d46d22ee4 100644 --- a/automation/tincrepo/main/pxf/features/hdfs/readable/json/array_as_text/expected/query01.ans +++ b/automation/tincrepo/main/pxf/features/hdfs/readable/json/array_as_text/expected/query01.ans @@ -3,26 +3,26 @@ Null display is "NIL". SELECT * FROM jsontest_array_as_text ORDER BY id; id | emp_arr | emp_obj | num_arr | bool_arr | str_arr | arr_arr | obj_arr | obj -----+---------+---------+----------------------------------+-------------------+--------------------------+------------------------+------------------------+----------------------------------------------------------------------------------------------------- - 0 | [] | {} | [null,1,-1.3,1.2345678901234567] | [null,true,false] | [null,"hello","wor\"ld"] | [null,["a","b"],[1,2]] | [null,{"a":1},{"a":2}] | {"level":0,"key":"a","data":{"level":1,"key":"b","data":{"level":2,"key":"x","data":["wow",null]}}} - 1 | [] | {} | [2,null,-2.3,2.234567890123457] | [true,null,false] | ["hello",null,"wor\"ld"] | [["a","b"],null,[1,2]] | [{"a":1},null,{"a":2}] | {"level":0,"key":"a","data":{"level":1,"key":"b","data":{"level":2,"key":"y","data":["wow",null]}}} - 2 | [] | {} | [3,-3.3,null,3.234567890123457] | [true,false,null] | ["hello","wor\"ld",null] | [["a","b"],[1,2],null] | [{"a":1},{"a":2},null] | {"level":0,"key":"a","data":{"level":1,"key":"b","data":{"level":2,"key":"z","data":["wow",null]}}} +----+---------+---------+------------------------------------------+-------------------+--------------------------+------------------------+------------------------+----------------------------------------------------------------------------------------------------- + 0 | [] | {} | [null,1,-1.3,1.234567890123456789012345] | [null,true,false] | [null,"hello","wor\"ld"] | [null,["a","b"],[1,2]] | [null,{"a":1},{"a":2}] | {"level":0,"key":"a","data":{"level":1,"key":"b","data":{"level":2,"key":"x","data":["wow",null]}}} + 1 | [] | {} | [2,null,-2.3,2.234567890123456789012345] | [true,null,false] | ["hello",null,"wor\"ld"] | [["a","b"],null,[1,2]] | [{"a":1},null,{"a":2}] | {"level":0,"key":"a","data":{"level":1,"key":"b","data":{"level":2,"key":"y","data":["wow",null]}}} + 2 | [] | {} | [3,-3.3,null,3.234567890123456789012345] | [true,false,null] | ["hello","wor\"ld",null] | [["a","b"],[1,2],null] | [{"a":1},{"a":2},null] | {"level":0,"key":"a","data":{"level":1,"key":"b","data":{"level":2,"key":"z","data":["wow",null]}}} (3 rows) SELECT * FROM jsontest_array_as_varchar ORDER BY id; id | emp_arr | emp_obj | num_arr | bool_arr | str_arr | arr_arr | obj_arr | obj -----+---------+---------+----------------------------------+-------------------+--------------------------+------------------------+------------------------+----------------------------------------------------------------------------------------------------- - 0 | [] | {} | [null,1,-1.3,1.2345678901234567] | [null,true,false] | [null,"hello","wor\"ld"] | [null,["a","b"],[1,2]] | [null,{"a":1},{"a":2}] | {"level":0,"key":"a","data":{"level":1,"key":"b","data":{"level":2,"key":"x","data":["wow",null]}}} - 1 | [] | {} | [2,null,-2.3,2.234567890123457] | [true,null,false] | ["hello",null,"wor\"ld"] | [["a","b"],null,[1,2]] | [{"a":1},null,{"a":2}] | {"level":0,"key":"a","data":{"level":1,"key":"b","data":{"level":2,"key":"y","data":["wow",null]}}} - 2 | [] | {} | [3,-3.3,null,3.234567890123457] | [true,false,null] | ["hello","wor\"ld",null] | [["a","b"],[1,2],null] | [{"a":1},{"a":2},null] | {"level":0,"key":"a","data":{"level":1,"key":"b","data":{"level":2,"key":"z","data":["wow",null]}}} +----+---------+---------+------------------------------------------+-------------------+--------------------------+------------------------+------------------------+----------------------------------------------------------------------------------------------------- + 0 | [] | {} | [null,1,-1.3,1.234567890123456789012345] | [null,true,false] | [null,"hello","wor\"ld"] | [null,["a","b"],[1,2]] | [null,{"a":1},{"a":2}] | {"level":0,"key":"a","data":{"level":1,"key":"b","data":{"level":2,"key":"x","data":["wow",null]}}} + 1 | [] | {} | [2,null,-2.3,2.234567890123456789012345] | [true,null,false] | ["hello",null,"wor\"ld"] | [["a","b"],null,[1,2]] | [{"a":1},null,{"a":2}] | {"level":0,"key":"a","data":{"level":1,"key":"b","data":{"level":2,"key":"y","data":["wow",null]}}} + 2 | [] | {} | [3,-3.3,null,3.234567890123456789012345] | [true,false,null] | ["hello","wor\"ld",null] | [["a","b"],[1,2],null] | [{"a":1},{"a":2},null] | {"level":0,"key":"a","data":{"level":1,"key":"b","data":{"level":2,"key":"z","data":["wow",null]}}} (3 rows) SELECT * FROM jsontest_array_as_bpchar ORDER BY id; id | emp_arr | emp_obj | num_arr | bool_arr | str_arr | arr_arr | obj_arr | obj ----+---------+------------+--------------------------------------------+-------------------+---------------------------+---------------------------+-------------------------+------------------------------------------------------------------------------------------------------ - 0 | [] | {} | [null,1,-1.3,1.2345678901234567] | [null,true,false] | [null,"hello","wor\"ld"] | [null,["a","b"],[1,2]] | [null,{"a":1},{"a":2}] | {"level":0,"key":"a","data":{"level":1,"key":"b","data":{"level":2,"key":"x","data":["wow",null]}}} - 1 | [] | {} | [2,null,-2.3,2.234567890123457] | [true,null,false] | ["hello",null,"wor\"ld"] | [["a","b"],null,[1,2]] | [{"a":1},null,{"a":2}] | {"level":0,"key":"a","data":{"level":1,"key":"b","data":{"level":2,"key":"y","data":["wow",null]}}} - 2 | [] | {} | [3,-3.3,null,3.234567890123457] | [true,false,null] | ["hello","wor\"ld",null] | [["a","b"],[1,2],null] | [{"a":1},{"a":2},null] | {"level":0,"key":"a","data":{"level":1,"key":"b","data":{"level":2,"key":"z","data":["wow",null]}}} + 0 | [] | {} | [null,1,-1.3,1.234567890123456789012345] | [null,true,false] | [null,"hello","wor\"ld"] | [null,["a","b"],[1,2]] | [null,{"a":1},{"a":2}] | {"level":0,"key":"a","data":{"level":1,"key":"b","data":{"level":2,"key":"x","data":["wow",null]}}} + 1 | [] | {} | [2,null,-2.3,2.234567890123456789012345] | [true,null,false] | ["hello",null,"wor\"ld"] | [["a","b"],null,[1,2]] | [{"a":1},null,{"a":2}] | {"level":0,"key":"a","data":{"level":1,"key":"b","data":{"level":2,"key":"y","data":["wow",null]}}} + 2 | [] | {} | [3,-3.3,null,3.234567890123456789012345] | [true,false,null] | ["hello","wor\"ld",null] | [["a","b"],[1,2],null] | [{"a":1},{"a":2},null] | {"level":0,"key":"a","data":{"level":1,"key":"b","data":{"level":2,"key":"z","data":["wow",null]}}} (3 rows) SELECT id, ((num_arr::json)->>0)::integer "num", ((bool_arr::json)->>1)::boolean "bool", (str_arr::json)->>2 "str", (arr_arr::json)->>0 "arr", diff --git a/automation/tincrepo/main/pxf/features/hdfs/readable/json/json_functions/expected/query01.ans b/automation/tincrepo/main/pxf/features/hdfs/readable/json/json_functions/expected/query01.ans index 52bf00b188..9c8fb60379 100644 --- a/automation/tincrepo/main/pxf/features/hdfs/readable/json/json_functions/expected/query01.ans +++ b/automation/tincrepo/main/pxf/features/hdfs/readable/json/json_functions/expected/query01.ans @@ -10,9 +10,9 @@ SELECT id, FROM jsontest_array_as_text ORDER BY id; id | numbers | booleans | strings | arrays -----+----------------------------------+------------+------------------------+-------------------------------- - 0 | {NULL,1,-1.3,1.2345678901234567} | {NULL,t,f} | {NULL,hello,"wor\"ld"} | {NULL,"[\"a\",\"b\"]","[1,2]"} - 1 | {2,NULL,-2.3,2.234567890123457} | {t,NULL,f} | {hello,NULL,"wor\"ld"} | {"[\"a\",\"b\"]",NULL,"[1,2]"} - 2 | {3,-3.3,NULL,3.234567890123457} | {t,f,NULL} | {hello,"wor\"ld",NULL} | {"[\"a\",\"b\"]","[1,2]",NULL} +----+------------------------------------------+------------+------------------------+-------------------------------- + 0 | {NULL,1,-1.3,1.234567890123456789012345} | {NULL,t,f} | {NULL,hello,"wor\"ld"} | {NULL,"[\"a\",\"b\"]","[1,2]"} + 1 | {2,NULL,-2.3,2.234567890123456789012345} | {t,NULL,f} | {hello,NULL,"wor\"ld"} | {"[\"a\",\"b\"]",NULL,"[1,2]"} + 2 | {3,-3.3,NULL,3.234567890123456789012345} | {t,f,NULL} | {hello,"wor\"ld",NULL} | {"[\"a\",\"b\"]","[1,2]",NULL} (3 rows) diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/__init__.py b/automation/tincrepo/main/pxf/features/hdfs/writable/json/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/array_types_object/__init__.py b/automation/tincrepo/main/pxf/features/hdfs/writable/json/array_types_object/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/array_types_object/expected/query01.ans b/automation/tincrepo/main/pxf/features/hdfs/writable/json/array_types_object/expected/query01.ans new file mode 100755 index 0000000000..8ae1065c87 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/hdfs/writable/json/array_types_object/expected/query01.ans @@ -0,0 +1,60 @@ +-- @description query01 for PXF HDFS Writable Json array types written in the object layout + +-- start_matchsubs +-- +-- # create a match/subs +-- +-- end_matchsubs +\pset null 'NIL' +Null display is "NIL". +SET bytea_output = 'hex'; +SET +SET TIME ZONE 'America/Los_Angeles'; +SET +SHOW TIME ZONE; + TimeZone +--------------------- + America/Los_Angeles +(1 row) + +SELECT id, name, sml, integ, bg, r, dp, dec, bool, cdate, ctime, tm, tmz, c1, vc1, bin from gpdb_array_types ORDER BY id; + id | name | sml | integ | bg | r | dp | dec | bool | cdate | ctime | tm | tmz | c1 | vc1 | bin +----+-----------------------------------------+------------+-------------------------+-------------------------------+-----------------------+-------------------------------------------+-----------------------------------------------------+------------+------------------------------+--------------------------+------------------------------------------------------------------+------------------------------------------------------------------------+----------------+--------------------+-------------------------------- + 0 | {row-00,"esc1\"","esc2\\","esc3}",NULL} | {0,1,NULL} | {1000000,-1000000,NULL} | {5555500000,-5555500000,NULL} | {0.0001,-0.0001,NULL} | {3.14159265358979,-3.14159265358979,NULL} | {12345678900000.000000,-12345678900000.000000,NULL} | {t,f,NULL} | {2010-01-01,2010-01-01,NULL} | {10:11:00,11:11:00,NULL} | {"2013-07-13 21:00:05.000456","2014-07-13 21:00:05.000456",NULL} | {"2013-07-13 21:00:05.000123-07","2014-07-13 21:00:05.000123-07",NULL} | {abc,cba,NULL} | {defhi,jklmn,NULL} | {"\\x622d30","\\x632d30",NULL} + 1 | NIL | {} | {1000001} | {NULL} | {1.0001,-1.0001} | {3.14159265358979,-3.14159265358979} | {12345678900000.000001,-12345678900000.000001} | {t,f} | {2010-01-02,2010-02-02} | {10:11:01,11:11:01} | {"2013-07-13 21:00:05.001456","2014-07-13 21:00:05.001456"} | {"2013-07-13 21:00:05.001123-07","2014-07-13 21:00:05.001123-07"} | {abc,cba,NULL} | {defhi,NULL,jklmn} | {NULL,"\\x622d31","\\x632d31"} + 2 | {NULL,"esc1\"","esc2\\","esc3}"} | NIL | {} | {5555500002} | {NULL} | {3.14159265358979,-3.14159265358979} | {12345678900000.000002,-12345678900000.000002} | {t,f} | {2010-01-03,2010-02-03} | {10:11:02,11:11:02} | {"2013-07-13 21:00:05.002456","2014-07-13 21:00:05.002456"} | {"2013-07-13 21:00:05.002123-07","2014-07-13 21:00:05.002123-07"} | {abc,cba} | {defhi,jklmn,NULL} | {"\\x622d32",NULL,"\\x632d32"} + 3 | {row-03,NULL,"esc2\\","esc3}"} | {NULL,3,4} | NIL | {} | {-3.0001} | {NULL} | {12345678900000.000003,-12345678900000.000003} | {t,f} | {2010-01-04,2010-02-04} | {10:11:03,11:11:03} | {"2013-07-13 21:00:05.003456","2014-07-13 21:00:05.003456"} | {"2013-07-13 21:00:05.003123-07","2014-07-13 21:00:05.003123-07"} | {abc,cba} | {defhi,jklmn} | {"\\x622d33","\\x632d33",NULL} + 4 | {row-04,"esc1\"",NULL,"esc3}"} | {4,NULL,5} | {NULL,1000004,-1000004} | NIL | {} | {3.14159265358979} | {NULL} | {t,f} | {2010-01-05,2010-02-05} | {10:11:04,11:11:04} | {"2013-07-13 21:00:05.004456","2014-07-13 21:00:05.004456"} | {"2013-07-13 21:00:05.004123-07","2014-07-13 21:00:05.004123-07"} | {abc,cba} | {defhi,jklmn} | {"\\x622d34","\\x632d34"} + 5 | {row-05,"esc1\"","esc2\\",NULL} | {5,6,NULL} | {1000005,NULL,-1000005} | {NULL,5555500005,-5555500005} | NIL | {} | {-12345678900000.000005} | {NULL} | {2010-01-06,2010-02-06} | {10:11:05,11:11:05} | {"2013-07-13 21:00:05.005456","2014-07-13 21:00:05.005456"} | {"2013-07-13 21:00:05.005123-07","2014-07-13 21:00:05.005123-07"} | {abc,cba} | {defhi,jklmn} | {"\\x622d35","\\x632d35"} + 6 | {row-06,"esc1\"","esc2\\","esc3}"} | {6,7} | {1000006,-1000006,NULL} | {5555500006,NULL,-5555500006} | {NULL,6.0001,-6.0001} | NIL | {} | {t} | {NULL} | {10:11:06,11:11:06} | {"2013-07-13 21:00:05.006456","2014-07-13 21:00:05.006456"} | {"2013-07-13 21:00:05.006123-07","2014-07-13 21:00:05.006123-07"} | {abc,cba} | {defhi,jklmn} | {"\\x622d36","\\x632d36"} + 7 | {row-07,"esc1\"","esc2\\","esc3}"} | {7,8} | {1000007,-1000007} | {5555500007,-5555500007,NULL} | {7.0001,NULL,-7.0001} | {NULL,3.14159265358979,-3.14159265358979} | NIL | {} | {2010-01-08} | {NULL} | {"2013-07-13 21:00:05.007456","2014-07-13 21:00:05.007456"} | {"2013-07-13 21:00:05.007123-07","2014-07-13 21:00:05.007123-07"} | {abc,cba} | {defhi,jklmn} | {"\\x622d37","\\x632d37"} + 8 | {row-08,"esc1\"","esc2\\","esc3}"} | {8,9} | {1000008,-1000008} | {5555500008,-5555500008} | {8.0001,-8.0001,NULL} | {3.14159265358979,NULL,-3.14159265358979} | {NULL,12345678900000.000008,-12345678900000.000008} | NIL | {} | {10:11:08} | {NULL} | {"2013-07-13 21:00:05.008123-07","2014-07-13 21:00:05.008123-07"} | {abc,cba} | {defhi,jklmn} | {"\\x622d38","\\x632d38"} + 9 | {row-09,"esc1\"","esc2\\","esc3}"} | {9,10} | {1000009,-1000009} | {5555500009,-5555500009} | {9.0001,-9.0001} | {3.14159265358979,-3.14159265358979,NULL} | {12345678900000.000009,NULL,-12345678900000.000009} | {NULL,t,f} | NIL | {} | {"2013-07-13 21:00:05.009456"} | {NULL} | {abc,cba} | {defhi,jklmn} | {"\\x622d39","\\x632d39"} + 10 | {row-10,"esc1\"","esc2\\","esc3}"} | {10,11} | {1000010,-1000010} | {5555500010,-5555500010} | {10.0001,-10.0001} | {3.14159265358979,-3.14159265358979} | {12345678900000.000010,-12345678900000.000010,NULL} | {t,NULL,f} | {NULL,2010-01-11,2010-02-11} | NIL | {} | {"2013-07-13 21:00:05.010123-07"} | {NULL} | {defhi,jklmn} | {"\\x622d3130","\\x632d3130"} + 11 | {row-11,"esc1\"","esc2\\","esc3}"} | {11,12} | {1000011,-1000011} | {5555500011,-5555500011} | {11.0001,-11.0001} | {3.14159265358979,-3.14159265358979} | {12345678900000.000011,-12345678900000.000011} | {t,f,NULL} | {2010-01-12,NULL,2010-02-12} | {NULL,10:11:11,11:11:11} | NIL | {} | {abc} | {NULL} | {"\\x622d3131","\\x632d3131"} + 12 | {row-12,"esc1\"","esc2\\","esc3}"} | {12,13} | {1000012,-1000012} | {5555500012,-5555500012} | {12.0001,-12.0001} | {3.14159265358979,-3.14159265358979} | {12345678900000.000012,-12345678900000.000012} | {t,f} | {2010-01-13,2010-02-13,NULL} | {10:11:12,NULL,11:11:12} | {NULL,"2013-07-13 21:00:05.012456","2014-07-13 21:00:05.012456"} | NIL | {} | {defhi} | {NULL} + 13 | {NULL} | {13,14} | {1000013,-1000013} | {5555500013,-5555500013} | {13.0001,-13.0001} | {3.14159265358979,-3.14159265358979} | {12345678900000.000013,-12345678900000.000013} | {t,f} | {2010-01-14,2010-02-14} | {10:11:13,11:11:13,NULL} | {"2013-07-13 21:00:05.013456",NULL,"2014-07-13 21:00:05.013456"} | {NULL,"2013-07-13 21:00:05.013123-07","2014-07-13 21:00:05.013123-07"} | NIL | {} | {"\\x622d3133"} + 14 | {row-14} | {NULL} | {1000014,-1000014} | {5555500014,-5555500014} | {14.0001,-14.0001} | {3.14159265358979,-3.14159265358979} | {12345678900000.000014,-12345678900000.000014} | {t,f} | {2010-01-15,2010-02-15} | {10:11:14,11:11:14} | {"2013-07-13 21:00:05.014456","2014-07-13 21:00:05.014456",NULL} | {"2013-07-13 21:00:05.014123-07",NULL,"2014-07-13 21:00:05.014123-07"} | {NULL,abc,cba} | NIL | {} + 15 | {} | {15} | {NULL} | {5555500015,-5555500015} | {15.0001,-15.0001} | {3.14159265358979,-3.14159265358979} | {12345678900000.000015,-12345678900000.000015} | {t,f} | {2010-01-16,2010-02-16} | {10:11:15,11:11:15} | {"2013-07-13 21:00:05.015456","2014-07-13 21:00:05.015456"} | {"2013-07-13 21:00:05.015123-07","2014-07-13 21:00:05.015123-07",NULL} | {abc,NULL,cba} | {NULL,defhi,jklmn} | NIL +(16 rows) + +SELECT id, name, sml, integ, bg, r, dp, dec, bool, cdate, ctime, tm, tmz, c1, vc1, bin from pxf_array_types_object_json_read ORDER BY id; + id | name | sml | integ | bg | r | dp | dec | bool | cdate | ctime | tm | tmz | c1 | vc1 | bin +----+-----------------------------------------+------------+-------------------------+-------------------------------+-----------------------+-------------------------------------------+-----------------------------------------------------+------------+------------------------------+--------------------------+------------------------------------------------------------------+------------------------------------------------------------------------+----------------+--------------------+-------------------------------- + 0 | {row-00,"esc1\"","esc2\\","esc3}",NULL} | {0,1,NULL} | {1000000,-1000000,NULL} | {5555500000,-5555500000,NULL} | {0.0001,-0.0001,NULL} | {3.14159265358979,-3.14159265358979,NULL} | {12345678900000,-12345678900000,NULL} | {t,f,NULL} | {2010-01-01,2010-01-01,NULL} | {10:11:00,11:11:00,NULL} | {"2013-07-13 21:00:05.000456","2014-07-13 21:00:05.000456",NULL} | {"2013-07-13 21:00:05.000123-07","2014-07-13 21:00:05.000123-07",NULL} | {abc,cba,NULL} | {defhi,jklmn,NULL} | {"\\x622d30","\\x632d30",NULL} + 1 | NIL | {} | {1000001} | {NULL} | {1.0001,-1.0001} | {3.14159265358979,-3.14159265358979} | {12345678900000.000001,-12345678900000.000001} | {t,f} | {2010-01-02,2010-02-02} | {10:11:01,11:11:01} | {"2013-07-13 21:00:05.001456","2014-07-13 21:00:05.001456"} | {"2013-07-13 21:00:05.001123-07","2014-07-13 21:00:05.001123-07"} | {abc,cba,NULL} | {defhi,NULL,jklmn} | {NULL,"\\x622d31","\\x632d31"} + 2 | {NULL,"esc1\"","esc2\\","esc3}"} | NIL | {} | {5555500002} | {NULL} | {3.14159265358979,-3.14159265358979} | {12345678900000.000002,-12345678900000.000002} | {t,f} | {2010-01-03,2010-02-03} | {10:11:02,11:11:02} | {"2013-07-13 21:00:05.002456","2014-07-13 21:00:05.002456"} | {"2013-07-13 21:00:05.002123-07","2014-07-13 21:00:05.002123-07"} | {abc,cba} | {defhi,jklmn,NULL} | {"\\x622d32",NULL,"\\x632d32"} + 3 | {row-03,NULL,"esc2\\","esc3}"} | {NULL,3,4} | NIL | {} | {-3.0001} | {NULL} | {12345678900000.000003,-12345678900000.000003} | {t,f} | {2010-01-04,2010-02-04} | {10:11:03,11:11:03} | {"2013-07-13 21:00:05.003456","2014-07-13 21:00:05.003456"} | {"2013-07-13 21:00:05.003123-07","2014-07-13 21:00:05.003123-07"} | {abc,cba} | {defhi,jklmn} | {"\\x622d33","\\x632d33",NULL} + 4 | {row-04,"esc1\"",NULL,"esc3}"} | {4,NULL,5} | {NULL,1000004,-1000004} | NIL | {} | {3.14159265358979} | {NULL} | {t,f} | {2010-01-05,2010-02-05} | {10:11:04,11:11:04} | {"2013-07-13 21:00:05.004456","2014-07-13 21:00:05.004456"} | {"2013-07-13 21:00:05.004123-07","2014-07-13 21:00:05.004123-07"} | {abc,cba} | {defhi,jklmn} | {"\\x622d34","\\x632d34"} + 5 | {row-05,"esc1\"","esc2\\",NULL} | {5,6,NULL} | {1000005,NULL,-1000005} | {NULL,5555500005,-5555500005} | NIL | {} | {-12345678900000.000005} | {NULL} | {2010-01-06,2010-02-06} | {10:11:05,11:11:05} | {"2013-07-13 21:00:05.005456","2014-07-13 21:00:05.005456"} | {"2013-07-13 21:00:05.005123-07","2014-07-13 21:00:05.005123-07"} | {abc,cba} | {defhi,jklmn} | {"\\x622d35","\\x632d35"} + 6 | {row-06,"esc1\"","esc2\\","esc3}"} | {6,7} | {1000006,-1000006,NULL} | {5555500006,NULL,-5555500006} | {NULL,6.0001,-6.0001} | NIL | {} | {t} | {NULL} | {10:11:06,11:11:06} | {"2013-07-13 21:00:05.006456","2014-07-13 21:00:05.006456"} | {"2013-07-13 21:00:05.006123-07","2014-07-13 21:00:05.006123-07"} | {abc,cba} | {defhi,jklmn} | {"\\x622d36","\\x632d36"} + 7 | {row-07,"esc1\"","esc2\\","esc3}"} | {7,8} | {1000007,-1000007} | {5555500007,-5555500007,NULL} | {7.0001,NULL,-7.0001} | {NULL,3.14159265358979,-3.14159265358979} | NIL | {} | {2010-01-08} | {NULL} | {"2013-07-13 21:00:05.007456","2014-07-13 21:00:05.007456"} | {"2013-07-13 21:00:05.007123-07","2014-07-13 21:00:05.007123-07"} | {abc,cba} | {defhi,jklmn} | {"\\x622d37","\\x632d37"} + 8 | {row-08,"esc1\"","esc2\\","esc3}"} | {8,9} | {1000008,-1000008} | {5555500008,-5555500008} | {8.0001,-8.0001,NULL} | {3.14159265358979,NULL,-3.14159265358979} | {NULL,12345678900000.000008,-12345678900000.000008} | NIL | {} | {10:11:08} | {NULL} | {"2013-07-13 21:00:05.008123-07","2014-07-13 21:00:05.008123-07"} | {abc,cba} | {defhi,jklmn} | {"\\x622d38","\\x632d38"} + 9 | {row-09,"esc1\"","esc2\\","esc3}"} | {9,10} | {1000009,-1000009} | {5555500009,-5555500009} | {9.0001,-9.0001} | {3.14159265358979,-3.14159265358979,NULL} | {12345678900000.000009,NULL,-12345678900000.000009} | {NULL,t,f} | NIL | {} | {"2013-07-13 21:00:05.009456"} | {NULL} | {abc,cba} | {defhi,jklmn} | {"\\x622d39","\\x632d39"} + 10 | {row-10,"esc1\"","esc2\\","esc3}"} | {10,11} | {1000010,-1000010} | {5555500010,-5555500010} | {10.0001,-10.0001} | {3.14159265358979,-3.14159265358979} | {12345678900000.00001,-12345678900000.00001,NULL} | {t,NULL,f} | {NULL,2010-01-11,2010-02-11} | NIL | {} | {"2013-07-13 21:00:05.010123-07"} | {NULL} | {defhi,jklmn} | {"\\x622d3130","\\x632d3130"} + 11 | {row-11,"esc1\"","esc2\\","esc3}"} | {11,12} | {1000011,-1000011} | {5555500011,-5555500011} | {11.0001,-11.0001} | {3.14159265358979,-3.14159265358979} | {12345678900000.000011,-12345678900000.000011} | {t,f,NULL} | {2010-01-12,NULL,2010-02-12} | {NULL,10:11:11,11:11:11} | NIL | {} | {abc} | {NULL} | {"\\x622d3131","\\x632d3131"} + 12 | {row-12,"esc1\"","esc2\\","esc3}"} | {12,13} | {1000012,-1000012} | {5555500012,-5555500012} | {12.0001,-12.0001} | {3.14159265358979,-3.14159265358979} | {12345678900000.000012,-12345678900000.000012} | {t,f} | {2010-01-13,2010-02-13,NULL} | {10:11:12,NULL,11:11:12} | {NULL,"2013-07-13 21:00:05.012456","2014-07-13 21:00:05.012456"} | NIL | {} | {defhi} | {NULL} + 13 | {NULL} | {13,14} | {1000013,-1000013} | {5555500013,-5555500013} | {13.0001,-13.0001} | {3.14159265358979,-3.14159265358979} | {12345678900000.000013,-12345678900000.000013} | {t,f} | {2010-01-14,2010-02-14} | {10:11:13,11:11:13,NULL} | {"2013-07-13 21:00:05.013456",NULL,"2014-07-13 21:00:05.013456"} | {NULL,"2013-07-13 21:00:05.013123-07","2014-07-13 21:00:05.013123-07"} | NIL | {} | {"\\x622d3133"} + 14 | {row-14} | {NULL} | {1000014,-1000014} | {5555500014,-5555500014} | {14.0001,-14.0001} | {3.14159265358979,-3.14159265358979} | {12345678900000.000014,-12345678900000.000014} | {t,f} | {2010-01-15,2010-02-15} | {10:11:14,11:11:14} | {"2013-07-13 21:00:05.014456","2014-07-13 21:00:05.014456",NULL} | {"2013-07-13 21:00:05.014123-07",NULL,"2014-07-13 21:00:05.014123-07"} | {NULL,abc,cba} | NIL | {} + 15 | {} | {15} | {NULL} | {5555500015,-5555500015} | {15.0001,-15.0001} | {3.14159265358979,-3.14159265358979} | {12345678900000.000015,-12345678900000.000015} | {t,f} | {2010-01-16,2010-02-16} | {10:11:15,11:11:15} | {"2013-07-13 21:00:05.015456","2014-07-13 21:00:05.015456"} | {"2013-07-13 21:00:05.015123-07","2014-07-13 21:00:05.015123-07",NULL} | {abc,NULL,cba} | {NULL,defhi,jklmn} | NIL +(16 rows) diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/array_types_object/runTest.py b/automation/tincrepo/main/pxf/features/hdfs/writable/json/array_types_object/runTest.py new file mode 100755 index 0000000000..9bd5bd99c4 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/hdfs/writable/json/array_types_object/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfHdfsJsonWritableArrayTypesObject(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/array_types_object/sql/query01.sql b/automation/tincrepo/main/pxf/features/hdfs/writable/json/array_types_object/sql/query01.sql new file mode 100755 index 0000000000..c533ca360a --- /dev/null +++ b/automation/tincrepo/main/pxf/features/hdfs/writable/json/array_types_object/sql/query01.sql @@ -0,0 +1,16 @@ +-- @description query01 for PXF HDFS Writable Json array types written in the object layout + +-- start_matchsubs +-- +-- # create a match/subs +-- +-- end_matchsubs + +\pset null 'NIL' +SET bytea_output = 'hex'; +SET TIME ZONE 'America/Los_Angeles'; +SHOW TIME ZONE; + +SELECT id, name, sml, integ, bg, r, dp, dec, bool, cdate, ctime, tm, tmz, c1, vc1, bin from gpdb_array_types ORDER BY id; + +SELECT id, name, sml, integ, bg, r, dp, dec, bool, cdate, ctime, tm, tmz, c1, vc1, bin from pxf_array_types_object_json_read ORDER BY id; diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/array_types_rows/__init__.py b/automation/tincrepo/main/pxf/features/hdfs/writable/json/array_types_rows/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/array_types_rows/expected/query01.ans b/automation/tincrepo/main/pxf/features/hdfs/writable/json/array_types_rows/expected/query01.ans new file mode 100755 index 0000000000..64940cb8b8 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/hdfs/writable/json/array_types_rows/expected/query01.ans @@ -0,0 +1,60 @@ +-- @description query01 for PXF HDFS Writable Json array types written in the rows layout + +-- start_matchsubs +-- +-- # create a match/subs +-- +-- end_matchsubs +\pset null 'NIL' +Null display is "NIL". +SET bytea_output = 'hex'; +SET +SET TIME ZONE 'America/Los_Angeles'; +SET +SHOW TIME ZONE; + TimeZone +--------------------- + America/Los_Angeles +(1 row) + +SELECT id, name, sml, integ, bg, r, dp, dec, bool, cdate, ctime, tm, tmz, c1, vc1, bin from gpdb_array_types ORDER BY id; + id | name | sml | integ | bg | r | dp | dec | bool | cdate | ctime | tm | tmz | c1 | vc1 | bin +----+-----------------------------------------+------------+-------------------------+-------------------------------+-----------------------+-------------------------------------------+-----------------------------------------------------+------------+------------------------------+--------------------------+------------------------------------------------------------------+------------------------------------------------------------------------+----------------+--------------------+-------------------------------- + 0 | {row-00,"esc1\"","esc2\\","esc3}",NULL} | {0,1,NULL} | {1000000,-1000000,NULL} | {5555500000,-5555500000,NULL} | {0.0001,-0.0001,NULL} | {3.14159265358979,-3.14159265358979,NULL} | {12345678900000.000000,-12345678900000.000000,NULL} | {t,f,NULL} | {2010-01-01,2010-01-01,NULL} | {10:11:00,11:11:00,NULL} | {"2013-07-13 21:00:05.000456","2014-07-13 21:00:05.000456",NULL} | {"2013-07-13 21:00:05.000123-07","2014-07-13 21:00:05.000123-07",NULL} | {abc,cba,NULL} | {defhi,jklmn,NULL} | {"\\x622d30","\\x632d30",NULL} + 1 | NIL | {} | {1000001} | {NULL} | {1.0001,-1.0001} | {3.14159265358979,-3.14159265358979} | {12345678900000.000001,-12345678900000.000001} | {t,f} | {2010-01-02,2010-02-02} | {10:11:01,11:11:01} | {"2013-07-13 21:00:05.001456","2014-07-13 21:00:05.001456"} | {"2013-07-13 21:00:05.001123-07","2014-07-13 21:00:05.001123-07"} | {abc,cba,NULL} | {defhi,NULL,jklmn} | {NULL,"\\x622d31","\\x632d31"} + 2 | {NULL,"esc1\"","esc2\\","esc3}"} | NIL | {} | {5555500002} | {NULL} | {3.14159265358979,-3.14159265358979} | {12345678900000.000002,-12345678900000.000002} | {t,f} | {2010-01-03,2010-02-03} | {10:11:02,11:11:02} | {"2013-07-13 21:00:05.002456","2014-07-13 21:00:05.002456"} | {"2013-07-13 21:00:05.002123-07","2014-07-13 21:00:05.002123-07"} | {abc,cba} | {defhi,jklmn,NULL} | {"\\x622d32",NULL,"\\x632d32"} + 3 | {row-03,NULL,"esc2\\","esc3}"} | {NULL,3,4} | NIL | {} | {-3.0001} | {NULL} | {12345678900000.000003,-12345678900000.000003} | {t,f} | {2010-01-04,2010-02-04} | {10:11:03,11:11:03} | {"2013-07-13 21:00:05.003456","2014-07-13 21:00:05.003456"} | {"2013-07-13 21:00:05.003123-07","2014-07-13 21:00:05.003123-07"} | {abc,cba} | {defhi,jklmn} | {"\\x622d33","\\x632d33",NULL} + 4 | {row-04,"esc1\"",NULL,"esc3}"} | {4,NULL,5} | {NULL,1000004,-1000004} | NIL | {} | {3.14159265358979} | {NULL} | {t,f} | {2010-01-05,2010-02-05} | {10:11:04,11:11:04} | {"2013-07-13 21:00:05.004456","2014-07-13 21:00:05.004456"} | {"2013-07-13 21:00:05.004123-07","2014-07-13 21:00:05.004123-07"} | {abc,cba} | {defhi,jklmn} | {"\\x622d34","\\x632d34"} + 5 | {row-05,"esc1\"","esc2\\",NULL} | {5,6,NULL} | {1000005,NULL,-1000005} | {NULL,5555500005,-5555500005} | NIL | {} | {-12345678900000.000005} | {NULL} | {2010-01-06,2010-02-06} | {10:11:05,11:11:05} | {"2013-07-13 21:00:05.005456","2014-07-13 21:00:05.005456"} | {"2013-07-13 21:00:05.005123-07","2014-07-13 21:00:05.005123-07"} | {abc,cba} | {defhi,jklmn} | {"\\x622d35","\\x632d35"} + 6 | {row-06,"esc1\"","esc2\\","esc3}"} | {6,7} | {1000006,-1000006,NULL} | {5555500006,NULL,-5555500006} | {NULL,6.0001,-6.0001} | NIL | {} | {t} | {NULL} | {10:11:06,11:11:06} | {"2013-07-13 21:00:05.006456","2014-07-13 21:00:05.006456"} | {"2013-07-13 21:00:05.006123-07","2014-07-13 21:00:05.006123-07"} | {abc,cba} | {defhi,jklmn} | {"\\x622d36","\\x632d36"} + 7 | {row-07,"esc1\"","esc2\\","esc3}"} | {7,8} | {1000007,-1000007} | {5555500007,-5555500007,NULL} | {7.0001,NULL,-7.0001} | {NULL,3.14159265358979,-3.14159265358979} | NIL | {} | {2010-01-08} | {NULL} | {"2013-07-13 21:00:05.007456","2014-07-13 21:00:05.007456"} | {"2013-07-13 21:00:05.007123-07","2014-07-13 21:00:05.007123-07"} | {abc,cba} | {defhi,jklmn} | {"\\x622d37","\\x632d37"} + 8 | {row-08,"esc1\"","esc2\\","esc3}"} | {8,9} | {1000008,-1000008} | {5555500008,-5555500008} | {8.0001,-8.0001,NULL} | {3.14159265358979,NULL,-3.14159265358979} | {NULL,12345678900000.000008,-12345678900000.000008} | NIL | {} | {10:11:08} | {NULL} | {"2013-07-13 21:00:05.008123-07","2014-07-13 21:00:05.008123-07"} | {abc,cba} | {defhi,jklmn} | {"\\x622d38","\\x632d38"} + 9 | {row-09,"esc1\"","esc2\\","esc3}"} | {9,10} | {1000009,-1000009} | {5555500009,-5555500009} | {9.0001,-9.0001} | {3.14159265358979,-3.14159265358979,NULL} | {12345678900000.000009,NULL,-12345678900000.000009} | {NULL,t,f} | NIL | {} | {"2013-07-13 21:00:05.009456"} | {NULL} | {abc,cba} | {defhi,jklmn} | {"\\x622d39","\\x632d39"} + 10 | {row-10,"esc1\"","esc2\\","esc3}"} | {10,11} | {1000010,-1000010} | {5555500010,-5555500010} | {10.0001,-10.0001} | {3.14159265358979,-3.14159265358979} | {12345678900000.000010,-12345678900000.000010,NULL} | {t,NULL,f} | {NULL,2010-01-11,2010-02-11} | NIL | {} | {"2013-07-13 21:00:05.010123-07"} | {NULL} | {defhi,jklmn} | {"\\x622d3130","\\x632d3130"} + 11 | {row-11,"esc1\"","esc2\\","esc3}"} | {11,12} | {1000011,-1000011} | {5555500011,-5555500011} | {11.0001,-11.0001} | {3.14159265358979,-3.14159265358979} | {12345678900000.000011,-12345678900000.000011} | {t,f,NULL} | {2010-01-12,NULL,2010-02-12} | {NULL,10:11:11,11:11:11} | NIL | {} | {abc} | {NULL} | {"\\x622d3131","\\x632d3131"} + 12 | {row-12,"esc1\"","esc2\\","esc3}"} | {12,13} | {1000012,-1000012} | {5555500012,-5555500012} | {12.0001,-12.0001} | {3.14159265358979,-3.14159265358979} | {12345678900000.000012,-12345678900000.000012} | {t,f} | {2010-01-13,2010-02-13,NULL} | {10:11:12,NULL,11:11:12} | {NULL,"2013-07-13 21:00:05.012456","2014-07-13 21:00:05.012456"} | NIL | {} | {defhi} | {NULL} + 13 | {NULL} | {13,14} | {1000013,-1000013} | {5555500013,-5555500013} | {13.0001,-13.0001} | {3.14159265358979,-3.14159265358979} | {12345678900000.000013,-12345678900000.000013} | {t,f} | {2010-01-14,2010-02-14} | {10:11:13,11:11:13,NULL} | {"2013-07-13 21:00:05.013456",NULL,"2014-07-13 21:00:05.013456"} | {NULL,"2013-07-13 21:00:05.013123-07","2014-07-13 21:00:05.013123-07"} | NIL | {} | {"\\x622d3133"} + 14 | {row-14} | {NULL} | {1000014,-1000014} | {5555500014,-5555500014} | {14.0001,-14.0001} | {3.14159265358979,-3.14159265358979} | {12345678900000.000014,-12345678900000.000014} | {t,f} | {2010-01-15,2010-02-15} | {10:11:14,11:11:14} | {"2013-07-13 21:00:05.014456","2014-07-13 21:00:05.014456",NULL} | {"2013-07-13 21:00:05.014123-07",NULL,"2014-07-13 21:00:05.014123-07"} | {NULL,abc,cba} | NIL | {} + 15 | {} | {15} | {NULL} | {5555500015,-5555500015} | {15.0001,-15.0001} | {3.14159265358979,-3.14159265358979} | {12345678900000.000015,-12345678900000.000015} | {t,f} | {2010-01-16,2010-02-16} | {10:11:15,11:11:15} | {"2013-07-13 21:00:05.015456","2014-07-13 21:00:05.015456"} | {"2013-07-13 21:00:05.015123-07","2014-07-13 21:00:05.015123-07",NULL} | {abc,NULL,cba} | {NULL,defhi,jklmn} | NIL +(16 rows) + +SELECT id, name, sml, integ, bg, r, dp, dec, bool, cdate, ctime, tm, tmz, c1, vc1, bin from pxf_array_types_rows_json_read ORDER BY id; + id | name | sml | integ | bg | r | dp | dec | bool | cdate | ctime | tm | tmz | c1 | vc1 | bin +----+-----------------------------------------+------------+-------------------------+-------------------------------+-----------------------+-------------------------------------------+-----------------------------------------------------+------------+------------------------------+--------------------------+------------------------------------------------------------------+------------------------------------------------------------------------+----------------+--------------------+-------------------------------- + 0 | {row-00,"esc1\"","esc2\\","esc3}",NULL} | {0,1,NULL} | {1000000,-1000000,NULL} | {5555500000,-5555500000,NULL} | {0.0001,-0.0001,NULL} | {3.14159265358979,-3.14159265358979,NULL} | {12345678900000,-12345678900000,NULL} | {t,f,NULL} | {2010-01-01,2010-01-01,NULL} | {10:11:00,11:11:00,NULL} | {"2013-07-13 21:00:05.000456","2014-07-13 21:00:05.000456",NULL} | {"2013-07-13 21:00:05.000123-07","2014-07-13 21:00:05.000123-07",NULL} | {abc,cba,NULL} | {defhi,jklmn,NULL} | {"\\x622d30","\\x632d30",NULL} + 1 | NIL | {} | {1000001} | {NULL} | {1.0001,-1.0001} | {3.14159265358979,-3.14159265358979} | {12345678900000.000001,-12345678900000.000001} | {t,f} | {2010-01-02,2010-02-02} | {10:11:01,11:11:01} | {"2013-07-13 21:00:05.001456","2014-07-13 21:00:05.001456"} | {"2013-07-13 21:00:05.001123-07","2014-07-13 21:00:05.001123-07"} | {abc,cba,NULL} | {defhi,NULL,jklmn} | {NULL,"\\x622d31","\\x632d31"} + 2 | {NULL,"esc1\"","esc2\\","esc3}"} | NIL | {} | {5555500002} | {NULL} | {3.14159265358979,-3.14159265358979} | {12345678900000.000002,-12345678900000.000002} | {t,f} | {2010-01-03,2010-02-03} | {10:11:02,11:11:02} | {"2013-07-13 21:00:05.002456","2014-07-13 21:00:05.002456"} | {"2013-07-13 21:00:05.002123-07","2014-07-13 21:00:05.002123-07"} | {abc,cba} | {defhi,jklmn,NULL} | {"\\x622d32",NULL,"\\x632d32"} + 3 | {row-03,NULL,"esc2\\","esc3}"} | {NULL,3,4} | NIL | {} | {-3.0001} | {NULL} | {12345678900000.000003,-12345678900000.000003} | {t,f} | {2010-01-04,2010-02-04} | {10:11:03,11:11:03} | {"2013-07-13 21:00:05.003456","2014-07-13 21:00:05.003456"} | {"2013-07-13 21:00:05.003123-07","2014-07-13 21:00:05.003123-07"} | {abc,cba} | {defhi,jklmn} | {"\\x622d33","\\x632d33",NULL} + 4 | {row-04,"esc1\"",NULL,"esc3}"} | {4,NULL,5} | {NULL,1000004,-1000004} | NIL | {} | {3.14159265358979} | {NULL} | {t,f} | {2010-01-05,2010-02-05} | {10:11:04,11:11:04} | {"2013-07-13 21:00:05.004456","2014-07-13 21:00:05.004456"} | {"2013-07-13 21:00:05.004123-07","2014-07-13 21:00:05.004123-07"} | {abc,cba} | {defhi,jklmn} | {"\\x622d34","\\x632d34"} + 5 | {row-05,"esc1\"","esc2\\",NULL} | {5,6,NULL} | {1000005,NULL,-1000005} | {NULL,5555500005,-5555500005} | NIL | {} | {-12345678900000.000005} | {NULL} | {2010-01-06,2010-02-06} | {10:11:05,11:11:05} | {"2013-07-13 21:00:05.005456","2014-07-13 21:00:05.005456"} | {"2013-07-13 21:00:05.005123-07","2014-07-13 21:00:05.005123-07"} | {abc,cba} | {defhi,jklmn} | {"\\x622d35","\\x632d35"} + 6 | {row-06,"esc1\"","esc2\\","esc3}"} | {6,7} | {1000006,-1000006,NULL} | {5555500006,NULL,-5555500006} | {NULL,6.0001,-6.0001} | NIL | {} | {t} | {NULL} | {10:11:06,11:11:06} | {"2013-07-13 21:00:05.006456","2014-07-13 21:00:05.006456"} | {"2013-07-13 21:00:05.006123-07","2014-07-13 21:00:05.006123-07"} | {abc,cba} | {defhi,jklmn} | {"\\x622d36","\\x632d36"} + 7 | {row-07,"esc1\"","esc2\\","esc3}"} | {7,8} | {1000007,-1000007} | {5555500007,-5555500007,NULL} | {7.0001,NULL,-7.0001} | {NULL,3.14159265358979,-3.14159265358979} | NIL | {} | {2010-01-08} | {NULL} | {"2013-07-13 21:00:05.007456","2014-07-13 21:00:05.007456"} | {"2013-07-13 21:00:05.007123-07","2014-07-13 21:00:05.007123-07"} | {abc,cba} | {defhi,jklmn} | {"\\x622d37","\\x632d37"} + 8 | {row-08,"esc1\"","esc2\\","esc3}"} | {8,9} | {1000008,-1000008} | {5555500008,-5555500008} | {8.0001,-8.0001,NULL} | {3.14159265358979,NULL,-3.14159265358979} | {NULL,12345678900000.000008,-12345678900000.000008} | NIL | {} | {10:11:08} | {NULL} | {"2013-07-13 21:00:05.008123-07","2014-07-13 21:00:05.008123-07"} | {abc,cba} | {defhi,jklmn} | {"\\x622d38","\\x632d38"} + 9 | {row-09,"esc1\"","esc2\\","esc3}"} | {9,10} | {1000009,-1000009} | {5555500009,-5555500009} | {9.0001,-9.0001} | {3.14159265358979,-3.14159265358979,NULL} | {12345678900000.000009,NULL,-12345678900000.000009} | {NULL,t,f} | NIL | {} | {"2013-07-13 21:00:05.009456"} | {NULL} | {abc,cba} | {defhi,jklmn} | {"\\x622d39","\\x632d39"} + 10 | {row-10,"esc1\"","esc2\\","esc3}"} | {10,11} | {1000010,-1000010} | {5555500010,-5555500010} | {10.0001,-10.0001} | {3.14159265358979,-3.14159265358979} | {12345678900000.00001,-12345678900000.00001,NULL} | {t,NULL,f} | {NULL,2010-01-11,2010-02-11} | NIL | {} | {"2013-07-13 21:00:05.010123-07"} | {NULL} | {defhi,jklmn} | {"\\x622d3130","\\x632d3130"} + 11 | {row-11,"esc1\"","esc2\\","esc3}"} | {11,12} | {1000011,-1000011} | {5555500011,-5555500011} | {11.0001,-11.0001} | {3.14159265358979,-3.14159265358979} | {12345678900000.000011,-12345678900000.000011} | {t,f,NULL} | {2010-01-12,NULL,2010-02-12} | {NULL,10:11:11,11:11:11} | NIL | {} | {abc} | {NULL} | {"\\x622d3131","\\x632d3131"} + 12 | {row-12,"esc1\"","esc2\\","esc3}"} | {12,13} | {1000012,-1000012} | {5555500012,-5555500012} | {12.0001,-12.0001} | {3.14159265358979,-3.14159265358979} | {12345678900000.000012,-12345678900000.000012} | {t,f} | {2010-01-13,2010-02-13,NULL} | {10:11:12,NULL,11:11:12} | {NULL,"2013-07-13 21:00:05.012456","2014-07-13 21:00:05.012456"} | NIL | {} | {defhi} | {NULL} + 13 | {NULL} | {13,14} | {1000013,-1000013} | {5555500013,-5555500013} | {13.0001,-13.0001} | {3.14159265358979,-3.14159265358979} | {12345678900000.000013,-12345678900000.000013} | {t,f} | {2010-01-14,2010-02-14} | {10:11:13,11:11:13,NULL} | {"2013-07-13 21:00:05.013456",NULL,"2014-07-13 21:00:05.013456"} | {NULL,"2013-07-13 21:00:05.013123-07","2014-07-13 21:00:05.013123-07"} | NIL | {} | {"\\x622d3133"} + 14 | {row-14} | {NULL} | {1000014,-1000014} | {5555500014,-5555500014} | {14.0001,-14.0001} | {3.14159265358979,-3.14159265358979} | {12345678900000.000014,-12345678900000.000014} | {t,f} | {2010-01-15,2010-02-15} | {10:11:14,11:11:14} | {"2013-07-13 21:00:05.014456","2014-07-13 21:00:05.014456",NULL} | {"2013-07-13 21:00:05.014123-07",NULL,"2014-07-13 21:00:05.014123-07"} | {NULL,abc,cba} | NIL | {} + 15 | {} | {15} | {NULL} | {5555500015,-5555500015} | {15.0001,-15.0001} | {3.14159265358979,-3.14159265358979} | {12345678900000.000015,-12345678900000.000015} | {t,f} | {2010-01-16,2010-02-16} | {10:11:15,11:11:15} | {"2013-07-13 21:00:05.015456","2014-07-13 21:00:05.015456"} | {"2013-07-13 21:00:05.015123-07","2014-07-13 21:00:05.015123-07",NULL} | {abc,NULL,cba} | {NULL,defhi,jklmn} | NIL +(16 rows) diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/array_types_rows/runTest.py b/automation/tincrepo/main/pxf/features/hdfs/writable/json/array_types_rows/runTest.py new file mode 100755 index 0000000000..311ebf600a --- /dev/null +++ b/automation/tincrepo/main/pxf/features/hdfs/writable/json/array_types_rows/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfHdfsJsonWritableArrayTypesRows(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/array_types_rows/sql/query01.sql b/automation/tincrepo/main/pxf/features/hdfs/writable/json/array_types_rows/sql/query01.sql new file mode 100755 index 0000000000..48f7a617eb --- /dev/null +++ b/automation/tincrepo/main/pxf/features/hdfs/writable/json/array_types_rows/sql/query01.sql @@ -0,0 +1,16 @@ +-- @description query01 for PXF HDFS Writable Json array types written in the rows layout + +-- start_matchsubs +-- +-- # create a match/subs +-- +-- end_matchsubs + +\pset null 'NIL' +SET bytea_output = 'hex'; +SET TIME ZONE 'America/Los_Angeles'; +SHOW TIME ZONE; + +SELECT id, name, sml, integ, bg, r, dp, dec, bool, cdate, ctime, tm, tmz, c1, vc1, bin from gpdb_array_types ORDER BY id; + +SELECT id, name, sml, integ, bg, r, dp, dec, bool, cdate, ctime, tm, tmz, c1, vc1, bin from pxf_array_types_rows_json_read ORDER BY id; diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/empty_root/__init__.py b/automation/tincrepo/main/pxf/features/hdfs/writable/json/empty_root/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/empty_root/expected/query01.ans b/automation/tincrepo/main/pxf/features/hdfs/writable/json/empty_root/expected/query01.ans new file mode 100755 index 0000000000..e32cbb8d36 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/hdfs/writable/json/empty_root/expected/query01.ans @@ -0,0 +1,10 @@ +-- @description query01 for PXF HDFS Writable Json where table is defined with an empty root option + +-- start_matchsubs +-- +-- # create a match/subs +-- +-- end_matchsubs + +INSERT INTO pxf_empty_root_json_write SELECT * from gpdb_primitive_types; +ERROR: PXF server error : Option ROOT can not have an empty value diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/empty_root/runTest.py b/automation/tincrepo/main/pxf/features/hdfs/writable/json/empty_root/runTest.py new file mode 100755 index 0000000000..6643ba36ba --- /dev/null +++ b/automation/tincrepo/main/pxf/features/hdfs/writable/json/empty_root/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfHdfsJsonWritableEmptyRoot(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/empty_root/sql/query01.sql b/automation/tincrepo/main/pxf/features/hdfs/writable/json/empty_root/sql/query01.sql new file mode 100755 index 0000000000..c04376e34f --- /dev/null +++ b/automation/tincrepo/main/pxf/features/hdfs/writable/json/empty_root/sql/query01.sql @@ -0,0 +1,9 @@ +-- @description query01 for PXF HDFS Writable Json where table is defined with an empty root option + +-- start_matchsubs +-- +-- # create a match/subs +-- +-- end_matchsubs + +INSERT INTO pxf_empty_root_json_write SELECT * from gpdb_primitive_types; diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/invalid_encoding/__init__.py b/automation/tincrepo/main/pxf/features/hdfs/writable/json/invalid_encoding/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/invalid_encoding/expected/query01.ans b/automation/tincrepo/main/pxf/features/hdfs/writable/json/invalid_encoding/expected/query01.ans new file mode 100755 index 0000000000..e5b7729b0d --- /dev/null +++ b/automation/tincrepo/main/pxf/features/hdfs/writable/json/invalid_encoding/expected/query01.ans @@ -0,0 +1,10 @@ +-- @description query01 for PXF HDFS Writable Json where table is defined with an invalid (non-UTF8) encoding + +-- start_matchsubs +-- +-- # create a match/subs +-- +-- end_matchsubs + +INSERT INTO pxf_invalid_encoding_json_write SELECT * from gpdb_primitive_types; +ERROR: pxfwritable_export formatter can only export UTF8 formatted data. Define the external table with ENCODING UTF8 diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/invalid_encoding/runTest.py b/automation/tincrepo/main/pxf/features/hdfs/writable/json/invalid_encoding/runTest.py new file mode 100755 index 0000000000..3e42f40670 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/hdfs/writable/json/invalid_encoding/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfHdfsJsonWritableInvalidEncoding(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/invalid_encoding/sql/query01.sql b/automation/tincrepo/main/pxf/features/hdfs/writable/json/invalid_encoding/sql/query01.sql new file mode 100755 index 0000000000..6bed0cf831 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/hdfs/writable/json/invalid_encoding/sql/query01.sql @@ -0,0 +1,9 @@ +-- @description query01 for PXF HDFS Writable Json where table is defined with an invalid (non-UTF8) encoding + +-- start_matchsubs +-- +-- # create a match/subs +-- +-- end_matchsubs + +INSERT INTO pxf_invalid_encoding_json_write SELECT * from gpdb_primitive_types; diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_escaping/__init__.py b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_escaping/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_escaping/expected/query01.ans b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_escaping/expected/query01.ans new file mode 100755 index 0000000000..7ddc54d397 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_escaping/expected/query01.ans @@ -0,0 +1,16 @@ +-- @description query01 for PXF HDFS Writable Json primitive types escaping + +-- start_matchsubs +-- +-- # create a match/subs +-- +-- end_matchsubs + +SELECT * from pxf_primitive_types_escaping_json_read ORDER BY id; + id | col space | text_escape | char_escape | varchar_escape +----+-------------+----------------+----------------+---------------- + 1 | col space 1 | text | char | varchar + 2 | col space 2 | s"b\{},:[] | s"b\{},:[] | s"b\{},:[] + 3 | col space 3 | d""b\\{},:[] | d""b\\{},:[] | d""b\\{},:[] + 4 | col space 4 | t"""b\\\{},:[] | t"""b\\\{},:[] | t"""b\\\{},:[] +(4 rows) diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_escaping/runTest.py b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_escaping/runTest.py new file mode 100755 index 0000000000..7d657235e2 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_escaping/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfHdfsJsonWritablePrimitiveTypesEscaping(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_escaping/sql/query01.sql b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_escaping/sql/query01.sql new file mode 100755 index 0000000000..6ffb846d1e --- /dev/null +++ b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_escaping/sql/query01.sql @@ -0,0 +1,9 @@ +-- @description query01 for PXF HDFS Writable Json primitive types escaping + +-- start_matchsubs +-- +-- # create a match/subs +-- +-- end_matchsubs + +SELECT * from pxf_primitive_types_escaping_json_read ORDER BY id; diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_object/__init__.py b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_object/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_object/expected/query01.ans b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_object/expected/query01.ans new file mode 100755 index 0000000000..f7418af649 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_object/expected/query01.ans @@ -0,0 +1,52 @@ +-- @description query01 for PXF HDFS Writable Json primitive types written in the object layout + +-- start_matchsubs +-- +-- # create a match/subs +-- +-- end_matchsubs +\pset null 'NIL' +Null display is "NIL". +SET bytea_output = 'hex'; +SET +SELECT id, name, sml, integ, bg, r, dp, dec, bool, cdate, ctime, tm, CAST(tmz AS TIMESTAMP WITH TIME ZONE) AT TIME ZONE 'PDT' as tmz, c1, vc1, bin from gpdb_primitive_types ORDER BY id; + id | name | sml | integ | bg | r | dp | dec | bool | cdate | ctime | tm | tmz | c1 | vc1 | bin +----+--------+-----+---------+------------+---------+------------------+------------------------+------+------------+----------+----------------------------+----------------------------+-----+-------+------------ + 0 | row-00 | 0 | 1000000 | 5555500000 | 0.0001 | 3.14159265358979 | 12345678900000.000000 | f | 2010-01-01 | 10:11:00 | 2013-07-13 21:00:05.000456 | 2013-07-13 21:00:05.000123 | abc | def | \x622d30 + 1 | NIL | 1 | 1000001 | 5555500001 | 1.0001 | 3.14159265358979 | 12345678900000.000001 | t | 2010-01-02 | 10:11:01 | 2013-07-13 21:00:05.001456 | 2013-07-13 21:00:05.001123 | abc | def | \x622d31 + 2 | row-02 | NIL | 1000002 | 5555500002 | 2.0001 | 3.14159265358979 | 12345678900000.000002 | f | 2010-01-03 | 10:11:02 | 2013-07-13 21:00:05.002456 | 2013-07-13 21:00:05.002123 | abc | def | \x622d32 + 3 | row-03 | 3 | NIL | 5555500003 | 3.0001 | 3.14159265358979 | 12345678900000.000003 | t | 2010-01-04 | 10:11:03 | 2013-07-13 21:00:05.003456 | 2013-07-13 21:00:05.003123 | abc | def | \x622d33 + 4 | row-04 | 4 | 1000004 | NIL | 4.0001 | 3.14159265358979 | 12345678900000.000004 | f | 2010-01-05 | 10:11:04 | 2013-07-13 21:00:05.004456 | 2013-07-13 21:00:05.004123 | abc | def | \x622d34 + 5 | row-05 | 5 | 1000005 | 5555500005 | NIL | 3.14159265358979 | 12345678900000.000005 | t | 2010-01-06 | 10:11:05 | 2013-07-13 21:00:05.005456 | 2013-07-13 21:00:05.005123 | abc | def | \x622d35 + 6 | row-06 | 6 | 1000006 | 5555500006 | 6.0001 | NIL | 12345678900000.000006 | f | 2010-01-07 | 10:11:06 | 2013-07-13 21:00:05.006456 | 2013-07-13 21:00:05.006123 | abc | def | \x622d36 + 7 | row-07 | 7 | 1000007 | 5555500007 | 7.0001 | 3.14159265358979 | NIL | t | 2010-01-08 | 10:11:07 | 2013-07-13 21:00:05.007456 | 2013-07-13 21:00:05.007123 | abc | def | \x622d37 + 8 | row-08 | 8 | 1000008 | 5555500008 | 8.0001 | 3.14159265358979 | 12345678900000.000008 | NIL | 2010-01-09 | 10:11:08 | 2013-07-13 21:00:05.008456 | 2013-07-13 21:00:05.008123 | abc | def | \x622d38 + 9 | row-09 | 9 | 1000009 | 5555500009 | 9.0001 | 3.14159265358979 | 12345678900000.000009 | t | NIL | 10:11:09 | 2013-07-13 21:00:05.009456 | 2013-07-13 21:00:05.009123 | abc | def | \x622d39 + 10 | row-10 | 10 | 1000010 | 5555500010 | 10.0001 | 3.14159265358979 | 12345678900000.0000010 | f | 2010-01-11 | NIL | 2013-07-13 21:00:05.010456 | 2013-07-13 21:00:05.010123 | abc | def | \x622d3130 + 11 | row-11 | 11 | 1000011 | 5555500011 | 11.0001 | 3.14159265358979 | 12345678900000.0000011 | t | 2010-01-12 | 10:11:11 | NIL | 2013-07-13 21:00:05.011123 | abc | def | \x622d3131 + 12 | row-12 | 12 | 1000012 | 5555500012 | 12.0001 | 3.14159265358979 | 12345678900000.0000012 | f | 2010-01-13 | 10:11:12 | 2013-07-13 21:00:05.012456 | NIL | abc | def | \x622d3132 + 13 | row-13 | 13 | 1000013 | 5555500013 | 13.0001 | 3.14159265358979 | 12345678900000.0000013 | t | 2010-01-14 | 10:11:13 | 2013-07-13 21:00:05.013456 | 2013-07-13 21:00:05.013123 | NIL | def | \x622d3133 + 14 | row-14 | 14 | 1000014 | 5555500014 | 14.0001 | 3.14159265358979 | 12345678900000.0000014 | f | 2010-01-15 | 10:11:14 | 2013-07-13 21:00:05.014456 | 2013-07-13 21:00:05.014123 | abc | NIL | \x622d3134 + 15 | row-15 | 15 | 1000015 | 5555500015 | 15.0001 | 3.14159265358979 | 12345678900000.0000015 | t | 2010-01-16 | 10:11:15 | 2013-07-13 21:00:05.015456 | 2013-07-13 21:00:05.015123 | abc | def | NIL +(16 rows) + +SELECT id, name, sml, integ, bg, r, dp, dec, bool, cdate, ctime, tm, CAST(tmz AS TIMESTAMP WITH TIME ZONE) AT TIME ZONE 'PDT' as tmz, c1, vc1, bin from pxf_primitive_types_object_json_read ORDER BY id; + id | name | sml | integ | bg | r | dp | dec | bool | cdate | ctime | tm | tmz | c1 | vc1 | bin +----+--------+-----+---------+------------+---------+------------------+------------------------+------+------------+----------+----------------------------+----------------------------+-----+-------+------------ + 0 | row-00 | 0 | 1000000 | 5555500000 | 0.0001 | 3.14159265358979 | 12345678900000 | f | 2010-01-01 | 10:11:00 | 2013-07-13 21:00:05.000456 | 2013-07-13 21:00:05.000123 | abc | def | \x622d30 + 1 | NIL | 1 | 1000001 | 5555500001 | 1.0001 | 3.14159265358979 | 12345678900000.000001 | t | 2010-01-02 | 10:11:01 | 2013-07-13 21:00:05.001456 | 2013-07-13 21:00:05.001123 | abc | def | \x622d31 + 2 | row-02 | NIL | 1000002 | 5555500002 | 2.0001 | 3.14159265358979 | 12345678900000.000002 | f | 2010-01-03 | 10:11:02 | 2013-07-13 21:00:05.002456 | 2013-07-13 21:00:05.002123 | abc | def | \x622d32 + 3 | row-03 | 3 | NIL | 5555500003 | 3.0001 | 3.14159265358979 | 12345678900000.000003 | t | 2010-01-04 | 10:11:03 | 2013-07-13 21:00:05.003456 | 2013-07-13 21:00:05.003123 | abc | def | \x622d33 + 4 | row-04 | 4 | 1000004 | NIL | 4.0001 | 3.14159265358979 | 12345678900000.000004 | f | 2010-01-05 | 10:11:04 | 2013-07-13 21:00:05.004456 | 2013-07-13 21:00:05.004123 | abc | def | \x622d34 + 5 | row-05 | 5 | 1000005 | 5555500005 | NIL | 3.14159265358979 | 12345678900000.000005 | t | 2010-01-06 | 10:11:05 | 2013-07-13 21:00:05.005456 | 2013-07-13 21:00:05.005123 | abc | def | \x622d35 + 6 | row-06 | 6 | 1000006 | 5555500006 | 6.0001 | NIL | 12345678900000.000006 | f | 2010-01-07 | 10:11:06 | 2013-07-13 21:00:05.006456 | 2013-07-13 21:00:05.006123 | abc | def | \x622d36 + 7 | row-07 | 7 | 1000007 | 5555500007 | 7.0001 | 3.14159265358979 | NIL | t | 2010-01-08 | 10:11:07 | 2013-07-13 21:00:05.007456 | 2013-07-13 21:00:05.007123 | abc | def | \x622d37 + 8 | row-08 | 8 | 1000008 | 5555500008 | 8.0001 | 3.14159265358979 | 12345678900000.000008 | NIL | 2010-01-09 | 10:11:08 | 2013-07-13 21:00:05.008456 | 2013-07-13 21:00:05.008123 | abc | def | \x622d38 + 9 | row-09 | 9 | 1000009 | 5555500009 | 9.0001 | 3.14159265358979 | 12345678900000.000009 | t | NIL | 10:11:09 | 2013-07-13 21:00:05.009456 | 2013-07-13 21:00:05.009123 | abc | def | \x622d39 + 10 | row-10 | 10 | 1000010 | 5555500010 | 10.0001 | 3.14159265358979 | 12345678900000.000001 | f | 2010-01-11 | NIL | 2013-07-13 21:00:05.010456 | 2013-07-13 21:00:05.010123 | abc | def | \x622d3130 + 11 | row-11 | 11 | 1000011 | 5555500011 | 11.0001 | 3.14159265358979 | 12345678900000.0000011 | t | 2010-01-12 | 10:11:11 | NIL | 2013-07-13 21:00:05.011123 | abc | def | \x622d3131 + 12 | row-12 | 12 | 1000012 | 5555500012 | 12.0001 | 3.14159265358979 | 12345678900000.0000012 | f | 2010-01-13 | 10:11:12 | 2013-07-13 21:00:05.012456 | NIL | abc | def | \x622d3132 + 13 | row-13 | 13 | 1000013 | 5555500013 | 13.0001 | 3.14159265358979 | 12345678900000.0000013 | t | 2010-01-14 | 10:11:13 | 2013-07-13 21:00:05.013456 | 2013-07-13 21:00:05.013123 | NIL | def | \x622d3133 + 14 | row-14 | 14 | 1000014 | 5555500014 | 14.0001 | 3.14159265358979 | 12345678900000.0000014 | f | 2010-01-15 | 10:11:14 | 2013-07-13 21:00:05.014456 | 2013-07-13 21:00:05.014123 | abc | NIL | \x622d3134 + 15 | row-15 | 15 | 1000015 | 5555500015 | 15.0001 | 3.14159265358979 | 12345678900000.0000015 | t | 2010-01-16 | 10:11:15 | 2013-07-13 21:00:05.015456 | 2013-07-13 21:00:05.015123 | abc | def | NIL +(16 rows) diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_object/runTest.py b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_object/runTest.py new file mode 100755 index 0000000000..7d3231e31c --- /dev/null +++ b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_object/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfHdfsJsonWritablePrimitiveTypesObject(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_object/sql/query01.sql b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_object/sql/query01.sql new file mode 100755 index 0000000000..e07d2d491a --- /dev/null +++ b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_object/sql/query01.sql @@ -0,0 +1,14 @@ +-- @description query01 for PXF HDFS Writable Json primitive types written in the object layout + +-- start_matchsubs +-- +-- # create a match/subs +-- +-- end_matchsubs + +\pset null 'NIL' +SET bytea_output = 'hex'; + +SELECT id, name, sml, integ, bg, r, dp, dec, bool, cdate, ctime, tm, CAST(tmz AS TIMESTAMP WITH TIME ZONE) AT TIME ZONE 'PDT' as tmz, c1, vc1, bin from gpdb_primitive_types ORDER BY id; + +SELECT id, name, sml, integ, bg, r, dp, dec, bool, cdate, ctime, tm, CAST(tmz AS TIMESTAMP WITH TIME ZONE) AT TIME ZONE 'PDT' as tmz, c1, vc1, bin from pxf_primitive_types_object_json_read ORDER BY id; diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_object_compressed/__init__.py b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_object_compressed/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_object_compressed/expected/query01.ans b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_object_compressed/expected/query01.ans new file mode 100755 index 0000000000..b4b5310496 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_object_compressed/expected/query01.ans @@ -0,0 +1,51 @@ +-- @description query01 for PXF HDFS Writable Json primitive types written in the object layout with compression +-- start_matchsubs +-- +-- # create a match/subs +-- +-- end_matchsubs +\pset null 'NIL' +Null display is "NIL". +SET bytea_output = 'hex'; +SET +SELECT id, name, sml, integ, bg, r, dp, dec, bool, cdate, ctime, tm, CAST(tmz AS TIMESTAMP WITH TIME ZONE) AT TIME ZONE 'PDT' as tmz, c1, vc1, bin from gpdb_primitive_types ORDER BY id; + id | name | sml | integ | bg | r | dp | dec | bool | cdate | ctime | tm | tmz | c1 | vc1 | bin +----+--------+-----+---------+------------+---------+------------------+------------------------+------+------------+----------+----------------------------+----------------------------+-----+-------+------------ + 0 | row-00 | 0 | 1000000 | 5555500000 | 0.0001 | 3.14159265358979 | 12345678900000.000000 | f | 2010-01-01 | 10:11:00 | 2013-07-13 21:00:05.000456 | 2013-07-13 21:00:05.000123 | abc | def | \x622d30 + 1 | NIL | 1 | 1000001 | 5555500001 | 1.0001 | 3.14159265358979 | 12345678900000.000001 | t | 2010-01-02 | 10:11:01 | 2013-07-13 21:00:05.001456 | 2013-07-13 21:00:05.001123 | abc | def | \x622d31 + 2 | row-02 | NIL | 1000002 | 5555500002 | 2.0001 | 3.14159265358979 | 12345678900000.000002 | f | 2010-01-03 | 10:11:02 | 2013-07-13 21:00:05.002456 | 2013-07-13 21:00:05.002123 | abc | def | \x622d32 + 3 | row-03 | 3 | NIL | 5555500003 | 3.0001 | 3.14159265358979 | 12345678900000.000003 | t | 2010-01-04 | 10:11:03 | 2013-07-13 21:00:05.003456 | 2013-07-13 21:00:05.003123 | abc | def | \x622d33 + 4 | row-04 | 4 | 1000004 | NIL | 4.0001 | 3.14159265358979 | 12345678900000.000004 | f | 2010-01-05 | 10:11:04 | 2013-07-13 21:00:05.004456 | 2013-07-13 21:00:05.004123 | abc | def | \x622d34 + 5 | row-05 | 5 | 1000005 | 5555500005 | NIL | 3.14159265358979 | 12345678900000.000005 | t | 2010-01-06 | 10:11:05 | 2013-07-13 21:00:05.005456 | 2013-07-13 21:00:05.005123 | abc | def | \x622d35 + 6 | row-06 | 6 | 1000006 | 5555500006 | 6.0001 | NIL | 12345678900000.000006 | f | 2010-01-07 | 10:11:06 | 2013-07-13 21:00:05.006456 | 2013-07-13 21:00:05.006123 | abc | def | \x622d36 + 7 | row-07 | 7 | 1000007 | 5555500007 | 7.0001 | 3.14159265358979 | NIL | t | 2010-01-08 | 10:11:07 | 2013-07-13 21:00:05.007456 | 2013-07-13 21:00:05.007123 | abc | def | \x622d37 + 8 | row-08 | 8 | 1000008 | 5555500008 | 8.0001 | 3.14159265358979 | 12345678900000.000008 | NIL | 2010-01-09 | 10:11:08 | 2013-07-13 21:00:05.008456 | 2013-07-13 21:00:05.008123 | abc | def | \x622d38 + 9 | row-09 | 9 | 1000009 | 5555500009 | 9.0001 | 3.14159265358979 | 12345678900000.000009 | t | NIL | 10:11:09 | 2013-07-13 21:00:05.009456 | 2013-07-13 21:00:05.009123 | abc | def | \x622d39 + 10 | row-10 | 10 | 1000010 | 5555500010 | 10.0001 | 3.14159265358979 | 12345678900000.0000010 | f | 2010-01-11 | NIL | 2013-07-13 21:00:05.010456 | 2013-07-13 21:00:05.010123 | abc | def | \x622d3130 + 11 | row-11 | 11 | 1000011 | 5555500011 | 11.0001 | 3.14159265358979 | 12345678900000.0000011 | t | 2010-01-12 | 10:11:11 | NIL | 2013-07-13 21:00:05.011123 | abc | def | \x622d3131 + 12 | row-12 | 12 | 1000012 | 5555500012 | 12.0001 | 3.14159265358979 | 12345678900000.0000012 | f | 2010-01-13 | 10:11:12 | 2013-07-13 21:00:05.012456 | NIL | abc | def | \x622d3132 + 13 | row-13 | 13 | 1000013 | 5555500013 | 13.0001 | 3.14159265358979 | 12345678900000.0000013 | t | 2010-01-14 | 10:11:13 | 2013-07-13 21:00:05.013456 | 2013-07-13 21:00:05.013123 | NIL | def | \x622d3133 + 14 | row-14 | 14 | 1000014 | 5555500014 | 14.0001 | 3.14159265358979 | 12345678900000.0000014 | f | 2010-01-15 | 10:11:14 | 2013-07-13 21:00:05.014456 | 2013-07-13 21:00:05.014123 | abc | NIL | \x622d3134 + 15 | row-15 | 15 | 1000015 | 5555500015 | 15.0001 | 3.14159265358979 | 12345678900000.0000015 | t | 2010-01-16 | 10:11:15 | 2013-07-13 21:00:05.015456 | 2013-07-13 21:00:05.015123 | abc | def | NIL +(16 rows) + +SELECT id, name, sml, integ, bg, r, dp, dec, bool, cdate, ctime, tm, CAST(tmz AS TIMESTAMP WITH TIME ZONE) AT TIME ZONE 'PDT' as tmz, c1, vc1, bin from pxf_primitive_types_object_compressed_json_read ORDER BY id; + id | name | sml | integ | bg | r | dp | dec | bool | cdate | ctime | tm | tmz | c1 | vc1 | bin +----+--------+-----+---------+------------+---------+------------------+------------------------+------+------------+----------+----------------------------+----------------------------+-----+-------+------------ + 0 | row-00 | 0 | 1000000 | 5555500000 | 0.0001 | 3.14159265358979 | 12345678900000 | f | 2010-01-01 | 10:11:00 | 2013-07-13 21:00:05.000456 | 2013-07-13 21:00:05.000123 | abc | def | \x622d30 + 1 | NIL | 1 | 1000001 | 5555500001 | 1.0001 | 3.14159265358979 | 12345678900000.000001 | t | 2010-01-02 | 10:11:01 | 2013-07-13 21:00:05.001456 | 2013-07-13 21:00:05.001123 | abc | def | \x622d31 + 2 | row-02 | NIL | 1000002 | 5555500002 | 2.0001 | 3.14159265358979 | 12345678900000.000002 | f | 2010-01-03 | 10:11:02 | 2013-07-13 21:00:05.002456 | 2013-07-13 21:00:05.002123 | abc | def | \x622d32 + 3 | row-03 | 3 | NIL | 5555500003 | 3.0001 | 3.14159265358979 | 12345678900000.000003 | t | 2010-01-04 | 10:11:03 | 2013-07-13 21:00:05.003456 | 2013-07-13 21:00:05.003123 | abc | def | \x622d33 + 4 | row-04 | 4 | 1000004 | NIL | 4.0001 | 3.14159265358979 | 12345678900000.000004 | f | 2010-01-05 | 10:11:04 | 2013-07-13 21:00:05.004456 | 2013-07-13 21:00:05.004123 | abc | def | \x622d34 + 5 | row-05 | 5 | 1000005 | 5555500005 | NIL | 3.14159265358979 | 12345678900000.000005 | t | 2010-01-06 | 10:11:05 | 2013-07-13 21:00:05.005456 | 2013-07-13 21:00:05.005123 | abc | def | \x622d35 + 6 | row-06 | 6 | 1000006 | 5555500006 | 6.0001 | NIL | 12345678900000.000006 | f | 2010-01-07 | 10:11:06 | 2013-07-13 21:00:05.006456 | 2013-07-13 21:00:05.006123 | abc | def | \x622d36 + 7 | row-07 | 7 | 1000007 | 5555500007 | 7.0001 | 3.14159265358979 | NIL | t | 2010-01-08 | 10:11:07 | 2013-07-13 21:00:05.007456 | 2013-07-13 21:00:05.007123 | abc | def | \x622d37 + 8 | row-08 | 8 | 1000008 | 5555500008 | 8.0001 | 3.14159265358979 | 12345678900000.000008 | NIL | 2010-01-09 | 10:11:08 | 2013-07-13 21:00:05.008456 | 2013-07-13 21:00:05.008123 | abc | def | \x622d38 + 9 | row-09 | 9 | 1000009 | 5555500009 | 9.0001 | 3.14159265358979 | 12345678900000.000009 | t | NIL | 10:11:09 | 2013-07-13 21:00:05.009456 | 2013-07-13 21:00:05.009123 | abc | def | \x622d39 + 10 | row-10 | 10 | 1000010 | 5555500010 | 10.0001 | 3.14159265358979 | 12345678900000.000001 | f | 2010-01-11 | NIL | 2013-07-13 21:00:05.010456 | 2013-07-13 21:00:05.010123 | abc | def | \x622d3130 + 11 | row-11 | 11 | 1000011 | 5555500011 | 11.0001 | 3.14159265358979 | 12345678900000.0000011 | t | 2010-01-12 | 10:11:11 | NIL | 2013-07-13 21:00:05.011123 | abc | def | \x622d3131 + 12 | row-12 | 12 | 1000012 | 5555500012 | 12.0001 | 3.14159265358979 | 12345678900000.0000012 | f | 2010-01-13 | 10:11:12 | 2013-07-13 21:00:05.012456 | NIL | abc | def | \x622d3132 + 13 | row-13 | 13 | 1000013 | 5555500013 | 13.0001 | 3.14159265358979 | 12345678900000.0000013 | t | 2010-01-14 | 10:11:13 | 2013-07-13 21:00:05.013456 | 2013-07-13 21:00:05.013123 | NIL | def | \x622d3133 + 14 | row-14 | 14 | 1000014 | 5555500014 | 14.0001 | 3.14159265358979 | 12345678900000.0000014 | f | 2010-01-15 | 10:11:14 | 2013-07-13 21:00:05.014456 | 2013-07-13 21:00:05.014123 | abc | NIL | \x622d3134 + 15 | row-15 | 15 | 1000015 | 5555500015 | 15.0001 | 3.14159265358979 | 12345678900000.0000015 | t | 2010-01-16 | 10:11:15 | 2013-07-13 21:00:05.015456 | 2013-07-13 21:00:05.015123 | abc | def | NIL +(16 rows) diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_object_compressed/runTest.py b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_object_compressed/runTest.py new file mode 100755 index 0000000000..2ec98ad106 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_object_compressed/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfHdfsJsonWritablePrimitiveTypesObjectCompressed(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_object_compressed/sql/query01.sql b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_object_compressed/sql/query01.sql new file mode 100755 index 0000000000..528e08c658 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_object_compressed/sql/query01.sql @@ -0,0 +1,14 @@ +-- @description query01 for PXF HDFS Writable Json primitive types written in the object layout with compression + +-- start_matchsubs +-- +-- # create a match/subs +-- +-- end_matchsubs + +\pset null 'NIL' +SET bytea_output = 'hex'; + +SELECT id, name, sml, integ, bg, r, dp, dec, bool, cdate, ctime, tm, CAST(tmz AS TIMESTAMP WITH TIME ZONE) AT TIME ZONE 'PDT' as tmz, c1, vc1, bin from gpdb_primitive_types ORDER BY id; + +SELECT id, name, sml, integ, bg, r, dp, dec, bool, cdate, ctime, tm, CAST(tmz AS TIMESTAMP WITH TIME ZONE) AT TIME ZONE 'PDT' as tmz, c1, vc1, bin from pxf_primitive_types_object_compressed_json_read ORDER BY id; diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_rows/__init__.py b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_rows/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_rows/expected/query01.ans b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_rows/expected/query01.ans new file mode 100755 index 0000000000..ac5915f522 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_rows/expected/query01.ans @@ -0,0 +1,52 @@ +-- @description query01 for PXF HDFS Writable Json primitive types written in the rows layout + +-- start_matchsubs +-- +-- # create a match/subs +-- +-- end_matchsubs +\pset null 'NIL' +Null display is "NIL". +SET bytea_output = 'hex'; +SET +SELECT id, name, sml, integ, bg, r, dp, dec, bool, cdate, ctime, tm, CAST(tmz AS TIMESTAMP WITH TIME ZONE) AT TIME ZONE 'PDT' as tmz, c1, vc1, bin from gpdb_primitive_types ORDER BY id; + id | name | sml | integ | bg | r | dp | dec | bool | cdate | ctime | tm | tmz | c1 | vc1 | bin +----+--------+-----+---------+------------+---------+------------------+------------------------+------+------------+----------+----------------------------+----------------------------+-----+-------+------------ + 0 | row-00 | 0 | 1000000 | 5555500000 | 0.0001 | 3.14159265358979 | 12345678900000.000000 | f | 2010-01-01 | 10:11:00 | 2013-07-13 21:00:05.000456 | 2013-07-13 21:00:05.000123 | abc | def | \x622d30 + 1 | NIL | 1 | 1000001 | 5555500001 | 1.0001 | 3.14159265358979 | 12345678900000.000001 | t | 2010-01-02 | 10:11:01 | 2013-07-13 21:00:05.001456 | 2013-07-13 21:00:05.001123 | abc | def | \x622d31 + 2 | row-02 | NIL | 1000002 | 5555500002 | 2.0001 | 3.14159265358979 | 12345678900000.000002 | f | 2010-01-03 | 10:11:02 | 2013-07-13 21:00:05.002456 | 2013-07-13 21:00:05.002123 | abc | def | \x622d32 + 3 | row-03 | 3 | NIL | 5555500003 | 3.0001 | 3.14159265358979 | 12345678900000.000003 | t | 2010-01-04 | 10:11:03 | 2013-07-13 21:00:05.003456 | 2013-07-13 21:00:05.003123 | abc | def | \x622d33 + 4 | row-04 | 4 | 1000004 | NIL | 4.0001 | 3.14159265358979 | 12345678900000.000004 | f | 2010-01-05 | 10:11:04 | 2013-07-13 21:00:05.004456 | 2013-07-13 21:00:05.004123 | abc | def | \x622d34 + 5 | row-05 | 5 | 1000005 | 5555500005 | NIL | 3.14159265358979 | 12345678900000.000005 | t | 2010-01-06 | 10:11:05 | 2013-07-13 21:00:05.005456 | 2013-07-13 21:00:05.005123 | abc | def | \x622d35 + 6 | row-06 | 6 | 1000006 | 5555500006 | 6.0001 | NIL | 12345678900000.000006 | f | 2010-01-07 | 10:11:06 | 2013-07-13 21:00:05.006456 | 2013-07-13 21:00:05.006123 | abc | def | \x622d36 + 7 | row-07 | 7 | 1000007 | 5555500007 | 7.0001 | 3.14159265358979 | NIL | t | 2010-01-08 | 10:11:07 | 2013-07-13 21:00:05.007456 | 2013-07-13 21:00:05.007123 | abc | def | \x622d37 + 8 | row-08 | 8 | 1000008 | 5555500008 | 8.0001 | 3.14159265358979 | 12345678900000.000008 | NIL | 2010-01-09 | 10:11:08 | 2013-07-13 21:00:05.008456 | 2013-07-13 21:00:05.008123 | abc | def | \x622d38 + 9 | row-09 | 9 | 1000009 | 5555500009 | 9.0001 | 3.14159265358979 | 12345678900000.000009 | t | NIL | 10:11:09 | 2013-07-13 21:00:05.009456 | 2013-07-13 21:00:05.009123 | abc | def | \x622d39 + 10 | row-10 | 10 | 1000010 | 5555500010 | 10.0001 | 3.14159265358979 | 12345678900000.0000010 | f | 2010-01-11 | NIL | 2013-07-13 21:00:05.010456 | 2013-07-13 21:00:05.010123 | abc | def | \x622d3130 + 11 | row-11 | 11 | 1000011 | 5555500011 | 11.0001 | 3.14159265358979 | 12345678900000.0000011 | t | 2010-01-12 | 10:11:11 | NIL | 2013-07-13 21:00:05.011123 | abc | def | \x622d3131 + 12 | row-12 | 12 | 1000012 | 5555500012 | 12.0001 | 3.14159265358979 | 12345678900000.0000012 | f | 2010-01-13 | 10:11:12 | 2013-07-13 21:00:05.012456 | NIL | abc | def | \x622d3132 + 13 | row-13 | 13 | 1000013 | 5555500013 | 13.0001 | 3.14159265358979 | 12345678900000.0000013 | t | 2010-01-14 | 10:11:13 | 2013-07-13 21:00:05.013456 | 2013-07-13 21:00:05.013123 | NIL | def | \x622d3133 + 14 | row-14 | 14 | 1000014 | 5555500014 | 14.0001 | 3.14159265358979 | 12345678900000.0000014 | f | 2010-01-15 | 10:11:14 | 2013-07-13 21:00:05.014456 | 2013-07-13 21:00:05.014123 | abc | NIL | \x622d3134 + 15 | row-15 | 15 | 1000015 | 5555500015 | 15.0001 | 3.14159265358979 | 12345678900000.0000015 | t | 2010-01-16 | 10:11:15 | 2013-07-13 21:00:05.015456 | 2013-07-13 21:00:05.015123 | abc | def | NIL +(16 rows) + +SELECT id, name, sml, integ, bg, r, dp, dec, bool, cdate, ctime, tm, CAST(tmz AS TIMESTAMP WITH TIME ZONE) AT TIME ZONE 'PDT' as tmz, c1, vc1, bin from pxf_primitive_types_rows_json_read ORDER BY id; + id | name | sml | integ | bg | r | dp | dec | bool | cdate | ctime | tm | tmz | c1 | vc1 | bin +----+--------+-----+---------+------------+---------+------------------+------------------------+------+------------+----------+----------------------------+----------------------------+-----+-------+------------ + 0 | row-00 | 0 | 1000000 | 5555500000 | 0.0001 | 3.14159265358979 | 12345678900000 | f | 2010-01-01 | 10:11:00 | 2013-07-13 21:00:05.000456 | 2013-07-13 21:00:05.000123 | abc | def | \x622d30 + 1 | NIL | 1 | 1000001 | 5555500001 | 1.0001 | 3.14159265358979 | 12345678900000.000001 | t | 2010-01-02 | 10:11:01 | 2013-07-13 21:00:05.001456 | 2013-07-13 21:00:05.001123 | abc | def | \x622d31 + 2 | row-02 | NIL | 1000002 | 5555500002 | 2.0001 | 3.14159265358979 | 12345678900000.000002 | f | 2010-01-03 | 10:11:02 | 2013-07-13 21:00:05.002456 | 2013-07-13 21:00:05.002123 | abc | def | \x622d32 + 3 | row-03 | 3 | NIL | 5555500003 | 3.0001 | 3.14159265358979 | 12345678900000.000003 | t | 2010-01-04 | 10:11:03 | 2013-07-13 21:00:05.003456 | 2013-07-13 21:00:05.003123 | abc | def | \x622d33 + 4 | row-04 | 4 | 1000004 | NIL | 4.0001 | 3.14159265358979 | 12345678900000.000004 | f | 2010-01-05 | 10:11:04 | 2013-07-13 21:00:05.004456 | 2013-07-13 21:00:05.004123 | abc | def | \x622d34 + 5 | row-05 | 5 | 1000005 | 5555500005 | NIL | 3.14159265358979 | 12345678900000.000005 | t | 2010-01-06 | 10:11:05 | 2013-07-13 21:00:05.005456 | 2013-07-13 21:00:05.005123 | abc | def | \x622d35 + 6 | row-06 | 6 | 1000006 | 5555500006 | 6.0001 | NIL | 12345678900000.000006 | f | 2010-01-07 | 10:11:06 | 2013-07-13 21:00:05.006456 | 2013-07-13 21:00:05.006123 | abc | def | \x622d36 + 7 | row-07 | 7 | 1000007 | 5555500007 | 7.0001 | 3.14159265358979 | NIL | t | 2010-01-08 | 10:11:07 | 2013-07-13 21:00:05.007456 | 2013-07-13 21:00:05.007123 | abc | def | \x622d37 + 8 | row-08 | 8 | 1000008 | 5555500008 | 8.0001 | 3.14159265358979 | 12345678900000.000008 | NIL | 2010-01-09 | 10:11:08 | 2013-07-13 21:00:05.008456 | 2013-07-13 21:00:05.008123 | abc | def | \x622d38 + 9 | row-09 | 9 | 1000009 | 5555500009 | 9.0001 | 3.14159265358979 | 12345678900000.000009 | t | NIL | 10:11:09 | 2013-07-13 21:00:05.009456 | 2013-07-13 21:00:05.009123 | abc | def | \x622d39 + 10 | row-10 | 10 | 1000010 | 5555500010 | 10.0001 | 3.14159265358979 | 12345678900000.000001 | f | 2010-01-11 | NIL | 2013-07-13 21:00:05.010456 | 2013-07-13 21:00:05.010123 | abc | def | \x622d3130 + 11 | row-11 | 11 | 1000011 | 5555500011 | 11.0001 | 3.14159265358979 | 12345678900000.0000011 | t | 2010-01-12 | 10:11:11 | NIL | 2013-07-13 21:00:05.011123 | abc | def | \x622d3131 + 12 | row-12 | 12 | 1000012 | 5555500012 | 12.0001 | 3.14159265358979 | 12345678900000.0000012 | f | 2010-01-13 | 10:11:12 | 2013-07-13 21:00:05.012456 | NIL | abc | def | \x622d3132 + 13 | row-13 | 13 | 1000013 | 5555500013 | 13.0001 | 3.14159265358979 | 12345678900000.0000013 | t | 2010-01-14 | 10:11:13 | 2013-07-13 21:00:05.013456 | 2013-07-13 21:00:05.013123 | NIL | def | \x622d3133 + 14 | row-14 | 14 | 1000014 | 5555500014 | 14.0001 | 3.14159265358979 | 12345678900000.0000014 | f | 2010-01-15 | 10:11:14 | 2013-07-13 21:00:05.014456 | 2013-07-13 21:00:05.014123 | abc | NIL | \x622d3134 + 15 | row-15 | 15 | 1000015 | 5555500015 | 15.0001 | 3.14159265358979 | 12345678900000.0000015 | t | 2010-01-16 | 10:11:15 | 2013-07-13 21:00:05.015456 | 2013-07-13 21:00:05.015123 | abc | def | NIL +(16 rows) diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_rows/runTest.py b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_rows/runTest.py new file mode 100755 index 0000000000..ce85d2437d --- /dev/null +++ b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_rows/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfHdfsJsonWritablePrimitiveTypesRows(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_rows/sql/query01.sql b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_rows/sql/query01.sql new file mode 100755 index 0000000000..7c3b5953ac --- /dev/null +++ b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_rows/sql/query01.sql @@ -0,0 +1,14 @@ +-- @description query01 for PXF HDFS Writable Json primitive types written in the rows layout + +-- start_matchsubs +-- +-- # create a match/subs +-- +-- end_matchsubs + +\pset null 'NIL' +SET bytea_output = 'hex'; + +SELECT id, name, sml, integ, bg, r, dp, dec, bool, cdate, ctime, tm, CAST(tmz AS TIMESTAMP WITH TIME ZONE) AT TIME ZONE 'PDT' as tmz, c1, vc1, bin from gpdb_primitive_types ORDER BY id; + +SELECT id, name, sml, integ, bg, r, dp, dec, bool, cdate, ctime, tm, CAST(tmz AS TIMESTAMP WITH TIME ZONE) AT TIME ZONE 'PDT' as tmz, c1, vc1, bin from pxf_primitive_types_rows_json_read ORDER BY id; diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_rows_compressed/__init__.py b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_rows_compressed/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_rows_compressed/expected/query01.ans b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_rows_compressed/expected/query01.ans new file mode 100755 index 0000000000..ddfc7c0736 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_rows_compressed/expected/query01.ans @@ -0,0 +1,51 @@ +-- @description query01 for PXF HDFS Writable Json primitive types written in the rows layout with compression +-- start_matchsubs +-- +-- # create a match/subs +-- +-- end_matchsubs +\pset null 'NIL' +Null display is "NIL". +SET bytea_output = 'hex'; +SET +SELECT id, name, sml, integ, bg, r, dp, dec, bool, cdate, ctime, tm, CAST(tmz AS TIMESTAMP WITH TIME ZONE) AT TIME ZONE 'PDT' as tmz, c1, vc1, bin from gpdb_primitive_types ORDER BY id; + id | name | sml | integ | bg | r | dp | dec | bool | cdate | ctime | tm | tmz | c1 | vc1 | bin +----+--------+-----+---------+------------+---------+------------------+------------------------+------+------------+----------+----------------------------+----------------------------+-----+-------+------------ + 0 | row-00 | 0 | 1000000 | 5555500000 | 0.0001 | 3.14159265358979 | 12345678900000.000000 | f | 2010-01-01 | 10:11:00 | 2013-07-13 21:00:05.000456 | 2013-07-13 21:00:05.000123 | abc | def | \x622d30 + 1 | NIL | 1 | 1000001 | 5555500001 | 1.0001 | 3.14159265358979 | 12345678900000.000001 | t | 2010-01-02 | 10:11:01 | 2013-07-13 21:00:05.001456 | 2013-07-13 21:00:05.001123 | abc | def | \x622d31 + 2 | row-02 | NIL | 1000002 | 5555500002 | 2.0001 | 3.14159265358979 | 12345678900000.000002 | f | 2010-01-03 | 10:11:02 | 2013-07-13 21:00:05.002456 | 2013-07-13 21:00:05.002123 | abc | def | \x622d32 + 3 | row-03 | 3 | NIL | 5555500003 | 3.0001 | 3.14159265358979 | 12345678900000.000003 | t | 2010-01-04 | 10:11:03 | 2013-07-13 21:00:05.003456 | 2013-07-13 21:00:05.003123 | abc | def | \x622d33 + 4 | row-04 | 4 | 1000004 | NIL | 4.0001 | 3.14159265358979 | 12345678900000.000004 | f | 2010-01-05 | 10:11:04 | 2013-07-13 21:00:05.004456 | 2013-07-13 21:00:05.004123 | abc | def | \x622d34 + 5 | row-05 | 5 | 1000005 | 5555500005 | NIL | 3.14159265358979 | 12345678900000.000005 | t | 2010-01-06 | 10:11:05 | 2013-07-13 21:00:05.005456 | 2013-07-13 21:00:05.005123 | abc | def | \x622d35 + 6 | row-06 | 6 | 1000006 | 5555500006 | 6.0001 | NIL | 12345678900000.000006 | f | 2010-01-07 | 10:11:06 | 2013-07-13 21:00:05.006456 | 2013-07-13 21:00:05.006123 | abc | def | \x622d36 + 7 | row-07 | 7 | 1000007 | 5555500007 | 7.0001 | 3.14159265358979 | NIL | t | 2010-01-08 | 10:11:07 | 2013-07-13 21:00:05.007456 | 2013-07-13 21:00:05.007123 | abc | def | \x622d37 + 8 | row-08 | 8 | 1000008 | 5555500008 | 8.0001 | 3.14159265358979 | 12345678900000.000008 | NIL | 2010-01-09 | 10:11:08 | 2013-07-13 21:00:05.008456 | 2013-07-13 21:00:05.008123 | abc | def | \x622d38 + 9 | row-09 | 9 | 1000009 | 5555500009 | 9.0001 | 3.14159265358979 | 12345678900000.000009 | t | NIL | 10:11:09 | 2013-07-13 21:00:05.009456 | 2013-07-13 21:00:05.009123 | abc | def | \x622d39 + 10 | row-10 | 10 | 1000010 | 5555500010 | 10.0001 | 3.14159265358979 | 12345678900000.0000010 | f | 2010-01-11 | NIL | 2013-07-13 21:00:05.010456 | 2013-07-13 21:00:05.010123 | abc | def | \x622d3130 + 11 | row-11 | 11 | 1000011 | 5555500011 | 11.0001 | 3.14159265358979 | 12345678900000.0000011 | t | 2010-01-12 | 10:11:11 | NIL | 2013-07-13 21:00:05.011123 | abc | def | \x622d3131 + 12 | row-12 | 12 | 1000012 | 5555500012 | 12.0001 | 3.14159265358979 | 12345678900000.0000012 | f | 2010-01-13 | 10:11:12 | 2013-07-13 21:00:05.012456 | NIL | abc | def | \x622d3132 + 13 | row-13 | 13 | 1000013 | 5555500013 | 13.0001 | 3.14159265358979 | 12345678900000.0000013 | t | 2010-01-14 | 10:11:13 | 2013-07-13 21:00:05.013456 | 2013-07-13 21:00:05.013123 | NIL | def | \x622d3133 + 14 | row-14 | 14 | 1000014 | 5555500014 | 14.0001 | 3.14159265358979 | 12345678900000.0000014 | f | 2010-01-15 | 10:11:14 | 2013-07-13 21:00:05.014456 | 2013-07-13 21:00:05.014123 | abc | NIL | \x622d3134 + 15 | row-15 | 15 | 1000015 | 5555500015 | 15.0001 | 3.14159265358979 | 12345678900000.0000015 | t | 2010-01-16 | 10:11:15 | 2013-07-13 21:00:05.015456 | 2013-07-13 21:00:05.015123 | abc | def | NIL +(16 rows) + +SELECT id, name, sml, integ, bg, r, dp, dec, bool, cdate, ctime, tm, CAST(tmz AS TIMESTAMP WITH TIME ZONE) AT TIME ZONE 'PDT' as tmz, c1, vc1, bin from pxf_primitive_types_rows_compressed_json_read ORDER BY id; + id | name | sml | integ | bg | r | dp | dec | bool | cdate | ctime | tm | tmz | c1 | vc1 | bin +----+--------+-----+---------+------------+---------+------------------+------------------------+------+------------+----------+----------------------------+----------------------------+-----+-------+------------ + 0 | row-00 | 0 | 1000000 | 5555500000 | 0.0001 | 3.14159265358979 | 12345678900000 | f | 2010-01-01 | 10:11:00 | 2013-07-13 21:00:05.000456 | 2013-07-13 21:00:05.000123 | abc | def | \x622d30 + 1 | NIL | 1 | 1000001 | 5555500001 | 1.0001 | 3.14159265358979 | 12345678900000.000001 | t | 2010-01-02 | 10:11:01 | 2013-07-13 21:00:05.001456 | 2013-07-13 21:00:05.001123 | abc | def | \x622d31 + 2 | row-02 | NIL | 1000002 | 5555500002 | 2.0001 | 3.14159265358979 | 12345678900000.000002 | f | 2010-01-03 | 10:11:02 | 2013-07-13 21:00:05.002456 | 2013-07-13 21:00:05.002123 | abc | def | \x622d32 + 3 | row-03 | 3 | NIL | 5555500003 | 3.0001 | 3.14159265358979 | 12345678900000.000003 | t | 2010-01-04 | 10:11:03 | 2013-07-13 21:00:05.003456 | 2013-07-13 21:00:05.003123 | abc | def | \x622d33 + 4 | row-04 | 4 | 1000004 | NIL | 4.0001 | 3.14159265358979 | 12345678900000.000004 | f | 2010-01-05 | 10:11:04 | 2013-07-13 21:00:05.004456 | 2013-07-13 21:00:05.004123 | abc | def | \x622d34 + 5 | row-05 | 5 | 1000005 | 5555500005 | NIL | 3.14159265358979 | 12345678900000.000005 | t | 2010-01-06 | 10:11:05 | 2013-07-13 21:00:05.005456 | 2013-07-13 21:00:05.005123 | abc | def | \x622d35 + 6 | row-06 | 6 | 1000006 | 5555500006 | 6.0001 | NIL | 12345678900000.000006 | f | 2010-01-07 | 10:11:06 | 2013-07-13 21:00:05.006456 | 2013-07-13 21:00:05.006123 | abc | def | \x622d36 + 7 | row-07 | 7 | 1000007 | 5555500007 | 7.0001 | 3.14159265358979 | NIL | t | 2010-01-08 | 10:11:07 | 2013-07-13 21:00:05.007456 | 2013-07-13 21:00:05.007123 | abc | def | \x622d37 + 8 | row-08 | 8 | 1000008 | 5555500008 | 8.0001 | 3.14159265358979 | 12345678900000.000008 | NIL | 2010-01-09 | 10:11:08 | 2013-07-13 21:00:05.008456 | 2013-07-13 21:00:05.008123 | abc | def | \x622d38 + 9 | row-09 | 9 | 1000009 | 5555500009 | 9.0001 | 3.14159265358979 | 12345678900000.000009 | t | NIL | 10:11:09 | 2013-07-13 21:00:05.009456 | 2013-07-13 21:00:05.009123 | abc | def | \x622d39 + 10 | row-10 | 10 | 1000010 | 5555500010 | 10.0001 | 3.14159265358979 | 12345678900000.000001 | f | 2010-01-11 | NIL | 2013-07-13 21:00:05.010456 | 2013-07-13 21:00:05.010123 | abc | def | \x622d3130 + 11 | row-11 | 11 | 1000011 | 5555500011 | 11.0001 | 3.14159265358979 | 12345678900000.0000011 | t | 2010-01-12 | 10:11:11 | NIL | 2013-07-13 21:00:05.011123 | abc | def | \x622d3131 + 12 | row-12 | 12 | 1000012 | 5555500012 | 12.0001 | 3.14159265358979 | 12345678900000.0000012 | f | 2010-01-13 | 10:11:12 | 2013-07-13 21:00:05.012456 | NIL | abc | def | \x622d3132 + 13 | row-13 | 13 | 1000013 | 5555500013 | 13.0001 | 3.14159265358979 | 12345678900000.0000013 | t | 2010-01-14 | 10:11:13 | 2013-07-13 21:00:05.013456 | 2013-07-13 21:00:05.013123 | NIL | def | \x622d3133 + 14 | row-14 | 14 | 1000014 | 5555500014 | 14.0001 | 3.14159265358979 | 12345678900000.0000014 | f | 2010-01-15 | 10:11:14 | 2013-07-13 21:00:05.014456 | 2013-07-13 21:00:05.014123 | abc | NIL | \x622d3134 + 15 | row-15 | 15 | 1000015 | 5555500015 | 15.0001 | 3.14159265358979 | 12345678900000.0000015 | t | 2010-01-16 | 10:11:15 | 2013-07-13 21:00:05.015456 | 2013-07-13 21:00:05.015123 | abc | def | NIL +(16 rows) diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_rows_compressed/runTest.py b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_rows_compressed/runTest.py new file mode 100755 index 0000000000..0b86af2136 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_rows_compressed/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfHdfsJsonWritablePrimitiveTypesRowsCompressed(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_rows_compressed/sql/query01.sql b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_rows_compressed/sql/query01.sql new file mode 100755 index 0000000000..4ce32197cd --- /dev/null +++ b/automation/tincrepo/main/pxf/features/hdfs/writable/json/primitive_types_rows_compressed/sql/query01.sql @@ -0,0 +1,14 @@ +-- @description query01 for PXF HDFS Writable Json primitive types written in the rows layout with compression + +-- start_matchsubs +-- +-- # create a match/subs +-- +-- end_matchsubs + +\pset null 'NIL' +SET bytea_output = 'hex'; + +SELECT id, name, sml, integ, bg, r, dp, dec, bool, cdate, ctime, tm, CAST(tmz AS TIMESTAMP WITH TIME ZONE) AT TIME ZONE 'PDT' as tmz, c1, vc1, bin from gpdb_primitive_types ORDER BY id; + +SELECT id, name, sml, integ, bg, r, dp, dec, bool, cdate, ctime, tm, CAST(tmz AS TIMESTAMP WITH TIME ZONE) AT TIME ZONE 'PDT' as tmz, c1, vc1, bin from pxf_primitive_types_rows_compressed_json_read ORDER BY id; diff --git a/automation/tincrepo/main/pxf/features/jdbc/column_projection/expected/query04.ans b/automation/tincrepo/main/pxf/features/jdbc/column_projection/expected/query04.ans index 63ed45ed6f..dfeaaf0bba 100644 --- a/automation/tincrepo/main/pxf/features/jdbc/column_projection/expected/query04.ans +++ b/automation/tincrepo/main/pxf/features/jdbc/column_projection/expected/query04.ans @@ -32,6 +32,9 @@ SELECT "n@m2", "t", "num 1" FROM pxf_jdbc_superset_of_fields ORDER BY "t"; -- m/CONTEXT:.*line.*/ -- s/line \d* of //g -- +-- m/CONTEXT:.*file.*/ +-- s/, file.*//g +-- -- end_matchsubs SELECT * FROM pxf_jdbc_superset_of_fields ORDER BY "t"; ERROR: column "does_not_exist_on_source" does not exist diff --git a/automation/tincrepo/main/pxf/features/jdbc/column_projection/sql/query04.sql b/automation/tincrepo/main/pxf/features/jdbc/column_projection/sql/query04.sql index eee127edcd..0a94536f1d 100644 --- a/automation/tincrepo/main/pxf/features/jdbc/column_projection/sql/query04.sql +++ b/automation/tincrepo/main/pxf/features/jdbc/column_projection/sql/query04.sql @@ -25,5 +25,8 @@ SELECT "n@m2", "t", "num 1" FROM pxf_jdbc_superset_of_fields ORDER BY "t"; -- m/CONTEXT:.*line.*/ -- s/line \d* of //g -- +-- m/CONTEXT:.*file.*/ +-- s/, file.*//g +-- -- end_matchsubs SELECT * FROM pxf_jdbc_superset_of_fields ORDER BY "t"; diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/__init__.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding/__init__.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding/expected/query01.ans b/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding/expected/query01.ans new file mode 100755 index 0000000000..54c87af484 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding/expected/query01.ans @@ -0,0 +1,9 @@ +-- @description query01 for PXF Multibyte delimiter differentEncoding case + +SELECT * from pxf_multibyte_encoding ORDER BY num1; + num1 | word +------+-------------------------------------- + 3 | règles d'automation + 4 | tá sé seo le tástáil dea- + 5 | minden amire szüksége van a szeretet +(3 rows) \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding/runTest.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding/runTest.py new file mode 100755 index 0000000000..067825963b --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfMultibyteDelimDifferentEncoding(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding/sql/query01.sql b/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding/sql/query01.sql new file mode 100755 index 0000000000..ccfec08c22 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding/sql/query01.sql @@ -0,0 +1,3 @@ +-- @description query01 for PXF Multibyte delimiter differentEncoding case + +SELECT * from pxf_multibyte_encoding ORDER BY num1; \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding_bytes/__init__.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding_bytes/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding_bytes/expected/query01.ans b/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding_bytes/expected/query01.ans new file mode 100755 index 0000000000..fd256950b4 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding_bytes/expected/query01.ans @@ -0,0 +1,9 @@ +-- @description query01 for PXF Multibyte delimiter encoding case with byte representation + +SELECT * from pxf_multibyte_encoding_bytes ORDER BY num1; + num1 | word +------+-------------------------------------- + 3 | règles d'automation + 4 | tá sé seo le tástáil dea- + 5 | minden amire szüksége van a szeretet +(3 rows) \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding_bytes/runTest.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding_bytes/runTest.py new file mode 100755 index 0000000000..b37b69894f --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding_bytes/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfMultibyteDelimDifferentEncodingBytes(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding_bytes/sql/query01.sql b/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding_bytes/sql/query01.sql new file mode 100755 index 0000000000..e72c483fb6 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding_bytes/sql/query01.sql @@ -0,0 +1,3 @@ +-- @description query01 for PXF Multibyte delimiter encoding case with byte representation + +SELECT * from pxf_multibyte_encoding_bytes ORDER BY num1; \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding_quote/__init__.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding_quote/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding_quote/expected/query01.ans b/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding_quote/expected/query01.ans new file mode 100755 index 0000000000..d89be58292 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding_quote/expected/query01.ans @@ -0,0 +1,9 @@ +-- @description query01 for PXF Multibyte delimiter differentEncoding case with quotes + +SELECT * from pxf_multibyte_encoding_quote ORDER BY num1; + num1 | word +------+-------------------------------------- + 3 | règles d'automation + 4 | tá sé seo le tástáil dea- + 5 | minden amire szüksége van a szeretet +(3 rows) \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding_quote/runTest.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding_quote/runTest.py new file mode 100755 index 0000000000..c0aa68cf43 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding_quote/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfMultibyteDelimDifferentEncodingWithQuote(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding_quote/sql/query01.sql b/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding_quote/sql/query01.sql new file mode 100755 index 0000000000..fb691ec1b8 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding_quote/sql/query01.sql @@ -0,0 +1,3 @@ +-- @description query01 for PXF Multibyte delimiter differentEncoding case with quotes + +SELECT * from pxf_multibyte_encoding_quote ORDER BY num1; \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding_quote_escape/__init__.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding_quote_escape/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding_quote_escape/expected/query01.ans b/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding_quote_escape/expected/query01.ans new file mode 100755 index 0000000000..c434a74dd7 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding_quote_escape/expected/query01.ans @@ -0,0 +1,9 @@ +-- @description query01 for PXF Multibyte delimiter differentEncoding case with quotes and escape + +SELECT * from pxf_multibyte_encoding_quote_escape ORDER BY num1; + num1 | word +------+-------------------------------------- + 3 | règles d'automation + 4 | tá sé seo le tástáil dea- + 5 | minden amire szüksége van a szeretet +(3 rows) \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding_quote_escape/runTest.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding_quote_escape/runTest.py new file mode 100755 index 0000000000..5e0b7172b1 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding_quote_escape/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfMultibyteDelimDifferentEncodingWithQuoteEscape(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding_quote_escape/sql/query01.sql b/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding_quote_escape/sql/query01.sql new file mode 100755 index 0000000000..43251ef4ca --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/encoding_quote_escape/sql/query01.sql @@ -0,0 +1,3 @@ +-- @description query01 for PXF Multibyte delimiter differentEncoding case with quotes and escape + +SELECT * from pxf_multibyte_encoding_quote_escape ORDER BY num1; \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/four_byte/__init__.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/four_byte/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/four_byte/expected/query01.ans b/automation/tincrepo/main/pxf/features/multibyte_delimiter/four_byte/expected/query01.ans new file mode 100755 index 0000000000..fc6afde521 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/four_byte/expected/query01.ans @@ -0,0 +1,201 @@ +-- @description query01 for PXF Multibyte delimiter, 4-byte delim cases + +SELECT * from pxf_multibyte_fourbyte_data ORDER BY n1; + s1 | s2 | s3 | d1 | n1 | n2 | n3 | n4 | n5 | n6 | n7 | s11 | s12 | s13 | d11 | n11 | n12 | n13 | n14 | n15 | n16 | n17 +-------+--------+---------+---------------------+-----+------+-------+-------+-------+-------+-------+-------+--------+---------+---------------------+-----+------+-------+-------+-------+-------+------- + s_1 | s_10 | s_100 | 1973-03-03 09:46:40 | 1 | 10 | 100 | 100 | 100 | 100 | 100 | s_1 | s_10 | s_100 | 1973-03-03 09:46:40 | 1 | 10 | 100 | 100 | 100 | 100 | 100 + s_2 | s_20 | s_200 | 1976-05-03 07:33:20 | 2 | 20 | 200 | 200 | 200 | 200 | 200 | s_2 | s_20 | s_200 | 1976-05-03 07:33:20 | 2 | 20 | 200 | 200 | 200 | 200 | 200 + s_3 | s_30 | s_300 | 1979-07-05 05:20:00 | 3 | 30 | 300 | 300 | 300 | 300 | 300 | s_3 | s_30 | s_300 | 1979-07-05 05:20:00 | 3 | 30 | 300 | 300 | 300 | 300 | 300 + s_4 | s_40 | s_400 | 1982-09-04 03:06:40 | 4 | 40 | 400 | 400 | 400 | 400 | 400 | s_4 | s_40 | s_400 | 1982-09-04 03:06:40 | 4 | 40 | 400 | 400 | 400 | 400 | 400 + s_5 | s_50 | s_500 | 1985-11-05 12:53:20 | 5 | 50 | 500 | 500 | 500 | 500 | 500 | s_5 | s_50 | s_500 | 1985-11-05 12:53:20 | 5 | 50 | 500 | 500 | 500 | 500 | 500 + s_6 | s_60 | s_600 | 1989-01-05 10:40:00 | 6 | 60 | 600 | 600 | 600 | 600 | 600 | s_6 | s_60 | s_600 | 1989-01-05 10:40:00 | 6 | 60 | 600 | 600 | 600 | 600 | 600 + s_7 | s_70 | s_700 | 1992-03-07 08:26:40 | 7 | 70 | 700 | 700 | 700 | 700 | 700 | s_7 | s_70 | s_700 | 1992-03-07 08:26:40 | 7 | 70 | 700 | 700 | 700 | 700 | 700 + s_8 | s_80 | s_800 | 1995-05-09 06:13:20 | 8 | 80 | 800 | 800 | 800 | 800 | 800 | s_8 | s_80 | s_800 | 1995-05-09 06:13:20 | 8 | 80 | 800 | 800 | 800 | 800 | 800 + s_9 | s_90 | s_900 | 1998-07-09 04:00:00 | 9 | 90 | 900 | 900 | 900 | 900 | 900 | s_9 | s_90 | s_900 | 1998-07-09 04:00:00 | 9 | 90 | 900 | 900 | 900 | 900 | 900 + s_10 | s_100 | s_1000 | 2001-09-09 01:46:40 | 10 | 100 | 1000 | 1000 | 1000 | 1000 | 1000 | s_10 | s_100 | s_1000 | 2001-09-09 01:46:40 | 10 | 100 | 1000 | 1000 | 1000 | 1000 | 1000 + s_11 | s_110 | s_1100 | 2004-11-09 11:33:20 | 11 | 110 | 1100 | 1100 | 1100 | 1100 | 1100 | s_11 | s_110 | s_1100 | 2004-11-09 11:33:20 | 11 | 110 | 1100 | 1100 | 1100 | 1100 | 1100 + s_12 | s_120 | s_1200 | 2008-01-10 09:20:00 | 12 | 120 | 1200 | 1200 | 1200 | 1200 | 1200 | s_12 | s_120 | s_1200 | 2008-01-10 09:20:00 | 12 | 120 | 1200 | 1200 | 1200 | 1200 | 1200 + s_13 | s_130 | s_1300 | 2011-03-13 07:06:40 | 13 | 130 | 1300 | 1300 | 1300 | 1300 | 1300 | s_13 | s_130 | s_1300 | 2011-03-13 07:06:40 | 13 | 130 | 1300 | 1300 | 1300 | 1300 | 1300 + s_14 | s_140 | s_1400 | 2014-05-13 04:53:20 | 14 | 140 | 1400 | 1400 | 1400 | 1400 | 1400 | s_14 | s_140 | s_1400 | 2014-05-13 04:53:20 | 14 | 140 | 1400 | 1400 | 1400 | 1400 | 1400 + s_15 | s_150 | s_1500 | 2017-07-14 02:40:00 | 15 | 150 | 1500 | 1500 | 1500 | 1500 | 1500 | s_15 | s_150 | s_1500 | 2017-07-14 02:40:00 | 15 | 150 | 1500 | 1500 | 1500 | 1500 | 1500 + s_16 | s_160 | s_1600 | 2020-09-13 12:26:40 | 16 | 160 | 1600 | 1600 | 1600 | 1600 | 1600 | s_16 | s_160 | s_1600 | 2020-09-13 12:26:40 | 16 | 160 | 1600 | 1600 | 1600 | 1600 | 1600 + s_17 | s_170 | s_1700 | 2023-11-14 10:13:20 | 17 | 170 | 1700 | 1700 | 1700 | 1700 | 1700 | s_17 | s_170 | s_1700 | 2023-11-14 10:13:20 | 17 | 170 | 1700 | 1700 | 1700 | 1700 | 1700 + s_18 | s_180 | s_1800 | 2027-01-15 08:00:00 | 18 | 180 | 1800 | 1800 | 1800 | 1800 | 1800 | s_18 | s_180 | s_1800 | 2027-01-15 08:00:00 | 18 | 180 | 1800 | 1800 | 1800 | 1800 | 1800 + s_19 | s_190 | s_1900 | 2030-03-17 05:46:40 | 19 | 190 | 1900 | 1900 | 1900 | 1900 | 1900 | s_19 | s_190 | s_1900 | 2030-03-17 05:46:40 | 19 | 190 | 1900 | 1900 | 1900 | 1900 | 1900 + s_20 | s_200 | s_2000 | 2033-05-18 03:33:20 | 20 | 200 | 2000 | 2000 | 2000 | 2000 | 2000 | s_20 | s_200 | s_2000 | 2033-05-18 03:33:20 | 20 | 200 | 2000 | 2000 | 2000 | 2000 | 2000 + s_21 | s_210 | s_2100 | 2036-07-18 01:20:00 | 21 | 210 | 2100 | 2100 | 2100 | 2100 | 2100 | s_21 | s_210 | s_2100 | 2036-07-18 01:20:00 | 21 | 210 | 2100 | 2100 | 2100 | 2100 | 2100 + s_22 | s_220 | s_2200 | 2039-09-18 11:06:40 | 22 | 220 | 2200 | 2200 | 2200 | 2200 | 2200 | s_22 | s_220 | s_2200 | 2039-09-18 11:06:40 | 22 | 220 | 2200 | 2200 | 2200 | 2200 | 2200 + s_23 | s_230 | s_2300 | 2042-11-19 08:53:20 | 23 | 230 | 2300 | 2300 | 2300 | 2300 | 2300 | s_23 | s_230 | s_2300 | 2042-11-19 08:53:20 | 23 | 230 | 2300 | 2300 | 2300 | 2300 | 2300 + s_24 | s_240 | s_2400 | 2046-01-19 06:40:00 | 24 | 240 | 2400 | 2400 | 2400 | 2400 | 2400 | s_24 | s_240 | s_2400 | 2046-01-19 06:40:00 | 24 | 240 | 2400 | 2400 | 2400 | 2400 | 2400 + s_25 | s_250 | s_2500 | 2049-03-22 04:26:40 | 25 | 250 | 2500 | 2500 | 2500 | 2500 | 2500 | s_25 | s_250 | s_2500 | 2049-03-22 04:26:40 | 25 | 250 | 2500 | 2500 | 2500 | 2500 | 2500 + s_26 | s_260 | s_2600 | 2052-05-22 02:13:20 | 26 | 260 | 2600 | 2600 | 2600 | 2600 | 2600 | s_26 | s_260 | s_2600 | 2052-05-22 02:13:20 | 26 | 260 | 2600 | 2600 | 2600 | 2600 | 2600 + s_27 | s_270 | s_2700 | 2055-07-24 12:00:00 | 27 | 270 | 2700 | 2700 | 2700 | 2700 | 2700 | s_27 | s_270 | s_2700 | 2055-07-24 12:00:00 | 27 | 270 | 2700 | 2700 | 2700 | 2700 | 2700 + s_28 | s_280 | s_2800 | 2058-09-23 09:46:40 | 28 | 280 | 2800 | 2800 | 2800 | 2800 | 2800 | s_28 | s_280 | s_2800 | 2058-09-23 09:46:40 | 28 | 280 | 2800 | 2800 | 2800 | 2800 | 2800 + s_29 | s_290 | s_2900 | 2061-11-23 07:33:20 | 29 | 290 | 2900 | 2900 | 2900 | 2900 | 2900 | s_29 | s_290 | s_2900 | 2061-11-23 07:33:20 | 29 | 290 | 2900 | 2900 | 2900 | 2900 | 2900 + s_30 | s_300 | s_3000 | 2065-01-24 05:20:00 | 30 | 300 | 3000 | 3000 | 3000 | 3000 | 3000 | s_30 | s_300 | s_3000 | 2065-01-24 05:20:00 | 30 | 300 | 3000 | 3000 | 3000 | 3000 | 3000 + s_31 | s_310 | s_3100 | 2068-03-26 03:06:40 | 31 | 310 | 3100 | 3100 | 3100 | 3100 | 3100 | s_31 | s_310 | s_3100 | 2068-03-26 03:06:40 | 31 | 310 | 3100 | 3100 | 3100 | 3100 | 3100 + s_32 | s_320 | s_3200 | 2071-05-28 12:53:20 | 32 | 320 | 3200 | 3200 | 3200 | 3200 | 3200 | s_32 | s_320 | s_3200 | 2071-05-28 12:53:20 | 32 | 320 | 3200 | 3200 | 3200 | 3200 | 3200 + s_33 | s_330 | s_3300 | 2074-07-28 10:40:00 | 33 | 330 | 3300 | 3300 | 3300 | 3300 | 3300 | s_33 | s_330 | s_3300 | 2074-07-28 10:40:00 | 33 | 330 | 3300 | 3300 | 3300 | 3300 | 3300 + s_34 | s_340 | s_3400 | 2077-09-27 08:26:40 | 34 | 340 | 3400 | 3400 | 3400 | 3400 | 3400 | s_34 | s_340 | s_3400 | 2077-09-27 08:26:40 | 34 | 340 | 3400 | 3400 | 3400 | 3400 | 3400 + s_35 | s_350 | s_3500 | 2080-11-28 06:13:20 | 35 | 350 | 3500 | 3500 | 3500 | 3500 | 3500 | s_35 | s_350 | s_3500 | 2080-11-28 06:13:20 | 35 | 350 | 3500 | 3500 | 3500 | 3500 | 3500 + s_36 | s_360 | s_3600 | 2084-01-29 04:00:00 | 36 | 360 | 3600 | 3600 | 3600 | 3600 | 3600 | s_36 | s_360 | s_3600 | 2084-01-29 04:00:00 | 36 | 360 | 3600 | 3600 | 3600 | 3600 | 3600 + s_37 | s_370 | s_3700 | 2087-04-01 01:46:40 | 37 | 370 | 3700 | 3700 | 3700 | 3700 | 3700 | s_37 | s_370 | s_3700 | 2087-04-01 01:46:40 | 37 | 370 | 3700 | 3700 | 3700 | 3700 | 3700 + s_38 | s_380 | s_3800 | 2090-06-01 11:33:20 | 38 | 380 | 3800 | 3800 | 3800 | 3800 | 3800 | s_38 | s_380 | s_3800 | 2090-06-01 11:33:20 | 38 | 380 | 3800 | 3800 | 3800 | 3800 | 3800 + s_39 | s_390 | s_3900 | 2093-08-01 09:20:00 | 39 | 390 | 3900 | 3900 | 3900 | 3900 | 3900 | s_39 | s_390 | s_3900 | 2093-08-01 09:20:00 | 39 | 390 | 3900 | 3900 | 3900 | 3900 | 3900 + s_40 | s_400 | s_4000 | 2096-10-02 07:06:40 | 40 | 400 | 4000 | 4000 | 4000 | 4000 | 4000 | s_40 | s_400 | s_4000 | 2096-10-02 07:06:40 | 40 | 400 | 4000 | 4000 | 4000 | 4000 | 4000 + s_41 | s_410 | s_4100 | 2099-12-03 04:53:20 | 41 | 410 | 4100 | 4100 | 4100 | 4100 | 4100 | s_41 | s_410 | s_4100 | 2099-12-03 04:53:20 | 41 | 410 | 4100 | 4100 | 4100 | 4100 | 4100 + s_42 | s_420 | s_4200 | 2103-02-04 02:40:00 | 42 | 420 | 4200 | 4200 | 4200 | 4200 | 4200 | s_42 | s_420 | s_4200 | 2103-02-04 02:40:00 | 42 | 420 | 4200 | 4200 | 4200 | 4200 | 4200 + s_43 | s_430 | s_4300 | 2106-04-06 12:26:40 | 43 | 430 | 4300 | 4300 | 4300 | 4300 | 4300 | s_43 | s_430 | s_4300 | 2106-04-06 12:26:40 | 43 | 430 | 4300 | 4300 | 4300 | 4300 | 4300 + s_44 | s_440 | s_4400 | 2109-06-06 10:13:20 | 44 | 440 | 4400 | 4400 | 4400 | 4400 | 4400 | s_44 | s_440 | s_4400 | 2109-06-06 10:13:20 | 44 | 440 | 4400 | 4400 | 4400 | 4400 | 4400 + s_45 | s_450 | s_4500 | 2112-08-07 08:00:00 | 45 | 450 | 4500 | 4500 | 4500 | 4500 | 4500 | s_45 | s_450 | s_4500 | 2112-08-07 08:00:00 | 45 | 450 | 4500 | 4500 | 4500 | 4500 | 4500 + s_46 | s_460 | s_4600 | 2115-10-08 05:46:40 | 46 | 460 | 4600 | 4600 | 4600 | 4600 | 4600 | s_46 | s_460 | s_4600 | 2115-10-08 05:46:40 | 46 | 460 | 4600 | 4600 | 4600 | 4600 | 4600 + s_47 | s_470 | s_4700 | 2118-12-09 03:33:20 | 47 | 470 | 4700 | 4700 | 4700 | 4700 | 4700 | s_47 | s_470 | s_4700 | 2118-12-09 03:33:20 | 47 | 470 | 4700 | 4700 | 4700 | 4700 | 4700 + s_48 | s_480 | s_4800 | 2122-02-08 01:20:00 | 48 | 480 | 4800 | 4800 | 4800 | 4800 | 4800 | s_48 | s_480 | s_4800 | 2122-02-08 01:20:00 | 48 | 480 | 4800 | 4800 | 4800 | 4800 | 4800 + s_49 | s_490 | s_4900 | 2125-04-10 11:06:40 | 49 | 490 | 4900 | 4900 | 4900 | 4900 | 4900 | s_49 | s_490 | s_4900 | 2125-04-10 11:06:40 | 49 | 490 | 4900 | 4900 | 4900 | 4900 | 4900 + s_50 | s_500 | s_5000 | 2128-06-11 08:53:20 | 50 | 500 | 5000 | 5000 | 5000 | 5000 | 5000 | s_50 | s_500 | s_5000 | 2128-06-11 08:53:20 | 50 | 500 | 5000 | 5000 | 5000 | 5000 | 5000 + s_51 | s_510 | s_5100 | 2131-08-12 06:40:00 | 51 | 510 | 5100 | 5100 | 5100 | 5100 | 5100 | s_51 | s_510 | s_5100 | 2131-08-12 06:40:00 | 51 | 510 | 5100 | 5100 | 5100 | 5100 | 5100 + s_52 | s_520 | s_5200 | 2134-10-13 04:26:40 | 52 | 520 | 5200 | 5200 | 5200 | 5200 | 5200 | s_52 | s_520 | s_5200 | 2134-10-13 04:26:40 | 52 | 520 | 5200 | 5200 | 5200 | 5200 | 5200 + s_53 | s_530 | s_5300 | 2137-12-13 02:13:20 | 53 | 530 | 5300 | 5300 | 5300 | 5300 | 5300 | s_53 | s_530 | s_5300 | 2137-12-13 02:13:20 | 53 | 530 | 5300 | 5300 | 5300 | 5300 | 5300 + s_54 | s_540 | s_5400 | 2141-02-13 12:00:00 | 54 | 540 | 5400 | 5400 | 5400 | 5400 | 5400 | s_54 | s_540 | s_5400 | 2141-02-13 12:00:00 | 54 | 540 | 5400 | 5400 | 5400 | 5400 | 5400 + s_55 | s_550 | s_5500 | 2144-04-15 09:46:40 | 55 | 550 | 5500 | 5500 | 5500 | 5500 | 5500 | s_55 | s_550 | s_5500 | 2144-04-15 09:46:40 | 55 | 550 | 5500 | 5500 | 5500 | 5500 | 5500 + s_56 | s_560 | s_5600 | 2147-06-16 07:33:20 | 56 | 560 | 5600 | 5600 | 5600 | 5600 | 5600 | s_56 | s_560 | s_5600 | 2147-06-16 07:33:20 | 56 | 560 | 5600 | 5600 | 5600 | 5600 | 5600 + s_57 | s_570 | s_5700 | 2150-08-17 05:20:00 | 57 | 570 | 5700 | 5700 | 5700 | 5700 | 5700 | s_57 | s_570 | s_5700 | 2150-08-17 05:20:00 | 57 | 570 | 5700 | 5700 | 5700 | 5700 | 5700 + s_58 | s_580 | s_5800 | 2153-10-17 03:06:40 | 58 | 580 | 5800 | 5800 | 5800 | 5800 | 5800 | s_58 | s_580 | s_5800 | 2153-10-17 03:06:40 | 58 | 580 | 5800 | 5800 | 5800 | 5800 | 5800 + s_59 | s_590 | s_5900 | 2156-12-18 12:53:20 | 59 | 590 | 5900 | 5900 | 5900 | 5900 | 5900 | s_59 | s_590 | s_5900 | 2156-12-18 12:53:20 | 59 | 590 | 5900 | 5900 | 5900 | 5900 | 5900 + s_60 | s_600 | s_6000 | 2160-02-18 10:40:00 | 60 | 600 | 6000 | 6000 | 6000 | 6000 | 6000 | s_60 | s_600 | s_6000 | 2160-02-18 10:40:00 | 60 | 600 | 6000 | 6000 | 6000 | 6000 | 6000 + s_61 | s_610 | s_6100 | 2163-04-20 08:26:40 | 61 | 610 | 6100 | 6100 | 6100 | 6100 | 6100 | s_61 | s_610 | s_6100 | 2163-04-20 08:26:40 | 61 | 610 | 6100 | 6100 | 6100 | 6100 | 6100 + s_62 | s_620 | s_6200 | 2166-06-21 06:13:20 | 62 | 620 | 6200 | 6200 | 6200 | 6200 | 6200 | s_62 | s_620 | s_6200 | 2166-06-21 06:13:20 | 62 | 620 | 6200 | 6200 | 6200 | 6200 | 6200 + s_63 | s_630 | s_6300 | 2169-08-21 04:00:00 | 63 | 630 | 6300 | 6300 | 6300 | 6300 | 6300 | s_63 | s_630 | s_6300 | 2169-08-21 04:00:00 | 63 | 630 | 6300 | 6300 | 6300 | 6300 | 6300 + s_64 | s_640 | s_6400 | 2172-10-22 01:46:40 | 64 | 640 | 6400 | 6400 | 6400 | 6400 | 6400 | s_64 | s_640 | s_6400 | 2172-10-22 01:46:40 | 64 | 640 | 6400 | 6400 | 6400 | 6400 | 6400 + s_65 | s_650 | s_6500 | 2175-12-23 11:33:20 | 65 | 650 | 6500 | 6500 | 6500 | 6500 | 6500 | s_65 | s_650 | s_6500 | 2175-12-23 11:33:20 | 65 | 650 | 6500 | 6500 | 6500 | 6500 | 6500 + s_66 | s_660 | s_6600 | 2179-02-22 09:20:00 | 66 | 660 | 6600 | 6600 | 6600 | 6600 | 6600 | s_66 | s_660 | s_6600 | 2179-02-22 09:20:00 | 66 | 660 | 6600 | 6600 | 6600 | 6600 | 6600 + s_67 | s_670 | s_6700 | 2182-04-25 07:06:40 | 67 | 670 | 6700 | 6700 | 6700 | 6700 | 6700 | s_67 | s_670 | s_6700 | 2182-04-25 07:06:40 | 67 | 670 | 6700 | 6700 | 6700 | 6700 | 6700 + s_68 | s_680 | s_6800 | 2185-06-25 04:53:20 | 68 | 680 | 6800 | 6800 | 6800 | 6800 | 6800 | s_68 | s_680 | s_6800 | 2185-06-25 04:53:20 | 68 | 680 | 6800 | 6800 | 6800 | 6800 | 6800 + s_69 | s_690 | s_6900 | 2188-08-26 02:40:00 | 69 | 690 | 6900 | 6900 | 6900 | 6900 | 6900 | s_69 | s_690 | s_6900 | 2188-08-26 02:40:00 | 69 | 690 | 6900 | 6900 | 6900 | 6900 | 6900 + s_70 | s_700 | s_7000 | 2191-10-27 12:26:40 | 70 | 700 | 7000 | 7000 | 7000 | 7000 | 7000 | s_70 | s_700 | s_7000 | 2191-10-27 12:26:40 | 70 | 700 | 7000 | 7000 | 7000 | 7000 | 7000 + s_71 | s_710 | s_7100 | 2194-12-27 10:13:20 | 71 | 710 | 7100 | 7100 | 7100 | 7100 | 7100 | s_71 | s_710 | s_7100 | 2194-12-27 10:13:20 | 71 | 710 | 7100 | 7100 | 7100 | 7100 | 7100 + s_72 | s_720 | s_7200 | 2198-02-27 08:00:00 | 72 | 720 | 7200 | 7200 | 7200 | 7200 | 7200 | s_72 | s_720 | s_7200 | 2198-02-27 08:00:00 | 72 | 720 | 7200 | 7200 | 7200 | 7200 | 7200 + s_73 | s_730 | s_7300 | 2201-04-30 05:46:40 | 73 | 730 | 7300 | 7300 | 7300 | 7300 | 7300 | s_73 | s_730 | s_7300 | 2201-04-30 05:46:40 | 73 | 730 | 7300 | 7300 | 7300 | 7300 | 7300 + s_74 | s_740 | s_7400 | 2204-07-01 03:33:20 | 74 | 740 | 7400 | 7400 | 7400 | 7400 | 7400 | s_74 | s_740 | s_7400 | 2204-07-01 03:33:20 | 74 | 740 | 7400 | 7400 | 7400 | 7400 | 7400 + s_75 | s_750 | s_7500 | 2207-09-01 01:20:00 | 75 | 750 | 7500 | 7500 | 7500 | 7500 | 7500 | s_75 | s_750 | s_7500 | 2207-09-01 01:20:00 | 75 | 750 | 7500 | 7500 | 7500 | 7500 | 7500 + s_76 | s_760 | s_7600 | 2210-11-01 11:06:40 | 76 | 760 | 7600 | 7600 | 7600 | 7600 | 7600 | s_76 | s_760 | s_7600 | 2210-11-01 11:06:40 | 76 | 760 | 7600 | 7600 | 7600 | 7600 | 7600 + s_77 | s_770 | s_7700 | 2214-01-02 08:53:20 | 77 | 770 | 7700 | 7700 | 7700 | 7700 | 7700 | s_77 | s_770 | s_7700 | 2214-01-02 08:53:20 | 77 | 770 | 7700 | 7700 | 7700 | 7700 | 7700 + s_78 | s_780 | s_7800 | 2217-03-04 06:40:00 | 78 | 780 | 7800 | 7800 | 7800 | 7800 | 7800 | s_78 | s_780 | s_7800 | 2217-03-04 06:40:00 | 78 | 780 | 7800 | 7800 | 7800 | 7800 | 7800 + s_79 | s_790 | s_7900 | 2220-05-05 04:26:40 | 79 | 790 | 7900 | 7900 | 7900 | 7900 | 7900 | s_79 | s_790 | s_7900 | 2220-05-05 04:26:40 | 79 | 790 | 7900 | 7900 | 7900 | 7900 | 7900 + s_80 | s_800 | s_8000 | 2223-07-06 02:13:20 | 80 | 800 | 8000 | 8000 | 8000 | 8000 | 8000 | s_80 | s_800 | s_8000 | 2223-07-06 02:13:20 | 80 | 800 | 8000 | 8000 | 8000 | 8000 | 8000 + s_81 | s_810 | s_8100 | 2226-09-06 12:00:00 | 81 | 810 | 8100 | 8100 | 8100 | 8100 | 8100 | s_81 | s_810 | s_8100 | 2226-09-06 12:00:00 | 81 | 810 | 8100 | 8100 | 8100 | 8100 | 8100 + s_82 | s_820 | s_8200 | 2229-11-06 09:46:40 | 82 | 820 | 8200 | 8200 | 8200 | 8200 | 8200 | s_82 | s_820 | s_8200 | 2229-11-06 09:46:40 | 82 | 820 | 8200 | 8200 | 8200 | 8200 | 8200 + s_83 | s_830 | s_8300 | 2233-01-06 07:33:20 | 83 | 830 | 8300 | 8300 | 8300 | 8300 | 8300 | s_83 | s_830 | s_8300 | 2233-01-06 07:33:20 | 83 | 830 | 8300 | 8300 | 8300 | 8300 | 8300 + s_84 | s_840 | s_8400 | 2236-03-09 05:20:00 | 84 | 840 | 8400 | 8400 | 8400 | 8400 | 8400 | s_84 | s_840 | s_8400 | 2236-03-09 05:20:00 | 84 | 840 | 8400 | 8400 | 8400 | 8400 | 8400 + s_85 | s_850 | s_8500 | 2239-05-10 03:06:40 | 85 | 850 | 8500 | 8500 | 8500 | 8500 | 8500 | s_85 | s_850 | s_8500 | 2239-05-10 03:06:40 | 85 | 850 | 8500 | 8500 | 8500 | 8500 | 8500 + s_86 | s_860 | s_8600 | 2242-07-11 12:53:20 | 86 | 860 | 8600 | 8600 | 8600 | 8600 | 8600 | s_86 | s_860 | s_8600 | 2242-07-11 12:53:20 | 86 | 860 | 8600 | 8600 | 8600 | 8600 | 8600 + s_87 | s_870 | s_8700 | 2245-09-10 10:40:00 | 87 | 870 | 8700 | 8700 | 8700 | 8700 | 8700 | s_87 | s_870 | s_8700 | 2245-09-10 10:40:00 | 87 | 870 | 8700 | 8700 | 8700 | 8700 | 8700 + s_88 | s_880 | s_8800 | 2248-11-10 08:26:40 | 88 | 880 | 8800 | 8800 | 8800 | 8800 | 8800 | s_88 | s_880 | s_8800 | 2248-11-10 08:26:40 | 88 | 880 | 8800 | 8800 | 8800 | 8800 | 8800 + s_89 | s_890 | s_8900 | 2252-01-12 06:13:20 | 89 | 890 | 8900 | 8900 | 8900 | 8900 | 8900 | s_89 | s_890 | s_8900 | 2252-01-12 06:13:20 | 89 | 890 | 8900 | 8900 | 8900 | 8900 | 8900 + s_90 | s_900 | s_9000 | 2255-03-14 04:00:00 | 90 | 900 | 9000 | 9000 | 9000 | 9000 | 9000 | s_90 | s_900 | s_9000 | 2255-03-14 04:00:00 | 90 | 900 | 9000 | 9000 | 9000 | 9000 | 9000 + s_91 | s_910 | s_9100 | 2258-05-15 01:46:40 | 91 | 910 | 9100 | 9100 | 9100 | 9100 | 9100 | s_91 | s_910 | s_9100 | 2258-05-15 01:46:40 | 91 | 910 | 9100 | 9100 | 9100 | 9100 | 9100 + s_92 | s_920 | s_9200 | 2261-07-15 11:33:20 | 92 | 920 | 9200 | 9200 | 9200 | 9200 | 9200 | s_92 | s_920 | s_9200 | 2261-07-15 11:33:20 | 92 | 920 | 9200 | 9200 | 9200 | 9200 | 9200 + s_93 | s_930 | s_9300 | 2264-09-14 09:20:00 | 93 | 930 | 9300 | 9300 | 9300 | 9300 | 9300 | s_93 | s_930 | s_9300 | 2264-09-14 09:20:00 | 93 | 930 | 9300 | 9300 | 9300 | 9300 | 9300 + s_94 | s_940 | s_9400 | 2267-11-16 07:06:40 | 94 | 940 | 9400 | 9400 | 9400 | 9400 | 9400 | s_94 | s_940 | s_9400 | 2267-11-16 07:06:40 | 94 | 940 | 9400 | 9400 | 9400 | 9400 | 9400 + s_95 | s_950 | s_9500 | 2271-01-16 04:53:20 | 95 | 950 | 9500 | 9500 | 9500 | 9500 | 9500 | s_95 | s_950 | s_9500 | 2271-01-16 04:53:20 | 95 | 950 | 9500 | 9500 | 9500 | 9500 | 9500 + s_96 | s_960 | s_9600 | 2274-03-19 02:40:00 | 96 | 960 | 9600 | 9600 | 9600 | 9600 | 9600 | s_96 | s_960 | s_9600 | 2274-03-19 02:40:00 | 96 | 960 | 9600 | 9600 | 9600 | 9600 | 9600 + s_97 | s_970 | s_9700 | 2277-05-19 12:26:40 | 97 | 970 | 9700 | 9700 | 9700 | 9700 | 9700 | s_97 | s_970 | s_9700 | 2277-05-19 12:26:40 | 97 | 970 | 9700 | 9700 | 9700 | 9700 | 9700 + s_98 | s_980 | s_9800 | 2280-07-19 10:13:20 | 98 | 980 | 9800 | 9800 | 9800 | 9800 | 9800 | s_98 | s_980 | s_9800 | 2280-07-19 10:13:20 | 98 | 980 | 9800 | 9800 | 9800 | 9800 | 9800 + s_99 | s_990 | s_9900 | 2283-09-20 08:00:00 | 99 | 990 | 9900 | 9900 | 9900 | 9900 | 9900 | s_99 | s_990 | s_9900 | 2283-09-20 08:00:00 | 99 | 990 | 9900 | 9900 | 9900 | 9900 | 9900 + s_100 | s_1000 | s_10000 | 2286-11-20 05:46:40 | 100 | 1000 | 10000 | 10000 | 10000 | 10000 | 10000 | s_100 | s_1000 | s_10000 | 2286-11-20 05:46:40 | 100 | 1000 | 10000 | 10000 | 10000 | 10000 | 10000 +(100 rows) + +SELECT * from pxf_multibyte_fourbyte_data_with_skip ORDER BY n1; + s1 | s2 | s3 | d1 | n1 | n2 | n3 | n4 | n5 | n6 | n7 | s11 | s12 | s13 | d11 | n11 | n12 | n13 | n14 | n15 | n16 | n17 +-------+--------+---------+---------------------+-----+------+-------+-------+-------+-------+-------+-------+--------+---------+---------------------+-----+------+-------+-------+-------+-------+------- + s_11 | s_110 | s_1100 | 2004-11-09 11:33:20 | 11 | 110 | 1100 | 1100 | 1100 | 1100 | 1100 | s_11 | s_110 | s_1100 | 2004-11-09 11:33:20 | 11 | 110 | 1100 | 1100 | 1100 | 1100 | 1100 + s_12 | s_120 | s_1200 | 2008-01-10 09:20:00 | 12 | 120 | 1200 | 1200 | 1200 | 1200 | 1200 | s_12 | s_120 | s_1200 | 2008-01-10 09:20:00 | 12 | 120 | 1200 | 1200 | 1200 | 1200 | 1200 + s_13 | s_130 | s_1300 | 2011-03-13 07:06:40 | 13 | 130 | 1300 | 1300 | 1300 | 1300 | 1300 | s_13 | s_130 | s_1300 | 2011-03-13 07:06:40 | 13 | 130 | 1300 | 1300 | 1300 | 1300 | 1300 + s_14 | s_140 | s_1400 | 2014-05-13 04:53:20 | 14 | 140 | 1400 | 1400 | 1400 | 1400 | 1400 | s_14 | s_140 | s_1400 | 2014-05-13 04:53:20 | 14 | 140 | 1400 | 1400 | 1400 | 1400 | 1400 + s_15 | s_150 | s_1500 | 2017-07-14 02:40:00 | 15 | 150 | 1500 | 1500 | 1500 | 1500 | 1500 | s_15 | s_150 | s_1500 | 2017-07-14 02:40:00 | 15 | 150 | 1500 | 1500 | 1500 | 1500 | 1500 + s_16 | s_160 | s_1600 | 2020-09-13 12:26:40 | 16 | 160 | 1600 | 1600 | 1600 | 1600 | 1600 | s_16 | s_160 | s_1600 | 2020-09-13 12:26:40 | 16 | 160 | 1600 | 1600 | 1600 | 1600 | 1600 + s_17 | s_170 | s_1700 | 2023-11-14 10:13:20 | 17 | 170 | 1700 | 1700 | 1700 | 1700 | 1700 | s_17 | s_170 | s_1700 | 2023-11-14 10:13:20 | 17 | 170 | 1700 | 1700 | 1700 | 1700 | 1700 + s_18 | s_180 | s_1800 | 2027-01-15 08:00:00 | 18 | 180 | 1800 | 1800 | 1800 | 1800 | 1800 | s_18 | s_180 | s_1800 | 2027-01-15 08:00:00 | 18 | 180 | 1800 | 1800 | 1800 | 1800 | 1800 + s_19 | s_190 | s_1900 | 2030-03-17 05:46:40 | 19 | 190 | 1900 | 1900 | 1900 | 1900 | 1900 | s_19 | s_190 | s_1900 | 2030-03-17 05:46:40 | 19 | 190 | 1900 | 1900 | 1900 | 1900 | 1900 + s_20 | s_200 | s_2000 | 2033-05-18 03:33:20 | 20 | 200 | 2000 | 2000 | 2000 | 2000 | 2000 | s_20 | s_200 | s_2000 | 2033-05-18 03:33:20 | 20 | 200 | 2000 | 2000 | 2000 | 2000 | 2000 + s_21 | s_210 | s_2100 | 2036-07-18 01:20:00 | 21 | 210 | 2100 | 2100 | 2100 | 2100 | 2100 | s_21 | s_210 | s_2100 | 2036-07-18 01:20:00 | 21 | 210 | 2100 | 2100 | 2100 | 2100 | 2100 + s_22 | s_220 | s_2200 | 2039-09-18 11:06:40 | 22 | 220 | 2200 | 2200 | 2200 | 2200 | 2200 | s_22 | s_220 | s_2200 | 2039-09-18 11:06:40 | 22 | 220 | 2200 | 2200 | 2200 | 2200 | 2200 + s_23 | s_230 | s_2300 | 2042-11-19 08:53:20 | 23 | 230 | 2300 | 2300 | 2300 | 2300 | 2300 | s_23 | s_230 | s_2300 | 2042-11-19 08:53:20 | 23 | 230 | 2300 | 2300 | 2300 | 2300 | 2300 + s_24 | s_240 | s_2400 | 2046-01-19 06:40:00 | 24 | 240 | 2400 | 2400 | 2400 | 2400 | 2400 | s_24 | s_240 | s_2400 | 2046-01-19 06:40:00 | 24 | 240 | 2400 | 2400 | 2400 | 2400 | 2400 + s_25 | s_250 | s_2500 | 2049-03-22 04:26:40 | 25 | 250 | 2500 | 2500 | 2500 | 2500 | 2500 | s_25 | s_250 | s_2500 | 2049-03-22 04:26:40 | 25 | 250 | 2500 | 2500 | 2500 | 2500 | 2500 + s_26 | s_260 | s_2600 | 2052-05-22 02:13:20 | 26 | 260 | 2600 | 2600 | 2600 | 2600 | 2600 | s_26 | s_260 | s_2600 | 2052-05-22 02:13:20 | 26 | 260 | 2600 | 2600 | 2600 | 2600 | 2600 + s_27 | s_270 | s_2700 | 2055-07-24 12:00:00 | 27 | 270 | 2700 | 2700 | 2700 | 2700 | 2700 | s_27 | s_270 | s_2700 | 2055-07-24 12:00:00 | 27 | 270 | 2700 | 2700 | 2700 | 2700 | 2700 + s_28 | s_280 | s_2800 | 2058-09-23 09:46:40 | 28 | 280 | 2800 | 2800 | 2800 | 2800 | 2800 | s_28 | s_280 | s_2800 | 2058-09-23 09:46:40 | 28 | 280 | 2800 | 2800 | 2800 | 2800 | 2800 + s_29 | s_290 | s_2900 | 2061-11-23 07:33:20 | 29 | 290 | 2900 | 2900 | 2900 | 2900 | 2900 | s_29 | s_290 | s_2900 | 2061-11-23 07:33:20 | 29 | 290 | 2900 | 2900 | 2900 | 2900 | 2900 + s_30 | s_300 | s_3000 | 2065-01-24 05:20:00 | 30 | 300 | 3000 | 3000 | 3000 | 3000 | 3000 | s_30 | s_300 | s_3000 | 2065-01-24 05:20:00 | 30 | 300 | 3000 | 3000 | 3000 | 3000 | 3000 + s_31 | s_310 | s_3100 | 2068-03-26 03:06:40 | 31 | 310 | 3100 | 3100 | 3100 | 3100 | 3100 | s_31 | s_310 | s_3100 | 2068-03-26 03:06:40 | 31 | 310 | 3100 | 3100 | 3100 | 3100 | 3100 + s_32 | s_320 | s_3200 | 2071-05-28 12:53:20 | 32 | 320 | 3200 | 3200 | 3200 | 3200 | 3200 | s_32 | s_320 | s_3200 | 2071-05-28 12:53:20 | 32 | 320 | 3200 | 3200 | 3200 | 3200 | 3200 + s_33 | s_330 | s_3300 | 2074-07-28 10:40:00 | 33 | 330 | 3300 | 3300 | 3300 | 3300 | 3300 | s_33 | s_330 | s_3300 | 2074-07-28 10:40:00 | 33 | 330 | 3300 | 3300 | 3300 | 3300 | 3300 + s_34 | s_340 | s_3400 | 2077-09-27 08:26:40 | 34 | 340 | 3400 | 3400 | 3400 | 3400 | 3400 | s_34 | s_340 | s_3400 | 2077-09-27 08:26:40 | 34 | 340 | 3400 | 3400 | 3400 | 3400 | 3400 + s_35 | s_350 | s_3500 | 2080-11-28 06:13:20 | 35 | 350 | 3500 | 3500 | 3500 | 3500 | 3500 | s_35 | s_350 | s_3500 | 2080-11-28 06:13:20 | 35 | 350 | 3500 | 3500 | 3500 | 3500 | 3500 + s_36 | s_360 | s_3600 | 2084-01-29 04:00:00 | 36 | 360 | 3600 | 3600 | 3600 | 3600 | 3600 | s_36 | s_360 | s_3600 | 2084-01-29 04:00:00 | 36 | 360 | 3600 | 3600 | 3600 | 3600 | 3600 + s_37 | s_370 | s_3700 | 2087-04-01 01:46:40 | 37 | 370 | 3700 | 3700 | 3700 | 3700 | 3700 | s_37 | s_370 | s_3700 | 2087-04-01 01:46:40 | 37 | 370 | 3700 | 3700 | 3700 | 3700 | 3700 + s_38 | s_380 | s_3800 | 2090-06-01 11:33:20 | 38 | 380 | 3800 | 3800 | 3800 | 3800 | 3800 | s_38 | s_380 | s_3800 | 2090-06-01 11:33:20 | 38 | 380 | 3800 | 3800 | 3800 | 3800 | 3800 + s_39 | s_390 | s_3900 | 2093-08-01 09:20:00 | 39 | 390 | 3900 | 3900 | 3900 | 3900 | 3900 | s_39 | s_390 | s_3900 | 2093-08-01 09:20:00 | 39 | 390 | 3900 | 3900 | 3900 | 3900 | 3900 + s_40 | s_400 | s_4000 | 2096-10-02 07:06:40 | 40 | 400 | 4000 | 4000 | 4000 | 4000 | 4000 | s_40 | s_400 | s_4000 | 2096-10-02 07:06:40 | 40 | 400 | 4000 | 4000 | 4000 | 4000 | 4000 + s_41 | s_410 | s_4100 | 2099-12-03 04:53:20 | 41 | 410 | 4100 | 4100 | 4100 | 4100 | 4100 | s_41 | s_410 | s_4100 | 2099-12-03 04:53:20 | 41 | 410 | 4100 | 4100 | 4100 | 4100 | 4100 + s_42 | s_420 | s_4200 | 2103-02-04 02:40:00 | 42 | 420 | 4200 | 4200 | 4200 | 4200 | 4200 | s_42 | s_420 | s_4200 | 2103-02-04 02:40:00 | 42 | 420 | 4200 | 4200 | 4200 | 4200 | 4200 + s_43 | s_430 | s_4300 | 2106-04-06 12:26:40 | 43 | 430 | 4300 | 4300 | 4300 | 4300 | 4300 | s_43 | s_430 | s_4300 | 2106-04-06 12:26:40 | 43 | 430 | 4300 | 4300 | 4300 | 4300 | 4300 + s_44 | s_440 | s_4400 | 2109-06-06 10:13:20 | 44 | 440 | 4400 | 4400 | 4400 | 4400 | 4400 | s_44 | s_440 | s_4400 | 2109-06-06 10:13:20 | 44 | 440 | 4400 | 4400 | 4400 | 4400 | 4400 + s_45 | s_450 | s_4500 | 2112-08-07 08:00:00 | 45 | 450 | 4500 | 4500 | 4500 | 4500 | 4500 | s_45 | s_450 | s_4500 | 2112-08-07 08:00:00 | 45 | 450 | 4500 | 4500 | 4500 | 4500 | 4500 + s_46 | s_460 | s_4600 | 2115-10-08 05:46:40 | 46 | 460 | 4600 | 4600 | 4600 | 4600 | 4600 | s_46 | s_460 | s_4600 | 2115-10-08 05:46:40 | 46 | 460 | 4600 | 4600 | 4600 | 4600 | 4600 + s_47 | s_470 | s_4700 | 2118-12-09 03:33:20 | 47 | 470 | 4700 | 4700 | 4700 | 4700 | 4700 | s_47 | s_470 | s_4700 | 2118-12-09 03:33:20 | 47 | 470 | 4700 | 4700 | 4700 | 4700 | 4700 + s_48 | s_480 | s_4800 | 2122-02-08 01:20:00 | 48 | 480 | 4800 | 4800 | 4800 | 4800 | 4800 | s_48 | s_480 | s_4800 | 2122-02-08 01:20:00 | 48 | 480 | 4800 | 4800 | 4800 | 4800 | 4800 + s_49 | s_490 | s_4900 | 2125-04-10 11:06:40 | 49 | 490 | 4900 | 4900 | 4900 | 4900 | 4900 | s_49 | s_490 | s_4900 | 2125-04-10 11:06:40 | 49 | 490 | 4900 | 4900 | 4900 | 4900 | 4900 + s_50 | s_500 | s_5000 | 2128-06-11 08:53:20 | 50 | 500 | 5000 | 5000 | 5000 | 5000 | 5000 | s_50 | s_500 | s_5000 | 2128-06-11 08:53:20 | 50 | 500 | 5000 | 5000 | 5000 | 5000 | 5000 + s_51 | s_510 | s_5100 | 2131-08-12 06:40:00 | 51 | 510 | 5100 | 5100 | 5100 | 5100 | 5100 | s_51 | s_510 | s_5100 | 2131-08-12 06:40:00 | 51 | 510 | 5100 | 5100 | 5100 | 5100 | 5100 + s_52 | s_520 | s_5200 | 2134-10-13 04:26:40 | 52 | 520 | 5200 | 5200 | 5200 | 5200 | 5200 | s_52 | s_520 | s_5200 | 2134-10-13 04:26:40 | 52 | 520 | 5200 | 5200 | 5200 | 5200 | 5200 + s_53 | s_530 | s_5300 | 2137-12-13 02:13:20 | 53 | 530 | 5300 | 5300 | 5300 | 5300 | 5300 | s_53 | s_530 | s_5300 | 2137-12-13 02:13:20 | 53 | 530 | 5300 | 5300 | 5300 | 5300 | 5300 + s_54 | s_540 | s_5400 | 2141-02-13 12:00:00 | 54 | 540 | 5400 | 5400 | 5400 | 5400 | 5400 | s_54 | s_540 | s_5400 | 2141-02-13 12:00:00 | 54 | 540 | 5400 | 5400 | 5400 | 5400 | 5400 + s_55 | s_550 | s_5500 | 2144-04-15 09:46:40 | 55 | 550 | 5500 | 5500 | 5500 | 5500 | 5500 | s_55 | s_550 | s_5500 | 2144-04-15 09:46:40 | 55 | 550 | 5500 | 5500 | 5500 | 5500 | 5500 + s_56 | s_560 | s_5600 | 2147-06-16 07:33:20 | 56 | 560 | 5600 | 5600 | 5600 | 5600 | 5600 | s_56 | s_560 | s_5600 | 2147-06-16 07:33:20 | 56 | 560 | 5600 | 5600 | 5600 | 5600 | 5600 + s_57 | s_570 | s_5700 | 2150-08-17 05:20:00 | 57 | 570 | 5700 | 5700 | 5700 | 5700 | 5700 | s_57 | s_570 | s_5700 | 2150-08-17 05:20:00 | 57 | 570 | 5700 | 5700 | 5700 | 5700 | 5700 + s_58 | s_580 | s_5800 | 2153-10-17 03:06:40 | 58 | 580 | 5800 | 5800 | 5800 | 5800 | 5800 | s_58 | s_580 | s_5800 | 2153-10-17 03:06:40 | 58 | 580 | 5800 | 5800 | 5800 | 5800 | 5800 + s_59 | s_590 | s_5900 | 2156-12-18 12:53:20 | 59 | 590 | 5900 | 5900 | 5900 | 5900 | 5900 | s_59 | s_590 | s_5900 | 2156-12-18 12:53:20 | 59 | 590 | 5900 | 5900 | 5900 | 5900 | 5900 + s_60 | s_600 | s_6000 | 2160-02-18 10:40:00 | 60 | 600 | 6000 | 6000 | 6000 | 6000 | 6000 | s_60 | s_600 | s_6000 | 2160-02-18 10:40:00 | 60 | 600 | 6000 | 6000 | 6000 | 6000 | 6000 + s_61 | s_610 | s_6100 | 2163-04-20 08:26:40 | 61 | 610 | 6100 | 6100 | 6100 | 6100 | 6100 | s_61 | s_610 | s_6100 | 2163-04-20 08:26:40 | 61 | 610 | 6100 | 6100 | 6100 | 6100 | 6100 + s_62 | s_620 | s_6200 | 2166-06-21 06:13:20 | 62 | 620 | 6200 | 6200 | 6200 | 6200 | 6200 | s_62 | s_620 | s_6200 | 2166-06-21 06:13:20 | 62 | 620 | 6200 | 6200 | 6200 | 6200 | 6200 + s_63 | s_630 | s_6300 | 2169-08-21 04:00:00 | 63 | 630 | 6300 | 6300 | 6300 | 6300 | 6300 | s_63 | s_630 | s_6300 | 2169-08-21 04:00:00 | 63 | 630 | 6300 | 6300 | 6300 | 6300 | 6300 + s_64 | s_640 | s_6400 | 2172-10-22 01:46:40 | 64 | 640 | 6400 | 6400 | 6400 | 6400 | 6400 | s_64 | s_640 | s_6400 | 2172-10-22 01:46:40 | 64 | 640 | 6400 | 6400 | 6400 | 6400 | 6400 + s_65 | s_650 | s_6500 | 2175-12-23 11:33:20 | 65 | 650 | 6500 | 6500 | 6500 | 6500 | 6500 | s_65 | s_650 | s_6500 | 2175-12-23 11:33:20 | 65 | 650 | 6500 | 6500 | 6500 | 6500 | 6500 + s_66 | s_660 | s_6600 | 2179-02-22 09:20:00 | 66 | 660 | 6600 | 6600 | 6600 | 6600 | 6600 | s_66 | s_660 | s_6600 | 2179-02-22 09:20:00 | 66 | 660 | 6600 | 6600 | 6600 | 6600 | 6600 + s_67 | s_670 | s_6700 | 2182-04-25 07:06:40 | 67 | 670 | 6700 | 6700 | 6700 | 6700 | 6700 | s_67 | s_670 | s_6700 | 2182-04-25 07:06:40 | 67 | 670 | 6700 | 6700 | 6700 | 6700 | 6700 + s_68 | s_680 | s_6800 | 2185-06-25 04:53:20 | 68 | 680 | 6800 | 6800 | 6800 | 6800 | 6800 | s_68 | s_680 | s_6800 | 2185-06-25 04:53:20 | 68 | 680 | 6800 | 6800 | 6800 | 6800 | 6800 + s_69 | s_690 | s_6900 | 2188-08-26 02:40:00 | 69 | 690 | 6900 | 6900 | 6900 | 6900 | 6900 | s_69 | s_690 | s_6900 | 2188-08-26 02:40:00 | 69 | 690 | 6900 | 6900 | 6900 | 6900 | 6900 + s_70 | s_700 | s_7000 | 2191-10-27 12:26:40 | 70 | 700 | 7000 | 7000 | 7000 | 7000 | 7000 | s_70 | s_700 | s_7000 | 2191-10-27 12:26:40 | 70 | 700 | 7000 | 7000 | 7000 | 7000 | 7000 + s_71 | s_710 | s_7100 | 2194-12-27 10:13:20 | 71 | 710 | 7100 | 7100 | 7100 | 7100 | 7100 | s_71 | s_710 | s_7100 | 2194-12-27 10:13:20 | 71 | 710 | 7100 | 7100 | 7100 | 7100 | 7100 + s_72 | s_720 | s_7200 | 2198-02-27 08:00:00 | 72 | 720 | 7200 | 7200 | 7200 | 7200 | 7200 | s_72 | s_720 | s_7200 | 2198-02-27 08:00:00 | 72 | 720 | 7200 | 7200 | 7200 | 7200 | 7200 + s_73 | s_730 | s_7300 | 2201-04-30 05:46:40 | 73 | 730 | 7300 | 7300 | 7300 | 7300 | 7300 | s_73 | s_730 | s_7300 | 2201-04-30 05:46:40 | 73 | 730 | 7300 | 7300 | 7300 | 7300 | 7300 + s_74 | s_740 | s_7400 | 2204-07-01 03:33:20 | 74 | 740 | 7400 | 7400 | 7400 | 7400 | 7400 | s_74 | s_740 | s_7400 | 2204-07-01 03:33:20 | 74 | 740 | 7400 | 7400 | 7400 | 7400 | 7400 + s_75 | s_750 | s_7500 | 2207-09-01 01:20:00 | 75 | 750 | 7500 | 7500 | 7500 | 7500 | 7500 | s_75 | s_750 | s_7500 | 2207-09-01 01:20:00 | 75 | 750 | 7500 | 7500 | 7500 | 7500 | 7500 + s_76 | s_760 | s_7600 | 2210-11-01 11:06:40 | 76 | 760 | 7600 | 7600 | 7600 | 7600 | 7600 | s_76 | s_760 | s_7600 | 2210-11-01 11:06:40 | 76 | 760 | 7600 | 7600 | 7600 | 7600 | 7600 + s_77 | s_770 | s_7700 | 2214-01-02 08:53:20 | 77 | 770 | 7700 | 7700 | 7700 | 7700 | 7700 | s_77 | s_770 | s_7700 | 2214-01-02 08:53:20 | 77 | 770 | 7700 | 7700 | 7700 | 7700 | 7700 + s_78 | s_780 | s_7800 | 2217-03-04 06:40:00 | 78 | 780 | 7800 | 7800 | 7800 | 7800 | 7800 | s_78 | s_780 | s_7800 | 2217-03-04 06:40:00 | 78 | 780 | 7800 | 7800 | 7800 | 7800 | 7800 + s_79 | s_790 | s_7900 | 2220-05-05 04:26:40 | 79 | 790 | 7900 | 7900 | 7900 | 7900 | 7900 | s_79 | s_790 | s_7900 | 2220-05-05 04:26:40 | 79 | 790 | 7900 | 7900 | 7900 | 7900 | 7900 + s_80 | s_800 | s_8000 | 2223-07-06 02:13:20 | 80 | 800 | 8000 | 8000 | 8000 | 8000 | 8000 | s_80 | s_800 | s_8000 | 2223-07-06 02:13:20 | 80 | 800 | 8000 | 8000 | 8000 | 8000 | 8000 + s_81 | s_810 | s_8100 | 2226-09-06 12:00:00 | 81 | 810 | 8100 | 8100 | 8100 | 8100 | 8100 | s_81 | s_810 | s_8100 | 2226-09-06 12:00:00 | 81 | 810 | 8100 | 8100 | 8100 | 8100 | 8100 + s_82 | s_820 | s_8200 | 2229-11-06 09:46:40 | 82 | 820 | 8200 | 8200 | 8200 | 8200 | 8200 | s_82 | s_820 | s_8200 | 2229-11-06 09:46:40 | 82 | 820 | 8200 | 8200 | 8200 | 8200 | 8200 + s_83 | s_830 | s_8300 | 2233-01-06 07:33:20 | 83 | 830 | 8300 | 8300 | 8300 | 8300 | 8300 | s_83 | s_830 | s_8300 | 2233-01-06 07:33:20 | 83 | 830 | 8300 | 8300 | 8300 | 8300 | 8300 + s_84 | s_840 | s_8400 | 2236-03-09 05:20:00 | 84 | 840 | 8400 | 8400 | 8400 | 8400 | 8400 | s_84 | s_840 | s_8400 | 2236-03-09 05:20:00 | 84 | 840 | 8400 | 8400 | 8400 | 8400 | 8400 + s_85 | s_850 | s_8500 | 2239-05-10 03:06:40 | 85 | 850 | 8500 | 8500 | 8500 | 8500 | 8500 | s_85 | s_850 | s_8500 | 2239-05-10 03:06:40 | 85 | 850 | 8500 | 8500 | 8500 | 8500 | 8500 + s_86 | s_860 | s_8600 | 2242-07-11 12:53:20 | 86 | 860 | 8600 | 8600 | 8600 | 8600 | 8600 | s_86 | s_860 | s_8600 | 2242-07-11 12:53:20 | 86 | 860 | 8600 | 8600 | 8600 | 8600 | 8600 + s_87 | s_870 | s_8700 | 2245-09-10 10:40:00 | 87 | 870 | 8700 | 8700 | 8700 | 8700 | 8700 | s_87 | s_870 | s_8700 | 2245-09-10 10:40:00 | 87 | 870 | 8700 | 8700 | 8700 | 8700 | 8700 + s_88 | s_880 | s_8800 | 2248-11-10 08:26:40 | 88 | 880 | 8800 | 8800 | 8800 | 8800 | 8800 | s_88 | s_880 | s_8800 | 2248-11-10 08:26:40 | 88 | 880 | 8800 | 8800 | 8800 | 8800 | 8800 + s_89 | s_890 | s_8900 | 2252-01-12 06:13:20 | 89 | 890 | 8900 | 8900 | 8900 | 8900 | 8900 | s_89 | s_890 | s_8900 | 2252-01-12 06:13:20 | 89 | 890 | 8900 | 8900 | 8900 | 8900 | 8900 + s_90 | s_900 | s_9000 | 2255-03-14 04:00:00 | 90 | 900 | 9000 | 9000 | 9000 | 9000 | 9000 | s_90 | s_900 | s_9000 | 2255-03-14 04:00:00 | 90 | 900 | 9000 | 9000 | 9000 | 9000 | 9000 + s_91 | s_910 | s_9100 | 2258-05-15 01:46:40 | 91 | 910 | 9100 | 9100 | 9100 | 9100 | 9100 | s_91 | s_910 | s_9100 | 2258-05-15 01:46:40 | 91 | 910 | 9100 | 9100 | 9100 | 9100 | 9100 + s_92 | s_920 | s_9200 | 2261-07-15 11:33:20 | 92 | 920 | 9200 | 9200 | 9200 | 9200 | 9200 | s_92 | s_920 | s_9200 | 2261-07-15 11:33:20 | 92 | 920 | 9200 | 9200 | 9200 | 9200 | 9200 + s_93 | s_930 | s_9300 | 2264-09-14 09:20:00 | 93 | 930 | 9300 | 9300 | 9300 | 9300 | 9300 | s_93 | s_930 | s_9300 | 2264-09-14 09:20:00 | 93 | 930 | 9300 | 9300 | 9300 | 9300 | 9300 + s_94 | s_940 | s_9400 | 2267-11-16 07:06:40 | 94 | 940 | 9400 | 9400 | 9400 | 9400 | 9400 | s_94 | s_940 | s_9400 | 2267-11-16 07:06:40 | 94 | 940 | 9400 | 9400 | 9400 | 9400 | 9400 + s_95 | s_950 | s_9500 | 2271-01-16 04:53:20 | 95 | 950 | 9500 | 9500 | 9500 | 9500 | 9500 | s_95 | s_950 | s_9500 | 2271-01-16 04:53:20 | 95 | 950 | 9500 | 9500 | 9500 | 9500 | 9500 + s_96 | s_960 | s_9600 | 2274-03-19 02:40:00 | 96 | 960 | 9600 | 9600 | 9600 | 9600 | 9600 | s_96 | s_960 | s_9600 | 2274-03-19 02:40:00 | 96 | 960 | 9600 | 9600 | 9600 | 9600 | 9600 + s_97 | s_970 | s_9700 | 2277-05-19 12:26:40 | 97 | 970 | 9700 | 9700 | 9700 | 9700 | 9700 | s_97 | s_970 | s_9700 | 2277-05-19 12:26:40 | 97 | 970 | 9700 | 9700 | 9700 | 9700 | 9700 + s_98 | s_980 | s_9800 | 2280-07-19 10:13:20 | 98 | 980 | 9800 | 9800 | 9800 | 9800 | 9800 | s_98 | s_980 | s_9800 | 2280-07-19 10:13:20 | 98 | 980 | 9800 | 9800 | 9800 | 9800 | 9800 + s_99 | s_990 | s_9900 | 2283-09-20 08:00:00 | 99 | 990 | 9900 | 9900 | 9900 | 9900 | 9900 | s_99 | s_990 | s_9900 | 2283-09-20 08:00:00 | 99 | 990 | 9900 | 9900 | 9900 | 9900 | 9900 + s_100 | s_1000 | s_10000 | 2286-11-20 05:46:40 | 100 | 1000 | 10000 | 10000 | 10000 | 10000 | 10000 | s_100 | s_1000 | s_10000 | 2286-11-20 05:46:40 | 100 | 1000 | 10000 | 10000 | 10000 | 10000 | 10000 +(90 rows) \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/four_byte/runTest.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/four_byte/runTest.py new file mode 100755 index 0000000000..500cd9322d --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/four_byte/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfMultibyteFourByteDelimiterData(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/four_byte/sql/query01.sql b/automation/tincrepo/main/pxf/features/multibyte_delimiter/four_byte/sql/query01.sql new file mode 100755 index 0000000000..e4296c2c57 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/four_byte/sql/query01.sql @@ -0,0 +1,5 @@ +-- @description query01 for PXF Multibyte delimiter, 4-byte delim cases + +SELECT * from pxf_multibyte_fourbyte_data ORDER BY n1; + +SELECT * from pxf_multibyte_fourbyte_data_with_skip ORDER BY n1; \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/multi_char/__init__.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/multi_char/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/multi_char/expected/query01.ans b/automation/tincrepo/main/pxf/features/multibyte_delimiter/multi_char/expected/query01.ans new file mode 100755 index 0000000000..ad629380d2 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/multi_char/expected/query01.ans @@ -0,0 +1,201 @@ +-- @description query01 for PXF Multibyte delimiter, multi-character delim cases + +SELECT * from pxf_multibyte_multichar_data ORDER BY n1; + s1 | s2 | s3 | d1 | n1 | n2 | n3 | n4 | n5 | n6 | n7 | s11 | s12 | s13 | d11 | n11 | n12 | n13 | n14 | n15 | n16 | n17 +-------+--------+---------+---------------------+-----+------+-------+-------+-------+-------+-------+-------+--------+---------+---------------------+-----+------+-------+-------+-------+-------+------- + s_1 | s_10 | s_100 | 1973-03-03 09:46:40 | 1 | 10 | 100 | 100 | 100 | 100 | 100 | s_1 | s_10 | s_100 | 1973-03-03 09:46:40 | 1 | 10 | 100 | 100 | 100 | 100 | 100 + s_2 | s_20 | s_200 | 1976-05-03 07:33:20 | 2 | 20 | 200 | 200 | 200 | 200 | 200 | s_2 | s_20 | s_200 | 1976-05-03 07:33:20 | 2 | 20 | 200 | 200 | 200 | 200 | 200 + s_3 | s_30 | s_300 | 1979-07-05 05:20:00 | 3 | 30 | 300 | 300 | 300 | 300 | 300 | s_3 | s_30 | s_300 | 1979-07-05 05:20:00 | 3 | 30 | 300 | 300 | 300 | 300 | 300 + s_4 | s_40 | s_400 | 1982-09-04 03:06:40 | 4 | 40 | 400 | 400 | 400 | 400 | 400 | s_4 | s_40 | s_400 | 1982-09-04 03:06:40 | 4 | 40 | 400 | 400 | 400 | 400 | 400 + s_5 | s_50 | s_500 | 1985-11-05 12:53:20 | 5 | 50 | 500 | 500 | 500 | 500 | 500 | s_5 | s_50 | s_500 | 1985-11-05 12:53:20 | 5 | 50 | 500 | 500 | 500 | 500 | 500 + s_6 | s_60 | s_600 | 1989-01-05 10:40:00 | 6 | 60 | 600 | 600 | 600 | 600 | 600 | s_6 | s_60 | s_600 | 1989-01-05 10:40:00 | 6 | 60 | 600 | 600 | 600 | 600 | 600 + s_7 | s_70 | s_700 | 1992-03-07 08:26:40 | 7 | 70 | 700 | 700 | 700 | 700 | 700 | s_7 | s_70 | s_700 | 1992-03-07 08:26:40 | 7 | 70 | 700 | 700 | 700 | 700 | 700 + s_8 | s_80 | s_800 | 1995-05-09 06:13:20 | 8 | 80 | 800 | 800 | 800 | 800 | 800 | s_8 | s_80 | s_800 | 1995-05-09 06:13:20 | 8 | 80 | 800 | 800 | 800 | 800 | 800 + s_9 | s_90 | s_900 | 1998-07-09 04:00:00 | 9 | 90 | 900 | 900 | 900 | 900 | 900 | s_9 | s_90 | s_900 | 1998-07-09 04:00:00 | 9 | 90 | 900 | 900 | 900 | 900 | 900 + s_10 | s_100 | s_1000 | 2001-09-09 01:46:40 | 10 | 100 | 1000 | 1000 | 1000 | 1000 | 1000 | s_10 | s_100 | s_1000 | 2001-09-09 01:46:40 | 10 | 100 | 1000 | 1000 | 1000 | 1000 | 1000 + s_11 | s_110 | s_1100 | 2004-11-09 11:33:20 | 11 | 110 | 1100 | 1100 | 1100 | 1100 | 1100 | s_11 | s_110 | s_1100 | 2004-11-09 11:33:20 | 11 | 110 | 1100 | 1100 | 1100 | 1100 | 1100 + s_12 | s_120 | s_1200 | 2008-01-10 09:20:00 | 12 | 120 | 1200 | 1200 | 1200 | 1200 | 1200 | s_12 | s_120 | s_1200 | 2008-01-10 09:20:00 | 12 | 120 | 1200 | 1200 | 1200 | 1200 | 1200 + s_13 | s_130 | s_1300 | 2011-03-13 07:06:40 | 13 | 130 | 1300 | 1300 | 1300 | 1300 | 1300 | s_13 | s_130 | s_1300 | 2011-03-13 07:06:40 | 13 | 130 | 1300 | 1300 | 1300 | 1300 | 1300 + s_14 | s_140 | s_1400 | 2014-05-13 04:53:20 | 14 | 140 | 1400 | 1400 | 1400 | 1400 | 1400 | s_14 | s_140 | s_1400 | 2014-05-13 04:53:20 | 14 | 140 | 1400 | 1400 | 1400 | 1400 | 1400 + s_15 | s_150 | s_1500 | 2017-07-14 02:40:00 | 15 | 150 | 1500 | 1500 | 1500 | 1500 | 1500 | s_15 | s_150 | s_1500 | 2017-07-14 02:40:00 | 15 | 150 | 1500 | 1500 | 1500 | 1500 | 1500 + s_16 | s_160 | s_1600 | 2020-09-13 12:26:40 | 16 | 160 | 1600 | 1600 | 1600 | 1600 | 1600 | s_16 | s_160 | s_1600 | 2020-09-13 12:26:40 | 16 | 160 | 1600 | 1600 | 1600 | 1600 | 1600 + s_17 | s_170 | s_1700 | 2023-11-14 10:13:20 | 17 | 170 | 1700 | 1700 | 1700 | 1700 | 1700 | s_17 | s_170 | s_1700 | 2023-11-14 10:13:20 | 17 | 170 | 1700 | 1700 | 1700 | 1700 | 1700 + s_18 | s_180 | s_1800 | 2027-01-15 08:00:00 | 18 | 180 | 1800 | 1800 | 1800 | 1800 | 1800 | s_18 | s_180 | s_1800 | 2027-01-15 08:00:00 | 18 | 180 | 1800 | 1800 | 1800 | 1800 | 1800 + s_19 | s_190 | s_1900 | 2030-03-17 05:46:40 | 19 | 190 | 1900 | 1900 | 1900 | 1900 | 1900 | s_19 | s_190 | s_1900 | 2030-03-17 05:46:40 | 19 | 190 | 1900 | 1900 | 1900 | 1900 | 1900 + s_20 | s_200 | s_2000 | 2033-05-18 03:33:20 | 20 | 200 | 2000 | 2000 | 2000 | 2000 | 2000 | s_20 | s_200 | s_2000 | 2033-05-18 03:33:20 | 20 | 200 | 2000 | 2000 | 2000 | 2000 | 2000 + s_21 | s_210 | s_2100 | 2036-07-18 01:20:00 | 21 | 210 | 2100 | 2100 | 2100 | 2100 | 2100 | s_21 | s_210 | s_2100 | 2036-07-18 01:20:00 | 21 | 210 | 2100 | 2100 | 2100 | 2100 | 2100 + s_22 | s_220 | s_2200 | 2039-09-18 11:06:40 | 22 | 220 | 2200 | 2200 | 2200 | 2200 | 2200 | s_22 | s_220 | s_2200 | 2039-09-18 11:06:40 | 22 | 220 | 2200 | 2200 | 2200 | 2200 | 2200 + s_23 | s_230 | s_2300 | 2042-11-19 08:53:20 | 23 | 230 | 2300 | 2300 | 2300 | 2300 | 2300 | s_23 | s_230 | s_2300 | 2042-11-19 08:53:20 | 23 | 230 | 2300 | 2300 | 2300 | 2300 | 2300 + s_24 | s_240 | s_2400 | 2046-01-19 06:40:00 | 24 | 240 | 2400 | 2400 | 2400 | 2400 | 2400 | s_24 | s_240 | s_2400 | 2046-01-19 06:40:00 | 24 | 240 | 2400 | 2400 | 2400 | 2400 | 2400 + s_25 | s_250 | s_2500 | 2049-03-22 04:26:40 | 25 | 250 | 2500 | 2500 | 2500 | 2500 | 2500 | s_25 | s_250 | s_2500 | 2049-03-22 04:26:40 | 25 | 250 | 2500 | 2500 | 2500 | 2500 | 2500 + s_26 | s_260 | s_2600 | 2052-05-22 02:13:20 | 26 | 260 | 2600 | 2600 | 2600 | 2600 | 2600 | s_26 | s_260 | s_2600 | 2052-05-22 02:13:20 | 26 | 260 | 2600 | 2600 | 2600 | 2600 | 2600 + s_27 | s_270 | s_2700 | 2055-07-24 12:00:00 | 27 | 270 | 2700 | 2700 | 2700 | 2700 | 2700 | s_27 | s_270 | s_2700 | 2055-07-24 12:00:00 | 27 | 270 | 2700 | 2700 | 2700 | 2700 | 2700 + s_28 | s_280 | s_2800 | 2058-09-23 09:46:40 | 28 | 280 | 2800 | 2800 | 2800 | 2800 | 2800 | s_28 | s_280 | s_2800 | 2058-09-23 09:46:40 | 28 | 280 | 2800 | 2800 | 2800 | 2800 | 2800 + s_29 | s_290 | s_2900 | 2061-11-23 07:33:20 | 29 | 290 | 2900 | 2900 | 2900 | 2900 | 2900 | s_29 | s_290 | s_2900 | 2061-11-23 07:33:20 | 29 | 290 | 2900 | 2900 | 2900 | 2900 | 2900 + s_30 | s_300 | s_3000 | 2065-01-24 05:20:00 | 30 | 300 | 3000 | 3000 | 3000 | 3000 | 3000 | s_30 | s_300 | s_3000 | 2065-01-24 05:20:00 | 30 | 300 | 3000 | 3000 | 3000 | 3000 | 3000 + s_31 | s_310 | s_3100 | 2068-03-26 03:06:40 | 31 | 310 | 3100 | 3100 | 3100 | 3100 | 3100 | s_31 | s_310 | s_3100 | 2068-03-26 03:06:40 | 31 | 310 | 3100 | 3100 | 3100 | 3100 | 3100 + s_32 | s_320 | s_3200 | 2071-05-28 12:53:20 | 32 | 320 | 3200 | 3200 | 3200 | 3200 | 3200 | s_32 | s_320 | s_3200 | 2071-05-28 12:53:20 | 32 | 320 | 3200 | 3200 | 3200 | 3200 | 3200 + s_33 | s_330 | s_3300 | 2074-07-28 10:40:00 | 33 | 330 | 3300 | 3300 | 3300 | 3300 | 3300 | s_33 | s_330 | s_3300 | 2074-07-28 10:40:00 | 33 | 330 | 3300 | 3300 | 3300 | 3300 | 3300 + s_34 | s_340 | s_3400 | 2077-09-27 08:26:40 | 34 | 340 | 3400 | 3400 | 3400 | 3400 | 3400 | s_34 | s_340 | s_3400 | 2077-09-27 08:26:40 | 34 | 340 | 3400 | 3400 | 3400 | 3400 | 3400 + s_35 | s_350 | s_3500 | 2080-11-28 06:13:20 | 35 | 350 | 3500 | 3500 | 3500 | 3500 | 3500 | s_35 | s_350 | s_3500 | 2080-11-28 06:13:20 | 35 | 350 | 3500 | 3500 | 3500 | 3500 | 3500 + s_36 | s_360 | s_3600 | 2084-01-29 04:00:00 | 36 | 360 | 3600 | 3600 | 3600 | 3600 | 3600 | s_36 | s_360 | s_3600 | 2084-01-29 04:00:00 | 36 | 360 | 3600 | 3600 | 3600 | 3600 | 3600 + s_37 | s_370 | s_3700 | 2087-04-01 01:46:40 | 37 | 370 | 3700 | 3700 | 3700 | 3700 | 3700 | s_37 | s_370 | s_3700 | 2087-04-01 01:46:40 | 37 | 370 | 3700 | 3700 | 3700 | 3700 | 3700 + s_38 | s_380 | s_3800 | 2090-06-01 11:33:20 | 38 | 380 | 3800 | 3800 | 3800 | 3800 | 3800 | s_38 | s_380 | s_3800 | 2090-06-01 11:33:20 | 38 | 380 | 3800 | 3800 | 3800 | 3800 | 3800 + s_39 | s_390 | s_3900 | 2093-08-01 09:20:00 | 39 | 390 | 3900 | 3900 | 3900 | 3900 | 3900 | s_39 | s_390 | s_3900 | 2093-08-01 09:20:00 | 39 | 390 | 3900 | 3900 | 3900 | 3900 | 3900 + s_40 | s_400 | s_4000 | 2096-10-02 07:06:40 | 40 | 400 | 4000 | 4000 | 4000 | 4000 | 4000 | s_40 | s_400 | s_4000 | 2096-10-02 07:06:40 | 40 | 400 | 4000 | 4000 | 4000 | 4000 | 4000 + s_41 | s_410 | s_4100 | 2099-12-03 04:53:20 | 41 | 410 | 4100 | 4100 | 4100 | 4100 | 4100 | s_41 | s_410 | s_4100 | 2099-12-03 04:53:20 | 41 | 410 | 4100 | 4100 | 4100 | 4100 | 4100 + s_42 | s_420 | s_4200 | 2103-02-04 02:40:00 | 42 | 420 | 4200 | 4200 | 4200 | 4200 | 4200 | s_42 | s_420 | s_4200 | 2103-02-04 02:40:00 | 42 | 420 | 4200 | 4200 | 4200 | 4200 | 4200 + s_43 | s_430 | s_4300 | 2106-04-06 12:26:40 | 43 | 430 | 4300 | 4300 | 4300 | 4300 | 4300 | s_43 | s_430 | s_4300 | 2106-04-06 12:26:40 | 43 | 430 | 4300 | 4300 | 4300 | 4300 | 4300 + s_44 | s_440 | s_4400 | 2109-06-06 10:13:20 | 44 | 440 | 4400 | 4400 | 4400 | 4400 | 4400 | s_44 | s_440 | s_4400 | 2109-06-06 10:13:20 | 44 | 440 | 4400 | 4400 | 4400 | 4400 | 4400 + s_45 | s_450 | s_4500 | 2112-08-07 08:00:00 | 45 | 450 | 4500 | 4500 | 4500 | 4500 | 4500 | s_45 | s_450 | s_4500 | 2112-08-07 08:00:00 | 45 | 450 | 4500 | 4500 | 4500 | 4500 | 4500 + s_46 | s_460 | s_4600 | 2115-10-08 05:46:40 | 46 | 460 | 4600 | 4600 | 4600 | 4600 | 4600 | s_46 | s_460 | s_4600 | 2115-10-08 05:46:40 | 46 | 460 | 4600 | 4600 | 4600 | 4600 | 4600 + s_47 | s_470 | s_4700 | 2118-12-09 03:33:20 | 47 | 470 | 4700 | 4700 | 4700 | 4700 | 4700 | s_47 | s_470 | s_4700 | 2118-12-09 03:33:20 | 47 | 470 | 4700 | 4700 | 4700 | 4700 | 4700 + s_48 | s_480 | s_4800 | 2122-02-08 01:20:00 | 48 | 480 | 4800 | 4800 | 4800 | 4800 | 4800 | s_48 | s_480 | s_4800 | 2122-02-08 01:20:00 | 48 | 480 | 4800 | 4800 | 4800 | 4800 | 4800 + s_49 | s_490 | s_4900 | 2125-04-10 11:06:40 | 49 | 490 | 4900 | 4900 | 4900 | 4900 | 4900 | s_49 | s_490 | s_4900 | 2125-04-10 11:06:40 | 49 | 490 | 4900 | 4900 | 4900 | 4900 | 4900 + s_50 | s_500 | s_5000 | 2128-06-11 08:53:20 | 50 | 500 | 5000 | 5000 | 5000 | 5000 | 5000 | s_50 | s_500 | s_5000 | 2128-06-11 08:53:20 | 50 | 500 | 5000 | 5000 | 5000 | 5000 | 5000 + s_51 | s_510 | s_5100 | 2131-08-12 06:40:00 | 51 | 510 | 5100 | 5100 | 5100 | 5100 | 5100 | s_51 | s_510 | s_5100 | 2131-08-12 06:40:00 | 51 | 510 | 5100 | 5100 | 5100 | 5100 | 5100 + s_52 | s_520 | s_5200 | 2134-10-13 04:26:40 | 52 | 520 | 5200 | 5200 | 5200 | 5200 | 5200 | s_52 | s_520 | s_5200 | 2134-10-13 04:26:40 | 52 | 520 | 5200 | 5200 | 5200 | 5200 | 5200 + s_53 | s_530 | s_5300 | 2137-12-13 02:13:20 | 53 | 530 | 5300 | 5300 | 5300 | 5300 | 5300 | s_53 | s_530 | s_5300 | 2137-12-13 02:13:20 | 53 | 530 | 5300 | 5300 | 5300 | 5300 | 5300 + s_54 | s_540 | s_5400 | 2141-02-13 12:00:00 | 54 | 540 | 5400 | 5400 | 5400 | 5400 | 5400 | s_54 | s_540 | s_5400 | 2141-02-13 12:00:00 | 54 | 540 | 5400 | 5400 | 5400 | 5400 | 5400 + s_55 | s_550 | s_5500 | 2144-04-15 09:46:40 | 55 | 550 | 5500 | 5500 | 5500 | 5500 | 5500 | s_55 | s_550 | s_5500 | 2144-04-15 09:46:40 | 55 | 550 | 5500 | 5500 | 5500 | 5500 | 5500 + s_56 | s_560 | s_5600 | 2147-06-16 07:33:20 | 56 | 560 | 5600 | 5600 | 5600 | 5600 | 5600 | s_56 | s_560 | s_5600 | 2147-06-16 07:33:20 | 56 | 560 | 5600 | 5600 | 5600 | 5600 | 5600 + s_57 | s_570 | s_5700 | 2150-08-17 05:20:00 | 57 | 570 | 5700 | 5700 | 5700 | 5700 | 5700 | s_57 | s_570 | s_5700 | 2150-08-17 05:20:00 | 57 | 570 | 5700 | 5700 | 5700 | 5700 | 5700 + s_58 | s_580 | s_5800 | 2153-10-17 03:06:40 | 58 | 580 | 5800 | 5800 | 5800 | 5800 | 5800 | s_58 | s_580 | s_5800 | 2153-10-17 03:06:40 | 58 | 580 | 5800 | 5800 | 5800 | 5800 | 5800 + s_59 | s_590 | s_5900 | 2156-12-18 12:53:20 | 59 | 590 | 5900 | 5900 | 5900 | 5900 | 5900 | s_59 | s_590 | s_5900 | 2156-12-18 12:53:20 | 59 | 590 | 5900 | 5900 | 5900 | 5900 | 5900 + s_60 | s_600 | s_6000 | 2160-02-18 10:40:00 | 60 | 600 | 6000 | 6000 | 6000 | 6000 | 6000 | s_60 | s_600 | s_6000 | 2160-02-18 10:40:00 | 60 | 600 | 6000 | 6000 | 6000 | 6000 | 6000 + s_61 | s_610 | s_6100 | 2163-04-20 08:26:40 | 61 | 610 | 6100 | 6100 | 6100 | 6100 | 6100 | s_61 | s_610 | s_6100 | 2163-04-20 08:26:40 | 61 | 610 | 6100 | 6100 | 6100 | 6100 | 6100 + s_62 | s_620 | s_6200 | 2166-06-21 06:13:20 | 62 | 620 | 6200 | 6200 | 6200 | 6200 | 6200 | s_62 | s_620 | s_6200 | 2166-06-21 06:13:20 | 62 | 620 | 6200 | 6200 | 6200 | 6200 | 6200 + s_63 | s_630 | s_6300 | 2169-08-21 04:00:00 | 63 | 630 | 6300 | 6300 | 6300 | 6300 | 6300 | s_63 | s_630 | s_6300 | 2169-08-21 04:00:00 | 63 | 630 | 6300 | 6300 | 6300 | 6300 | 6300 + s_64 | s_640 | s_6400 | 2172-10-22 01:46:40 | 64 | 640 | 6400 | 6400 | 6400 | 6400 | 6400 | s_64 | s_640 | s_6400 | 2172-10-22 01:46:40 | 64 | 640 | 6400 | 6400 | 6400 | 6400 | 6400 + s_65 | s_650 | s_6500 | 2175-12-23 11:33:20 | 65 | 650 | 6500 | 6500 | 6500 | 6500 | 6500 | s_65 | s_650 | s_6500 | 2175-12-23 11:33:20 | 65 | 650 | 6500 | 6500 | 6500 | 6500 | 6500 + s_66 | s_660 | s_6600 | 2179-02-22 09:20:00 | 66 | 660 | 6600 | 6600 | 6600 | 6600 | 6600 | s_66 | s_660 | s_6600 | 2179-02-22 09:20:00 | 66 | 660 | 6600 | 6600 | 6600 | 6600 | 6600 + s_67 | s_670 | s_6700 | 2182-04-25 07:06:40 | 67 | 670 | 6700 | 6700 | 6700 | 6700 | 6700 | s_67 | s_670 | s_6700 | 2182-04-25 07:06:40 | 67 | 670 | 6700 | 6700 | 6700 | 6700 | 6700 + s_68 | s_680 | s_6800 | 2185-06-25 04:53:20 | 68 | 680 | 6800 | 6800 | 6800 | 6800 | 6800 | s_68 | s_680 | s_6800 | 2185-06-25 04:53:20 | 68 | 680 | 6800 | 6800 | 6800 | 6800 | 6800 + s_69 | s_690 | s_6900 | 2188-08-26 02:40:00 | 69 | 690 | 6900 | 6900 | 6900 | 6900 | 6900 | s_69 | s_690 | s_6900 | 2188-08-26 02:40:00 | 69 | 690 | 6900 | 6900 | 6900 | 6900 | 6900 + s_70 | s_700 | s_7000 | 2191-10-27 12:26:40 | 70 | 700 | 7000 | 7000 | 7000 | 7000 | 7000 | s_70 | s_700 | s_7000 | 2191-10-27 12:26:40 | 70 | 700 | 7000 | 7000 | 7000 | 7000 | 7000 + s_71 | s_710 | s_7100 | 2194-12-27 10:13:20 | 71 | 710 | 7100 | 7100 | 7100 | 7100 | 7100 | s_71 | s_710 | s_7100 | 2194-12-27 10:13:20 | 71 | 710 | 7100 | 7100 | 7100 | 7100 | 7100 + s_72 | s_720 | s_7200 | 2198-02-27 08:00:00 | 72 | 720 | 7200 | 7200 | 7200 | 7200 | 7200 | s_72 | s_720 | s_7200 | 2198-02-27 08:00:00 | 72 | 720 | 7200 | 7200 | 7200 | 7200 | 7200 + s_73 | s_730 | s_7300 | 2201-04-30 05:46:40 | 73 | 730 | 7300 | 7300 | 7300 | 7300 | 7300 | s_73 | s_730 | s_7300 | 2201-04-30 05:46:40 | 73 | 730 | 7300 | 7300 | 7300 | 7300 | 7300 + s_74 | s_740 | s_7400 | 2204-07-01 03:33:20 | 74 | 740 | 7400 | 7400 | 7400 | 7400 | 7400 | s_74 | s_740 | s_7400 | 2204-07-01 03:33:20 | 74 | 740 | 7400 | 7400 | 7400 | 7400 | 7400 + s_75 | s_750 | s_7500 | 2207-09-01 01:20:00 | 75 | 750 | 7500 | 7500 | 7500 | 7500 | 7500 | s_75 | s_750 | s_7500 | 2207-09-01 01:20:00 | 75 | 750 | 7500 | 7500 | 7500 | 7500 | 7500 + s_76 | s_760 | s_7600 | 2210-11-01 11:06:40 | 76 | 760 | 7600 | 7600 | 7600 | 7600 | 7600 | s_76 | s_760 | s_7600 | 2210-11-01 11:06:40 | 76 | 760 | 7600 | 7600 | 7600 | 7600 | 7600 + s_77 | s_770 | s_7700 | 2214-01-02 08:53:20 | 77 | 770 | 7700 | 7700 | 7700 | 7700 | 7700 | s_77 | s_770 | s_7700 | 2214-01-02 08:53:20 | 77 | 770 | 7700 | 7700 | 7700 | 7700 | 7700 + s_78 | s_780 | s_7800 | 2217-03-04 06:40:00 | 78 | 780 | 7800 | 7800 | 7800 | 7800 | 7800 | s_78 | s_780 | s_7800 | 2217-03-04 06:40:00 | 78 | 780 | 7800 | 7800 | 7800 | 7800 | 7800 + s_79 | s_790 | s_7900 | 2220-05-05 04:26:40 | 79 | 790 | 7900 | 7900 | 7900 | 7900 | 7900 | s_79 | s_790 | s_7900 | 2220-05-05 04:26:40 | 79 | 790 | 7900 | 7900 | 7900 | 7900 | 7900 + s_80 | s_800 | s_8000 | 2223-07-06 02:13:20 | 80 | 800 | 8000 | 8000 | 8000 | 8000 | 8000 | s_80 | s_800 | s_8000 | 2223-07-06 02:13:20 | 80 | 800 | 8000 | 8000 | 8000 | 8000 | 8000 + s_81 | s_810 | s_8100 | 2226-09-06 12:00:00 | 81 | 810 | 8100 | 8100 | 8100 | 8100 | 8100 | s_81 | s_810 | s_8100 | 2226-09-06 12:00:00 | 81 | 810 | 8100 | 8100 | 8100 | 8100 | 8100 + s_82 | s_820 | s_8200 | 2229-11-06 09:46:40 | 82 | 820 | 8200 | 8200 | 8200 | 8200 | 8200 | s_82 | s_820 | s_8200 | 2229-11-06 09:46:40 | 82 | 820 | 8200 | 8200 | 8200 | 8200 | 8200 + s_83 | s_830 | s_8300 | 2233-01-06 07:33:20 | 83 | 830 | 8300 | 8300 | 8300 | 8300 | 8300 | s_83 | s_830 | s_8300 | 2233-01-06 07:33:20 | 83 | 830 | 8300 | 8300 | 8300 | 8300 | 8300 + s_84 | s_840 | s_8400 | 2236-03-09 05:20:00 | 84 | 840 | 8400 | 8400 | 8400 | 8400 | 8400 | s_84 | s_840 | s_8400 | 2236-03-09 05:20:00 | 84 | 840 | 8400 | 8400 | 8400 | 8400 | 8400 + s_85 | s_850 | s_8500 | 2239-05-10 03:06:40 | 85 | 850 | 8500 | 8500 | 8500 | 8500 | 8500 | s_85 | s_850 | s_8500 | 2239-05-10 03:06:40 | 85 | 850 | 8500 | 8500 | 8500 | 8500 | 8500 + s_86 | s_860 | s_8600 | 2242-07-11 12:53:20 | 86 | 860 | 8600 | 8600 | 8600 | 8600 | 8600 | s_86 | s_860 | s_8600 | 2242-07-11 12:53:20 | 86 | 860 | 8600 | 8600 | 8600 | 8600 | 8600 + s_87 | s_870 | s_8700 | 2245-09-10 10:40:00 | 87 | 870 | 8700 | 8700 | 8700 | 8700 | 8700 | s_87 | s_870 | s_8700 | 2245-09-10 10:40:00 | 87 | 870 | 8700 | 8700 | 8700 | 8700 | 8700 + s_88 | s_880 | s_8800 | 2248-11-10 08:26:40 | 88 | 880 | 8800 | 8800 | 8800 | 8800 | 8800 | s_88 | s_880 | s_8800 | 2248-11-10 08:26:40 | 88 | 880 | 8800 | 8800 | 8800 | 8800 | 8800 + s_89 | s_890 | s_8900 | 2252-01-12 06:13:20 | 89 | 890 | 8900 | 8900 | 8900 | 8900 | 8900 | s_89 | s_890 | s_8900 | 2252-01-12 06:13:20 | 89 | 890 | 8900 | 8900 | 8900 | 8900 | 8900 + s_90 | s_900 | s_9000 | 2255-03-14 04:00:00 | 90 | 900 | 9000 | 9000 | 9000 | 9000 | 9000 | s_90 | s_900 | s_9000 | 2255-03-14 04:00:00 | 90 | 900 | 9000 | 9000 | 9000 | 9000 | 9000 + s_91 | s_910 | s_9100 | 2258-05-15 01:46:40 | 91 | 910 | 9100 | 9100 | 9100 | 9100 | 9100 | s_91 | s_910 | s_9100 | 2258-05-15 01:46:40 | 91 | 910 | 9100 | 9100 | 9100 | 9100 | 9100 + s_92 | s_920 | s_9200 | 2261-07-15 11:33:20 | 92 | 920 | 9200 | 9200 | 9200 | 9200 | 9200 | s_92 | s_920 | s_9200 | 2261-07-15 11:33:20 | 92 | 920 | 9200 | 9200 | 9200 | 9200 | 9200 + s_93 | s_930 | s_9300 | 2264-09-14 09:20:00 | 93 | 930 | 9300 | 9300 | 9300 | 9300 | 9300 | s_93 | s_930 | s_9300 | 2264-09-14 09:20:00 | 93 | 930 | 9300 | 9300 | 9300 | 9300 | 9300 + s_94 | s_940 | s_9400 | 2267-11-16 07:06:40 | 94 | 940 | 9400 | 9400 | 9400 | 9400 | 9400 | s_94 | s_940 | s_9400 | 2267-11-16 07:06:40 | 94 | 940 | 9400 | 9400 | 9400 | 9400 | 9400 + s_95 | s_950 | s_9500 | 2271-01-16 04:53:20 | 95 | 950 | 9500 | 9500 | 9500 | 9500 | 9500 | s_95 | s_950 | s_9500 | 2271-01-16 04:53:20 | 95 | 950 | 9500 | 9500 | 9500 | 9500 | 9500 + s_96 | s_960 | s_9600 | 2274-03-19 02:40:00 | 96 | 960 | 9600 | 9600 | 9600 | 9600 | 9600 | s_96 | s_960 | s_9600 | 2274-03-19 02:40:00 | 96 | 960 | 9600 | 9600 | 9600 | 9600 | 9600 + s_97 | s_970 | s_9700 | 2277-05-19 12:26:40 | 97 | 970 | 9700 | 9700 | 9700 | 9700 | 9700 | s_97 | s_970 | s_9700 | 2277-05-19 12:26:40 | 97 | 970 | 9700 | 9700 | 9700 | 9700 | 9700 + s_98 | s_980 | s_9800 | 2280-07-19 10:13:20 | 98 | 980 | 9800 | 9800 | 9800 | 9800 | 9800 | s_98 | s_980 | s_9800 | 2280-07-19 10:13:20 | 98 | 980 | 9800 | 9800 | 9800 | 9800 | 9800 + s_99 | s_990 | s_9900 | 2283-09-20 08:00:00 | 99 | 990 | 9900 | 9900 | 9900 | 9900 | 9900 | s_99 | s_990 | s_9900 | 2283-09-20 08:00:00 | 99 | 990 | 9900 | 9900 | 9900 | 9900 | 9900 + s_100 | s_1000 | s_10000 | 2286-11-20 05:46:40 | 100 | 1000 | 10000 | 10000 | 10000 | 10000 | 10000 | s_100 | s_1000 | s_10000 | 2286-11-20 05:46:40 | 100 | 1000 | 10000 | 10000 | 10000 | 10000 | 10000 +(100 rows) + +SELECT * from pxf_multibyte_multichar_data_with_skip ORDER BY n1; + s1 | s2 | s3 | d1 | n1 | n2 | n3 | n4 | n5 | n6 | n7 | s11 | s12 | s13 | d11 | n11 | n12 | n13 | n14 | n15 | n16 | n17 +-------+--------+---------+---------------------+-----+------+-------+-------+-------+-------+-------+-------+--------+---------+---------------------+-----+------+-------+-------+-------+-------+------- + s_11 | s_110 | s_1100 | 2004-11-09 11:33:20 | 11 | 110 | 1100 | 1100 | 1100 | 1100 | 1100 | s_11 | s_110 | s_1100 | 2004-11-09 11:33:20 | 11 | 110 | 1100 | 1100 | 1100 | 1100 | 1100 + s_12 | s_120 | s_1200 | 2008-01-10 09:20:00 | 12 | 120 | 1200 | 1200 | 1200 | 1200 | 1200 | s_12 | s_120 | s_1200 | 2008-01-10 09:20:00 | 12 | 120 | 1200 | 1200 | 1200 | 1200 | 1200 + s_13 | s_130 | s_1300 | 2011-03-13 07:06:40 | 13 | 130 | 1300 | 1300 | 1300 | 1300 | 1300 | s_13 | s_130 | s_1300 | 2011-03-13 07:06:40 | 13 | 130 | 1300 | 1300 | 1300 | 1300 | 1300 + s_14 | s_140 | s_1400 | 2014-05-13 04:53:20 | 14 | 140 | 1400 | 1400 | 1400 | 1400 | 1400 | s_14 | s_140 | s_1400 | 2014-05-13 04:53:20 | 14 | 140 | 1400 | 1400 | 1400 | 1400 | 1400 + s_15 | s_150 | s_1500 | 2017-07-14 02:40:00 | 15 | 150 | 1500 | 1500 | 1500 | 1500 | 1500 | s_15 | s_150 | s_1500 | 2017-07-14 02:40:00 | 15 | 150 | 1500 | 1500 | 1500 | 1500 | 1500 + s_16 | s_160 | s_1600 | 2020-09-13 12:26:40 | 16 | 160 | 1600 | 1600 | 1600 | 1600 | 1600 | s_16 | s_160 | s_1600 | 2020-09-13 12:26:40 | 16 | 160 | 1600 | 1600 | 1600 | 1600 | 1600 + s_17 | s_170 | s_1700 | 2023-11-14 10:13:20 | 17 | 170 | 1700 | 1700 | 1700 | 1700 | 1700 | s_17 | s_170 | s_1700 | 2023-11-14 10:13:20 | 17 | 170 | 1700 | 1700 | 1700 | 1700 | 1700 + s_18 | s_180 | s_1800 | 2027-01-15 08:00:00 | 18 | 180 | 1800 | 1800 | 1800 | 1800 | 1800 | s_18 | s_180 | s_1800 | 2027-01-15 08:00:00 | 18 | 180 | 1800 | 1800 | 1800 | 1800 | 1800 + s_19 | s_190 | s_1900 | 2030-03-17 05:46:40 | 19 | 190 | 1900 | 1900 | 1900 | 1900 | 1900 | s_19 | s_190 | s_1900 | 2030-03-17 05:46:40 | 19 | 190 | 1900 | 1900 | 1900 | 1900 | 1900 + s_20 | s_200 | s_2000 | 2033-05-18 03:33:20 | 20 | 200 | 2000 | 2000 | 2000 | 2000 | 2000 | s_20 | s_200 | s_2000 | 2033-05-18 03:33:20 | 20 | 200 | 2000 | 2000 | 2000 | 2000 | 2000 + s_21 | s_210 | s_2100 | 2036-07-18 01:20:00 | 21 | 210 | 2100 | 2100 | 2100 | 2100 | 2100 | s_21 | s_210 | s_2100 | 2036-07-18 01:20:00 | 21 | 210 | 2100 | 2100 | 2100 | 2100 | 2100 + s_22 | s_220 | s_2200 | 2039-09-18 11:06:40 | 22 | 220 | 2200 | 2200 | 2200 | 2200 | 2200 | s_22 | s_220 | s_2200 | 2039-09-18 11:06:40 | 22 | 220 | 2200 | 2200 | 2200 | 2200 | 2200 + s_23 | s_230 | s_2300 | 2042-11-19 08:53:20 | 23 | 230 | 2300 | 2300 | 2300 | 2300 | 2300 | s_23 | s_230 | s_2300 | 2042-11-19 08:53:20 | 23 | 230 | 2300 | 2300 | 2300 | 2300 | 2300 + s_24 | s_240 | s_2400 | 2046-01-19 06:40:00 | 24 | 240 | 2400 | 2400 | 2400 | 2400 | 2400 | s_24 | s_240 | s_2400 | 2046-01-19 06:40:00 | 24 | 240 | 2400 | 2400 | 2400 | 2400 | 2400 + s_25 | s_250 | s_2500 | 2049-03-22 04:26:40 | 25 | 250 | 2500 | 2500 | 2500 | 2500 | 2500 | s_25 | s_250 | s_2500 | 2049-03-22 04:26:40 | 25 | 250 | 2500 | 2500 | 2500 | 2500 | 2500 + s_26 | s_260 | s_2600 | 2052-05-22 02:13:20 | 26 | 260 | 2600 | 2600 | 2600 | 2600 | 2600 | s_26 | s_260 | s_2600 | 2052-05-22 02:13:20 | 26 | 260 | 2600 | 2600 | 2600 | 2600 | 2600 + s_27 | s_270 | s_2700 | 2055-07-24 12:00:00 | 27 | 270 | 2700 | 2700 | 2700 | 2700 | 2700 | s_27 | s_270 | s_2700 | 2055-07-24 12:00:00 | 27 | 270 | 2700 | 2700 | 2700 | 2700 | 2700 + s_28 | s_280 | s_2800 | 2058-09-23 09:46:40 | 28 | 280 | 2800 | 2800 | 2800 | 2800 | 2800 | s_28 | s_280 | s_2800 | 2058-09-23 09:46:40 | 28 | 280 | 2800 | 2800 | 2800 | 2800 | 2800 + s_29 | s_290 | s_2900 | 2061-11-23 07:33:20 | 29 | 290 | 2900 | 2900 | 2900 | 2900 | 2900 | s_29 | s_290 | s_2900 | 2061-11-23 07:33:20 | 29 | 290 | 2900 | 2900 | 2900 | 2900 | 2900 + s_30 | s_300 | s_3000 | 2065-01-24 05:20:00 | 30 | 300 | 3000 | 3000 | 3000 | 3000 | 3000 | s_30 | s_300 | s_3000 | 2065-01-24 05:20:00 | 30 | 300 | 3000 | 3000 | 3000 | 3000 | 3000 + s_31 | s_310 | s_3100 | 2068-03-26 03:06:40 | 31 | 310 | 3100 | 3100 | 3100 | 3100 | 3100 | s_31 | s_310 | s_3100 | 2068-03-26 03:06:40 | 31 | 310 | 3100 | 3100 | 3100 | 3100 | 3100 + s_32 | s_320 | s_3200 | 2071-05-28 12:53:20 | 32 | 320 | 3200 | 3200 | 3200 | 3200 | 3200 | s_32 | s_320 | s_3200 | 2071-05-28 12:53:20 | 32 | 320 | 3200 | 3200 | 3200 | 3200 | 3200 + s_33 | s_330 | s_3300 | 2074-07-28 10:40:00 | 33 | 330 | 3300 | 3300 | 3300 | 3300 | 3300 | s_33 | s_330 | s_3300 | 2074-07-28 10:40:00 | 33 | 330 | 3300 | 3300 | 3300 | 3300 | 3300 + s_34 | s_340 | s_3400 | 2077-09-27 08:26:40 | 34 | 340 | 3400 | 3400 | 3400 | 3400 | 3400 | s_34 | s_340 | s_3400 | 2077-09-27 08:26:40 | 34 | 340 | 3400 | 3400 | 3400 | 3400 | 3400 + s_35 | s_350 | s_3500 | 2080-11-28 06:13:20 | 35 | 350 | 3500 | 3500 | 3500 | 3500 | 3500 | s_35 | s_350 | s_3500 | 2080-11-28 06:13:20 | 35 | 350 | 3500 | 3500 | 3500 | 3500 | 3500 + s_36 | s_360 | s_3600 | 2084-01-29 04:00:00 | 36 | 360 | 3600 | 3600 | 3600 | 3600 | 3600 | s_36 | s_360 | s_3600 | 2084-01-29 04:00:00 | 36 | 360 | 3600 | 3600 | 3600 | 3600 | 3600 + s_37 | s_370 | s_3700 | 2087-04-01 01:46:40 | 37 | 370 | 3700 | 3700 | 3700 | 3700 | 3700 | s_37 | s_370 | s_3700 | 2087-04-01 01:46:40 | 37 | 370 | 3700 | 3700 | 3700 | 3700 | 3700 + s_38 | s_380 | s_3800 | 2090-06-01 11:33:20 | 38 | 380 | 3800 | 3800 | 3800 | 3800 | 3800 | s_38 | s_380 | s_3800 | 2090-06-01 11:33:20 | 38 | 380 | 3800 | 3800 | 3800 | 3800 | 3800 + s_39 | s_390 | s_3900 | 2093-08-01 09:20:00 | 39 | 390 | 3900 | 3900 | 3900 | 3900 | 3900 | s_39 | s_390 | s_3900 | 2093-08-01 09:20:00 | 39 | 390 | 3900 | 3900 | 3900 | 3900 | 3900 + s_40 | s_400 | s_4000 | 2096-10-02 07:06:40 | 40 | 400 | 4000 | 4000 | 4000 | 4000 | 4000 | s_40 | s_400 | s_4000 | 2096-10-02 07:06:40 | 40 | 400 | 4000 | 4000 | 4000 | 4000 | 4000 + s_41 | s_410 | s_4100 | 2099-12-03 04:53:20 | 41 | 410 | 4100 | 4100 | 4100 | 4100 | 4100 | s_41 | s_410 | s_4100 | 2099-12-03 04:53:20 | 41 | 410 | 4100 | 4100 | 4100 | 4100 | 4100 + s_42 | s_420 | s_4200 | 2103-02-04 02:40:00 | 42 | 420 | 4200 | 4200 | 4200 | 4200 | 4200 | s_42 | s_420 | s_4200 | 2103-02-04 02:40:00 | 42 | 420 | 4200 | 4200 | 4200 | 4200 | 4200 + s_43 | s_430 | s_4300 | 2106-04-06 12:26:40 | 43 | 430 | 4300 | 4300 | 4300 | 4300 | 4300 | s_43 | s_430 | s_4300 | 2106-04-06 12:26:40 | 43 | 430 | 4300 | 4300 | 4300 | 4300 | 4300 + s_44 | s_440 | s_4400 | 2109-06-06 10:13:20 | 44 | 440 | 4400 | 4400 | 4400 | 4400 | 4400 | s_44 | s_440 | s_4400 | 2109-06-06 10:13:20 | 44 | 440 | 4400 | 4400 | 4400 | 4400 | 4400 + s_45 | s_450 | s_4500 | 2112-08-07 08:00:00 | 45 | 450 | 4500 | 4500 | 4500 | 4500 | 4500 | s_45 | s_450 | s_4500 | 2112-08-07 08:00:00 | 45 | 450 | 4500 | 4500 | 4500 | 4500 | 4500 + s_46 | s_460 | s_4600 | 2115-10-08 05:46:40 | 46 | 460 | 4600 | 4600 | 4600 | 4600 | 4600 | s_46 | s_460 | s_4600 | 2115-10-08 05:46:40 | 46 | 460 | 4600 | 4600 | 4600 | 4600 | 4600 + s_47 | s_470 | s_4700 | 2118-12-09 03:33:20 | 47 | 470 | 4700 | 4700 | 4700 | 4700 | 4700 | s_47 | s_470 | s_4700 | 2118-12-09 03:33:20 | 47 | 470 | 4700 | 4700 | 4700 | 4700 | 4700 + s_48 | s_480 | s_4800 | 2122-02-08 01:20:00 | 48 | 480 | 4800 | 4800 | 4800 | 4800 | 4800 | s_48 | s_480 | s_4800 | 2122-02-08 01:20:00 | 48 | 480 | 4800 | 4800 | 4800 | 4800 | 4800 + s_49 | s_490 | s_4900 | 2125-04-10 11:06:40 | 49 | 490 | 4900 | 4900 | 4900 | 4900 | 4900 | s_49 | s_490 | s_4900 | 2125-04-10 11:06:40 | 49 | 490 | 4900 | 4900 | 4900 | 4900 | 4900 + s_50 | s_500 | s_5000 | 2128-06-11 08:53:20 | 50 | 500 | 5000 | 5000 | 5000 | 5000 | 5000 | s_50 | s_500 | s_5000 | 2128-06-11 08:53:20 | 50 | 500 | 5000 | 5000 | 5000 | 5000 | 5000 + s_51 | s_510 | s_5100 | 2131-08-12 06:40:00 | 51 | 510 | 5100 | 5100 | 5100 | 5100 | 5100 | s_51 | s_510 | s_5100 | 2131-08-12 06:40:00 | 51 | 510 | 5100 | 5100 | 5100 | 5100 | 5100 + s_52 | s_520 | s_5200 | 2134-10-13 04:26:40 | 52 | 520 | 5200 | 5200 | 5200 | 5200 | 5200 | s_52 | s_520 | s_5200 | 2134-10-13 04:26:40 | 52 | 520 | 5200 | 5200 | 5200 | 5200 | 5200 + s_53 | s_530 | s_5300 | 2137-12-13 02:13:20 | 53 | 530 | 5300 | 5300 | 5300 | 5300 | 5300 | s_53 | s_530 | s_5300 | 2137-12-13 02:13:20 | 53 | 530 | 5300 | 5300 | 5300 | 5300 | 5300 + s_54 | s_540 | s_5400 | 2141-02-13 12:00:00 | 54 | 540 | 5400 | 5400 | 5400 | 5400 | 5400 | s_54 | s_540 | s_5400 | 2141-02-13 12:00:00 | 54 | 540 | 5400 | 5400 | 5400 | 5400 | 5400 + s_55 | s_550 | s_5500 | 2144-04-15 09:46:40 | 55 | 550 | 5500 | 5500 | 5500 | 5500 | 5500 | s_55 | s_550 | s_5500 | 2144-04-15 09:46:40 | 55 | 550 | 5500 | 5500 | 5500 | 5500 | 5500 + s_56 | s_560 | s_5600 | 2147-06-16 07:33:20 | 56 | 560 | 5600 | 5600 | 5600 | 5600 | 5600 | s_56 | s_560 | s_5600 | 2147-06-16 07:33:20 | 56 | 560 | 5600 | 5600 | 5600 | 5600 | 5600 + s_57 | s_570 | s_5700 | 2150-08-17 05:20:00 | 57 | 570 | 5700 | 5700 | 5700 | 5700 | 5700 | s_57 | s_570 | s_5700 | 2150-08-17 05:20:00 | 57 | 570 | 5700 | 5700 | 5700 | 5700 | 5700 + s_58 | s_580 | s_5800 | 2153-10-17 03:06:40 | 58 | 580 | 5800 | 5800 | 5800 | 5800 | 5800 | s_58 | s_580 | s_5800 | 2153-10-17 03:06:40 | 58 | 580 | 5800 | 5800 | 5800 | 5800 | 5800 + s_59 | s_590 | s_5900 | 2156-12-18 12:53:20 | 59 | 590 | 5900 | 5900 | 5900 | 5900 | 5900 | s_59 | s_590 | s_5900 | 2156-12-18 12:53:20 | 59 | 590 | 5900 | 5900 | 5900 | 5900 | 5900 + s_60 | s_600 | s_6000 | 2160-02-18 10:40:00 | 60 | 600 | 6000 | 6000 | 6000 | 6000 | 6000 | s_60 | s_600 | s_6000 | 2160-02-18 10:40:00 | 60 | 600 | 6000 | 6000 | 6000 | 6000 | 6000 + s_61 | s_610 | s_6100 | 2163-04-20 08:26:40 | 61 | 610 | 6100 | 6100 | 6100 | 6100 | 6100 | s_61 | s_610 | s_6100 | 2163-04-20 08:26:40 | 61 | 610 | 6100 | 6100 | 6100 | 6100 | 6100 + s_62 | s_620 | s_6200 | 2166-06-21 06:13:20 | 62 | 620 | 6200 | 6200 | 6200 | 6200 | 6200 | s_62 | s_620 | s_6200 | 2166-06-21 06:13:20 | 62 | 620 | 6200 | 6200 | 6200 | 6200 | 6200 + s_63 | s_630 | s_6300 | 2169-08-21 04:00:00 | 63 | 630 | 6300 | 6300 | 6300 | 6300 | 6300 | s_63 | s_630 | s_6300 | 2169-08-21 04:00:00 | 63 | 630 | 6300 | 6300 | 6300 | 6300 | 6300 + s_64 | s_640 | s_6400 | 2172-10-22 01:46:40 | 64 | 640 | 6400 | 6400 | 6400 | 6400 | 6400 | s_64 | s_640 | s_6400 | 2172-10-22 01:46:40 | 64 | 640 | 6400 | 6400 | 6400 | 6400 | 6400 + s_65 | s_650 | s_6500 | 2175-12-23 11:33:20 | 65 | 650 | 6500 | 6500 | 6500 | 6500 | 6500 | s_65 | s_650 | s_6500 | 2175-12-23 11:33:20 | 65 | 650 | 6500 | 6500 | 6500 | 6500 | 6500 + s_66 | s_660 | s_6600 | 2179-02-22 09:20:00 | 66 | 660 | 6600 | 6600 | 6600 | 6600 | 6600 | s_66 | s_660 | s_6600 | 2179-02-22 09:20:00 | 66 | 660 | 6600 | 6600 | 6600 | 6600 | 6600 + s_67 | s_670 | s_6700 | 2182-04-25 07:06:40 | 67 | 670 | 6700 | 6700 | 6700 | 6700 | 6700 | s_67 | s_670 | s_6700 | 2182-04-25 07:06:40 | 67 | 670 | 6700 | 6700 | 6700 | 6700 | 6700 + s_68 | s_680 | s_6800 | 2185-06-25 04:53:20 | 68 | 680 | 6800 | 6800 | 6800 | 6800 | 6800 | s_68 | s_680 | s_6800 | 2185-06-25 04:53:20 | 68 | 680 | 6800 | 6800 | 6800 | 6800 | 6800 + s_69 | s_690 | s_6900 | 2188-08-26 02:40:00 | 69 | 690 | 6900 | 6900 | 6900 | 6900 | 6900 | s_69 | s_690 | s_6900 | 2188-08-26 02:40:00 | 69 | 690 | 6900 | 6900 | 6900 | 6900 | 6900 + s_70 | s_700 | s_7000 | 2191-10-27 12:26:40 | 70 | 700 | 7000 | 7000 | 7000 | 7000 | 7000 | s_70 | s_700 | s_7000 | 2191-10-27 12:26:40 | 70 | 700 | 7000 | 7000 | 7000 | 7000 | 7000 + s_71 | s_710 | s_7100 | 2194-12-27 10:13:20 | 71 | 710 | 7100 | 7100 | 7100 | 7100 | 7100 | s_71 | s_710 | s_7100 | 2194-12-27 10:13:20 | 71 | 710 | 7100 | 7100 | 7100 | 7100 | 7100 + s_72 | s_720 | s_7200 | 2198-02-27 08:00:00 | 72 | 720 | 7200 | 7200 | 7200 | 7200 | 7200 | s_72 | s_720 | s_7200 | 2198-02-27 08:00:00 | 72 | 720 | 7200 | 7200 | 7200 | 7200 | 7200 + s_73 | s_730 | s_7300 | 2201-04-30 05:46:40 | 73 | 730 | 7300 | 7300 | 7300 | 7300 | 7300 | s_73 | s_730 | s_7300 | 2201-04-30 05:46:40 | 73 | 730 | 7300 | 7300 | 7300 | 7300 | 7300 + s_74 | s_740 | s_7400 | 2204-07-01 03:33:20 | 74 | 740 | 7400 | 7400 | 7400 | 7400 | 7400 | s_74 | s_740 | s_7400 | 2204-07-01 03:33:20 | 74 | 740 | 7400 | 7400 | 7400 | 7400 | 7400 + s_75 | s_750 | s_7500 | 2207-09-01 01:20:00 | 75 | 750 | 7500 | 7500 | 7500 | 7500 | 7500 | s_75 | s_750 | s_7500 | 2207-09-01 01:20:00 | 75 | 750 | 7500 | 7500 | 7500 | 7500 | 7500 + s_76 | s_760 | s_7600 | 2210-11-01 11:06:40 | 76 | 760 | 7600 | 7600 | 7600 | 7600 | 7600 | s_76 | s_760 | s_7600 | 2210-11-01 11:06:40 | 76 | 760 | 7600 | 7600 | 7600 | 7600 | 7600 + s_77 | s_770 | s_7700 | 2214-01-02 08:53:20 | 77 | 770 | 7700 | 7700 | 7700 | 7700 | 7700 | s_77 | s_770 | s_7700 | 2214-01-02 08:53:20 | 77 | 770 | 7700 | 7700 | 7700 | 7700 | 7700 + s_78 | s_780 | s_7800 | 2217-03-04 06:40:00 | 78 | 780 | 7800 | 7800 | 7800 | 7800 | 7800 | s_78 | s_780 | s_7800 | 2217-03-04 06:40:00 | 78 | 780 | 7800 | 7800 | 7800 | 7800 | 7800 + s_79 | s_790 | s_7900 | 2220-05-05 04:26:40 | 79 | 790 | 7900 | 7900 | 7900 | 7900 | 7900 | s_79 | s_790 | s_7900 | 2220-05-05 04:26:40 | 79 | 790 | 7900 | 7900 | 7900 | 7900 | 7900 + s_80 | s_800 | s_8000 | 2223-07-06 02:13:20 | 80 | 800 | 8000 | 8000 | 8000 | 8000 | 8000 | s_80 | s_800 | s_8000 | 2223-07-06 02:13:20 | 80 | 800 | 8000 | 8000 | 8000 | 8000 | 8000 + s_81 | s_810 | s_8100 | 2226-09-06 12:00:00 | 81 | 810 | 8100 | 8100 | 8100 | 8100 | 8100 | s_81 | s_810 | s_8100 | 2226-09-06 12:00:00 | 81 | 810 | 8100 | 8100 | 8100 | 8100 | 8100 + s_82 | s_820 | s_8200 | 2229-11-06 09:46:40 | 82 | 820 | 8200 | 8200 | 8200 | 8200 | 8200 | s_82 | s_820 | s_8200 | 2229-11-06 09:46:40 | 82 | 820 | 8200 | 8200 | 8200 | 8200 | 8200 + s_83 | s_830 | s_8300 | 2233-01-06 07:33:20 | 83 | 830 | 8300 | 8300 | 8300 | 8300 | 8300 | s_83 | s_830 | s_8300 | 2233-01-06 07:33:20 | 83 | 830 | 8300 | 8300 | 8300 | 8300 | 8300 + s_84 | s_840 | s_8400 | 2236-03-09 05:20:00 | 84 | 840 | 8400 | 8400 | 8400 | 8400 | 8400 | s_84 | s_840 | s_8400 | 2236-03-09 05:20:00 | 84 | 840 | 8400 | 8400 | 8400 | 8400 | 8400 + s_85 | s_850 | s_8500 | 2239-05-10 03:06:40 | 85 | 850 | 8500 | 8500 | 8500 | 8500 | 8500 | s_85 | s_850 | s_8500 | 2239-05-10 03:06:40 | 85 | 850 | 8500 | 8500 | 8500 | 8500 | 8500 + s_86 | s_860 | s_8600 | 2242-07-11 12:53:20 | 86 | 860 | 8600 | 8600 | 8600 | 8600 | 8600 | s_86 | s_860 | s_8600 | 2242-07-11 12:53:20 | 86 | 860 | 8600 | 8600 | 8600 | 8600 | 8600 + s_87 | s_870 | s_8700 | 2245-09-10 10:40:00 | 87 | 870 | 8700 | 8700 | 8700 | 8700 | 8700 | s_87 | s_870 | s_8700 | 2245-09-10 10:40:00 | 87 | 870 | 8700 | 8700 | 8700 | 8700 | 8700 + s_88 | s_880 | s_8800 | 2248-11-10 08:26:40 | 88 | 880 | 8800 | 8800 | 8800 | 8800 | 8800 | s_88 | s_880 | s_8800 | 2248-11-10 08:26:40 | 88 | 880 | 8800 | 8800 | 8800 | 8800 | 8800 + s_89 | s_890 | s_8900 | 2252-01-12 06:13:20 | 89 | 890 | 8900 | 8900 | 8900 | 8900 | 8900 | s_89 | s_890 | s_8900 | 2252-01-12 06:13:20 | 89 | 890 | 8900 | 8900 | 8900 | 8900 | 8900 + s_90 | s_900 | s_9000 | 2255-03-14 04:00:00 | 90 | 900 | 9000 | 9000 | 9000 | 9000 | 9000 | s_90 | s_900 | s_9000 | 2255-03-14 04:00:00 | 90 | 900 | 9000 | 9000 | 9000 | 9000 | 9000 + s_91 | s_910 | s_9100 | 2258-05-15 01:46:40 | 91 | 910 | 9100 | 9100 | 9100 | 9100 | 9100 | s_91 | s_910 | s_9100 | 2258-05-15 01:46:40 | 91 | 910 | 9100 | 9100 | 9100 | 9100 | 9100 + s_92 | s_920 | s_9200 | 2261-07-15 11:33:20 | 92 | 920 | 9200 | 9200 | 9200 | 9200 | 9200 | s_92 | s_920 | s_9200 | 2261-07-15 11:33:20 | 92 | 920 | 9200 | 9200 | 9200 | 9200 | 9200 + s_93 | s_930 | s_9300 | 2264-09-14 09:20:00 | 93 | 930 | 9300 | 9300 | 9300 | 9300 | 9300 | s_93 | s_930 | s_9300 | 2264-09-14 09:20:00 | 93 | 930 | 9300 | 9300 | 9300 | 9300 | 9300 + s_94 | s_940 | s_9400 | 2267-11-16 07:06:40 | 94 | 940 | 9400 | 9400 | 9400 | 9400 | 9400 | s_94 | s_940 | s_9400 | 2267-11-16 07:06:40 | 94 | 940 | 9400 | 9400 | 9400 | 9400 | 9400 + s_95 | s_950 | s_9500 | 2271-01-16 04:53:20 | 95 | 950 | 9500 | 9500 | 9500 | 9500 | 9500 | s_95 | s_950 | s_9500 | 2271-01-16 04:53:20 | 95 | 950 | 9500 | 9500 | 9500 | 9500 | 9500 + s_96 | s_960 | s_9600 | 2274-03-19 02:40:00 | 96 | 960 | 9600 | 9600 | 9600 | 9600 | 9600 | s_96 | s_960 | s_9600 | 2274-03-19 02:40:00 | 96 | 960 | 9600 | 9600 | 9600 | 9600 | 9600 + s_97 | s_970 | s_9700 | 2277-05-19 12:26:40 | 97 | 970 | 9700 | 9700 | 9700 | 9700 | 9700 | s_97 | s_970 | s_9700 | 2277-05-19 12:26:40 | 97 | 970 | 9700 | 9700 | 9700 | 9700 | 9700 + s_98 | s_980 | s_9800 | 2280-07-19 10:13:20 | 98 | 980 | 9800 | 9800 | 9800 | 9800 | 9800 | s_98 | s_980 | s_9800 | 2280-07-19 10:13:20 | 98 | 980 | 9800 | 9800 | 9800 | 9800 | 9800 + s_99 | s_990 | s_9900 | 2283-09-20 08:00:00 | 99 | 990 | 9900 | 9900 | 9900 | 9900 | 9900 | s_99 | s_990 | s_9900 | 2283-09-20 08:00:00 | 99 | 990 | 9900 | 9900 | 9900 | 9900 | 9900 + s_100 | s_1000 | s_10000 | 2286-11-20 05:46:40 | 100 | 1000 | 10000 | 10000 | 10000 | 10000 | 10000 | s_100 | s_1000 | s_10000 | 2286-11-20 05:46:40 | 100 | 1000 | 10000 | 10000 | 10000 | 10000 | 10000 +(90 rows) \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/multi_char/runTest.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/multi_char/runTest.py new file mode 100755 index 0000000000..31d291f3ce --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/multi_char/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfMultibyteMultiCharByteDelimiterData(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/multi_char/sql/query01.sql b/automation/tincrepo/main/pxf/features/multibyte_delimiter/multi_char/sql/query01.sql new file mode 100755 index 0000000000..c81a76c0e0 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/multi_char/sql/query01.sql @@ -0,0 +1,5 @@ +-- @description query01 for PXF Multibyte delimiter, multi-character delim cases + +SELECT * from pxf_multibyte_multichar_data ORDER BY n1; + +SELECT * from pxf_multibyte_multichar_data_with_skip ORDER BY n1; \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/no_profile/__init__.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/no_profile/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/no_profile/expected/query01.ans b/automation/tincrepo/main/pxf/features/multibyte_delimiter/no_profile/expected/query01.ans new file mode 100755 index 0000000000..e64f72f7ca --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/no_profile/expected/query01.ans @@ -0,0 +1,8 @@ +-- start_ignore +-- end_ignore +-- @description query01 for PXF Multibyte delimiter, no profile case + +SELECT * from pxf_multibyte_no_profile ORDER BY age; +ERROR: The "pxfdelimited_import" formatter only works with *:text or *:csv profiles. +HINT: Please double check the profile option in the external table definition. +CONTEXT: External table pxf_multibyte_no_profile \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/no_profile/runTest.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/no_profile/runTest.py new file mode 100755 index 0000000000..b6129f0ed6 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/no_profile/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfMultibyteNoProfile(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/no_profile/sql/query01.sql b/automation/tincrepo/main/pxf/features/multibyte_delimiter/no_profile/sql/query01.sql new file mode 100755 index 0000000000..77da05b72f --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/no_profile/sql/query01.sql @@ -0,0 +1,3 @@ +-- @description query01 for PXF Multibyte delimiter, no profile case + +SELECT * from pxf_multibyte_no_profile ORDER BY age; \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/one_byte/__init__.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/one_byte/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/one_byte/expected/query01.ans b/automation/tincrepo/main/pxf/features/multibyte_delimiter/one_byte/expected/query01.ans new file mode 100755 index 0000000000..b9eafe7ff9 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/one_byte/expected/query01.ans @@ -0,0 +1,106 @@ +-- @description query01 for PXF Multibyte delimiter, one byte delimiter option + +SELECT * from pxf_multibyte_onebyte_data ORDER BY n1; + s1 | s2 | s3 | d1 | n1 | n2 | n3 | n4 | n5 | n6 | n7 | s11 | s12 | s13 | d11 | n11 | n12 | n13 | n14 | n15 | n16 | n17 +-------+--------+---------+---------------------+-----+------+-------+-------+-------+-------+-------+-------+--------+---------+---------------------+-----+------+-------+-------+-------+-------+------- + s_1 | s_10 | s_100 | 1973-03-03 09:46:40 | 1 | 10 | 100 | 100 | 100 | 100 | 100 | s_1 | s_10 | s_100 | 1973-03-03 09:46:40 | 1 | 10 | 100 | 100 | 100 | 100 | 100 + s_2 | s_20 | s_200 | 1976-05-03 07:33:20 | 2 | 20 | 200 | 200 | 200 | 200 | 200 | s_2 | s_20 | s_200 | 1976-05-03 07:33:20 | 2 | 20 | 200 | 200 | 200 | 200 | 200 + s_3 | s_30 | s_300 | 1979-07-05 05:20:00 | 3 | 30 | 300 | 300 | 300 | 300 | 300 | s_3 | s_30 | s_300 | 1979-07-05 05:20:00 | 3 | 30 | 300 | 300 | 300 | 300 | 300 + s_4 | s_40 | s_400 | 1982-09-04 03:06:40 | 4 | 40 | 400 | 400 | 400 | 400 | 400 | s_4 | s_40 | s_400 | 1982-09-04 03:06:40 | 4 | 40 | 400 | 400 | 400 | 400 | 400 + s_5 | s_50 | s_500 | 1985-11-05 12:53:20 | 5 | 50 | 500 | 500 | 500 | 500 | 500 | s_5 | s_50 | s_500 | 1985-11-05 12:53:20 | 5 | 50 | 500 | 500 | 500 | 500 | 500 + s_6 | s_60 | s_600 | 1989-01-05 10:40:00 | 6 | 60 | 600 | 600 | 600 | 600 | 600 | s_6 | s_60 | s_600 | 1989-01-05 10:40:00 | 6 | 60 | 600 | 600 | 600 | 600 | 600 + s_7 | s_70 | s_700 | 1992-03-07 08:26:40 | 7 | 70 | 700 | 700 | 700 | 700 | 700 | s_7 | s_70 | s_700 | 1992-03-07 08:26:40 | 7 | 70 | 700 | 700 | 700 | 700 | 700 + s_8 | s_80 | s_800 | 1995-05-09 06:13:20 | 8 | 80 | 800 | 800 | 800 | 800 | 800 | s_8 | s_80 | s_800 | 1995-05-09 06:13:20 | 8 | 80 | 800 | 800 | 800 | 800 | 800 + s_9 | s_90 | s_900 | 1998-07-09 04:00:00 | 9 | 90 | 900 | 900 | 900 | 900 | 900 | s_9 | s_90 | s_900 | 1998-07-09 04:00:00 | 9 | 90 | 900 | 900 | 900 | 900 | 900 + s_10 | s_100 | s_1000 | 2001-09-09 01:46:40 | 10 | 100 | 1000 | 1000 | 1000 | 1000 | 1000 | s_10 | s_100 | s_1000 | 2001-09-09 01:46:40 | 10 | 100 | 1000 | 1000 | 1000 | 1000 | 1000 + s_11 | s_110 | s_1100 | 2004-11-09 11:33:20 | 11 | 110 | 1100 | 1100 | 1100 | 1100 | 1100 | s_11 | s_110 | s_1100 | 2004-11-09 11:33:20 | 11 | 110 | 1100 | 1100 | 1100 | 1100 | 1100 + s_12 | s_120 | s_1200 | 2008-01-10 09:20:00 | 12 | 120 | 1200 | 1200 | 1200 | 1200 | 1200 | s_12 | s_120 | s_1200 | 2008-01-10 09:20:00 | 12 | 120 | 1200 | 1200 | 1200 | 1200 | 1200 + s_13 | s_130 | s_1300 | 2011-03-13 07:06:40 | 13 | 130 | 1300 | 1300 | 1300 | 1300 | 1300 | s_13 | s_130 | s_1300 | 2011-03-13 07:06:40 | 13 | 130 | 1300 | 1300 | 1300 | 1300 | 1300 + s_14 | s_140 | s_1400 | 2014-05-13 04:53:20 | 14 | 140 | 1400 | 1400 | 1400 | 1400 | 1400 | s_14 | s_140 | s_1400 | 2014-05-13 04:53:20 | 14 | 140 | 1400 | 1400 | 1400 | 1400 | 1400 + s_15 | s_150 | s_1500 | 2017-07-14 02:40:00 | 15 | 150 | 1500 | 1500 | 1500 | 1500 | 1500 | s_15 | s_150 | s_1500 | 2017-07-14 02:40:00 | 15 | 150 | 1500 | 1500 | 1500 | 1500 | 1500 + s_16 | s_160 | s_1600 | 2020-09-13 12:26:40 | 16 | 160 | 1600 | 1600 | 1600 | 1600 | 1600 | s_16 | s_160 | s_1600 | 2020-09-13 12:26:40 | 16 | 160 | 1600 | 1600 | 1600 | 1600 | 1600 + s_17 | s_170 | s_1700 | 2023-11-14 10:13:20 | 17 | 170 | 1700 | 1700 | 1700 | 1700 | 1700 | s_17 | s_170 | s_1700 | 2023-11-14 10:13:20 | 17 | 170 | 1700 | 1700 | 1700 | 1700 | 1700 + s_18 | s_180 | s_1800 | 2027-01-15 08:00:00 | 18 | 180 | 1800 | 1800 | 1800 | 1800 | 1800 | s_18 | s_180 | s_1800 | 2027-01-15 08:00:00 | 18 | 180 | 1800 | 1800 | 1800 | 1800 | 1800 + s_19 | s_190 | s_1900 | 2030-03-17 05:46:40 | 19 | 190 | 1900 | 1900 | 1900 | 1900 | 1900 | s_19 | s_190 | s_1900 | 2030-03-17 05:46:40 | 19 | 190 | 1900 | 1900 | 1900 | 1900 | 1900 + s_20 | s_200 | s_2000 | 2033-05-18 03:33:20 | 20 | 200 | 2000 | 2000 | 2000 | 2000 | 2000 | s_20 | s_200 | s_2000 | 2033-05-18 03:33:20 | 20 | 200 | 2000 | 2000 | 2000 | 2000 | 2000 + s_21 | s_210 | s_2100 | 2036-07-18 01:20:00 | 21 | 210 | 2100 | 2100 | 2100 | 2100 | 2100 | s_21 | s_210 | s_2100 | 2036-07-18 01:20:00 | 21 | 210 | 2100 | 2100 | 2100 | 2100 | 2100 + s_22 | s_220 | s_2200 | 2039-09-18 11:06:40 | 22 | 220 | 2200 | 2200 | 2200 | 2200 | 2200 | s_22 | s_220 | s_2200 | 2039-09-18 11:06:40 | 22 | 220 | 2200 | 2200 | 2200 | 2200 | 2200 + s_23 | s_230 | s_2300 | 2042-11-19 08:53:20 | 23 | 230 | 2300 | 2300 | 2300 | 2300 | 2300 | s_23 | s_230 | s_2300 | 2042-11-19 08:53:20 | 23 | 230 | 2300 | 2300 | 2300 | 2300 | 2300 + s_24 | s_240 | s_2400 | 2046-01-19 06:40:00 | 24 | 240 | 2400 | 2400 | 2400 | 2400 | 2400 | s_24 | s_240 | s_2400 | 2046-01-19 06:40:00 | 24 | 240 | 2400 | 2400 | 2400 | 2400 | 2400 + s_25 | s_250 | s_2500 | 2049-03-22 04:26:40 | 25 | 250 | 2500 | 2500 | 2500 | 2500 | 2500 | s_25 | s_250 | s_2500 | 2049-03-22 04:26:40 | 25 | 250 | 2500 | 2500 | 2500 | 2500 | 2500 + s_26 | s_260 | s_2600 | 2052-05-22 02:13:20 | 26 | 260 | 2600 | 2600 | 2600 | 2600 | 2600 | s_26 | s_260 | s_2600 | 2052-05-22 02:13:20 | 26 | 260 | 2600 | 2600 | 2600 | 2600 | 2600 + s_27 | s_270 | s_2700 | 2055-07-24 12:00:00 | 27 | 270 | 2700 | 2700 | 2700 | 2700 | 2700 | s_27 | s_270 | s_2700 | 2055-07-24 12:00:00 | 27 | 270 | 2700 | 2700 | 2700 | 2700 | 2700 + s_28 | s_280 | s_2800 | 2058-09-23 09:46:40 | 28 | 280 | 2800 | 2800 | 2800 | 2800 | 2800 | s_28 | s_280 | s_2800 | 2058-09-23 09:46:40 | 28 | 280 | 2800 | 2800 | 2800 | 2800 | 2800 + s_29 | s_290 | s_2900 | 2061-11-23 07:33:20 | 29 | 290 | 2900 | 2900 | 2900 | 2900 | 2900 | s_29 | s_290 | s_2900 | 2061-11-23 07:33:20 | 29 | 290 | 2900 | 2900 | 2900 | 2900 | 2900 + s_30 | s_300 | s_3000 | 2065-01-24 05:20:00 | 30 | 300 | 3000 | 3000 | 3000 | 3000 | 3000 | s_30 | s_300 | s_3000 | 2065-01-24 05:20:00 | 30 | 300 | 3000 | 3000 | 3000 | 3000 | 3000 + s_31 | s_310 | s_3100 | 2068-03-26 03:06:40 | 31 | 310 | 3100 | 3100 | 3100 | 3100 | 3100 | s_31 | s_310 | s_3100 | 2068-03-26 03:06:40 | 31 | 310 | 3100 | 3100 | 3100 | 3100 | 3100 + s_32 | s_320 | s_3200 | 2071-05-28 12:53:20 | 32 | 320 | 3200 | 3200 | 3200 | 3200 | 3200 | s_32 | s_320 | s_3200 | 2071-05-28 12:53:20 | 32 | 320 | 3200 | 3200 | 3200 | 3200 | 3200 + s_33 | s_330 | s_3300 | 2074-07-28 10:40:00 | 33 | 330 | 3300 | 3300 | 3300 | 3300 | 3300 | s_33 | s_330 | s_3300 | 2074-07-28 10:40:00 | 33 | 330 | 3300 | 3300 | 3300 | 3300 | 3300 + s_34 | s_340 | s_3400 | 2077-09-27 08:26:40 | 34 | 340 | 3400 | 3400 | 3400 | 3400 | 3400 | s_34 | s_340 | s_3400 | 2077-09-27 08:26:40 | 34 | 340 | 3400 | 3400 | 3400 | 3400 | 3400 + s_35 | s_350 | s_3500 | 2080-11-28 06:13:20 | 35 | 350 | 3500 | 3500 | 3500 | 3500 | 3500 | s_35 | s_350 | s_3500 | 2080-11-28 06:13:20 | 35 | 350 | 3500 | 3500 | 3500 | 3500 | 3500 + s_36 | s_360 | s_3600 | 2084-01-29 04:00:00 | 36 | 360 | 3600 | 3600 | 3600 | 3600 | 3600 | s_36 | s_360 | s_3600 | 2084-01-29 04:00:00 | 36 | 360 | 3600 | 3600 | 3600 | 3600 | 3600 + s_37 | s_370 | s_3700 | 2087-04-01 01:46:40 | 37 | 370 | 3700 | 3700 | 3700 | 3700 | 3700 | s_37 | s_370 | s_3700 | 2087-04-01 01:46:40 | 37 | 370 | 3700 | 3700 | 3700 | 3700 | 3700 + s_38 | s_380 | s_3800 | 2090-06-01 11:33:20 | 38 | 380 | 3800 | 3800 | 3800 | 3800 | 3800 | s_38 | s_380 | s_3800 | 2090-06-01 11:33:20 | 38 | 380 | 3800 | 3800 | 3800 | 3800 | 3800 + s_39 | s_390 | s_3900 | 2093-08-01 09:20:00 | 39 | 390 | 3900 | 3900 | 3900 | 3900 | 3900 | s_39 | s_390 | s_3900 | 2093-08-01 09:20:00 | 39 | 390 | 3900 | 3900 | 3900 | 3900 | 3900 + s_40 | s_400 | s_4000 | 2096-10-02 07:06:40 | 40 | 400 | 4000 | 4000 | 4000 | 4000 | 4000 | s_40 | s_400 | s_4000 | 2096-10-02 07:06:40 | 40 | 400 | 4000 | 4000 | 4000 | 4000 | 4000 + s_41 | s_410 | s_4100 | 2099-12-03 04:53:20 | 41 | 410 | 4100 | 4100 | 4100 | 4100 | 4100 | s_41 | s_410 | s_4100 | 2099-12-03 04:53:20 | 41 | 410 | 4100 | 4100 | 4100 | 4100 | 4100 + s_42 | s_420 | s_4200 | 2103-02-04 02:40:00 | 42 | 420 | 4200 | 4200 | 4200 | 4200 | 4200 | s_42 | s_420 | s_4200 | 2103-02-04 02:40:00 | 42 | 420 | 4200 | 4200 | 4200 | 4200 | 4200 + s_43 | s_430 | s_4300 | 2106-04-06 12:26:40 | 43 | 430 | 4300 | 4300 | 4300 | 4300 | 4300 | s_43 | s_430 | s_4300 | 2106-04-06 12:26:40 | 43 | 430 | 4300 | 4300 | 4300 | 4300 | 4300 + s_44 | s_440 | s_4400 | 2109-06-06 10:13:20 | 44 | 440 | 4400 | 4400 | 4400 | 4400 | 4400 | s_44 | s_440 | s_4400 | 2109-06-06 10:13:20 | 44 | 440 | 4400 | 4400 | 4400 | 4400 | 4400 + s_45 | s_450 | s_4500 | 2112-08-07 08:00:00 | 45 | 450 | 4500 | 4500 | 4500 | 4500 | 4500 | s_45 | s_450 | s_4500 | 2112-08-07 08:00:00 | 45 | 450 | 4500 | 4500 | 4500 | 4500 | 4500 + s_46 | s_460 | s_4600 | 2115-10-08 05:46:40 | 46 | 460 | 4600 | 4600 | 4600 | 4600 | 4600 | s_46 | s_460 | s_4600 | 2115-10-08 05:46:40 | 46 | 460 | 4600 | 4600 | 4600 | 4600 | 4600 + s_47 | s_470 | s_4700 | 2118-12-09 03:33:20 | 47 | 470 | 4700 | 4700 | 4700 | 4700 | 4700 | s_47 | s_470 | s_4700 | 2118-12-09 03:33:20 | 47 | 470 | 4700 | 4700 | 4700 | 4700 | 4700 + s_48 | s_480 | s_4800 | 2122-02-08 01:20:00 | 48 | 480 | 4800 | 4800 | 4800 | 4800 | 4800 | s_48 | s_480 | s_4800 | 2122-02-08 01:20:00 | 48 | 480 | 4800 | 4800 | 4800 | 4800 | 4800 + s_49 | s_490 | s_4900 | 2125-04-10 11:06:40 | 49 | 490 | 4900 | 4900 | 4900 | 4900 | 4900 | s_49 | s_490 | s_4900 | 2125-04-10 11:06:40 | 49 | 490 | 4900 | 4900 | 4900 | 4900 | 4900 + s_50 | s_500 | s_5000 | 2128-06-11 08:53:20 | 50 | 500 | 5000 | 5000 | 5000 | 5000 | 5000 | s_50 | s_500 | s_5000 | 2128-06-11 08:53:20 | 50 | 500 | 5000 | 5000 | 5000 | 5000 | 5000 + s_51 | s_510 | s_5100 | 2131-08-12 06:40:00 | 51 | 510 | 5100 | 5100 | 5100 | 5100 | 5100 | s_51 | s_510 | s_5100 | 2131-08-12 06:40:00 | 51 | 510 | 5100 | 5100 | 5100 | 5100 | 5100 + s_52 | s_520 | s_5200 | 2134-10-13 04:26:40 | 52 | 520 | 5200 | 5200 | 5200 | 5200 | 5200 | s_52 | s_520 | s_5200 | 2134-10-13 04:26:40 | 52 | 520 | 5200 | 5200 | 5200 | 5200 | 5200 + s_53 | s_530 | s_5300 | 2137-12-13 02:13:20 | 53 | 530 | 5300 | 5300 | 5300 | 5300 | 5300 | s_53 | s_530 | s_5300 | 2137-12-13 02:13:20 | 53 | 530 | 5300 | 5300 | 5300 | 5300 | 5300 + s_54 | s_540 | s_5400 | 2141-02-13 12:00:00 | 54 | 540 | 5400 | 5400 | 5400 | 5400 | 5400 | s_54 | s_540 | s_5400 | 2141-02-13 12:00:00 | 54 | 540 | 5400 | 5400 | 5400 | 5400 | 5400 + s_55 | s_550 | s_5500 | 2144-04-15 09:46:40 | 55 | 550 | 5500 | 5500 | 5500 | 5500 | 5500 | s_55 | s_550 | s_5500 | 2144-04-15 09:46:40 | 55 | 550 | 5500 | 5500 | 5500 | 5500 | 5500 + s_56 | s_560 | s_5600 | 2147-06-16 07:33:20 | 56 | 560 | 5600 | 5600 | 5600 | 5600 | 5600 | s_56 | s_560 | s_5600 | 2147-06-16 07:33:20 | 56 | 560 | 5600 | 5600 | 5600 | 5600 | 5600 + s_57 | s_570 | s_5700 | 2150-08-17 05:20:00 | 57 | 570 | 5700 | 5700 | 5700 | 5700 | 5700 | s_57 | s_570 | s_5700 | 2150-08-17 05:20:00 | 57 | 570 | 5700 | 5700 | 5700 | 5700 | 5700 + s_58 | s_580 | s_5800 | 2153-10-17 03:06:40 | 58 | 580 | 5800 | 5800 | 5800 | 5800 | 5800 | s_58 | s_580 | s_5800 | 2153-10-17 03:06:40 | 58 | 580 | 5800 | 5800 | 5800 | 5800 | 5800 + s_59 | s_590 | s_5900 | 2156-12-18 12:53:20 | 59 | 590 | 5900 | 5900 | 5900 | 5900 | 5900 | s_59 | s_590 | s_5900 | 2156-12-18 12:53:20 | 59 | 590 | 5900 | 5900 | 5900 | 5900 | 5900 + s_60 | s_600 | s_6000 | 2160-02-18 10:40:00 | 60 | 600 | 6000 | 6000 | 6000 | 6000 | 6000 | s_60 | s_600 | s_6000 | 2160-02-18 10:40:00 | 60 | 600 | 6000 | 6000 | 6000 | 6000 | 6000 + s_61 | s_610 | s_6100 | 2163-04-20 08:26:40 | 61 | 610 | 6100 | 6100 | 6100 | 6100 | 6100 | s_61 | s_610 | s_6100 | 2163-04-20 08:26:40 | 61 | 610 | 6100 | 6100 | 6100 | 6100 | 6100 + s_62 | s_620 | s_6200 | 2166-06-21 06:13:20 | 62 | 620 | 6200 | 6200 | 6200 | 6200 | 6200 | s_62 | s_620 | s_6200 | 2166-06-21 06:13:20 | 62 | 620 | 6200 | 6200 | 6200 | 6200 | 6200 + s_63 | s_630 | s_6300 | 2169-08-21 04:00:00 | 63 | 630 | 6300 | 6300 | 6300 | 6300 | 6300 | s_63 | s_630 | s_6300 | 2169-08-21 04:00:00 | 63 | 630 | 6300 | 6300 | 6300 | 6300 | 6300 + s_64 | s_640 | s_6400 | 2172-10-22 01:46:40 | 64 | 640 | 6400 | 6400 | 6400 | 6400 | 6400 | s_64 | s_640 | s_6400 | 2172-10-22 01:46:40 | 64 | 640 | 6400 | 6400 | 6400 | 6400 | 6400 + s_65 | s_650 | s_6500 | 2175-12-23 11:33:20 | 65 | 650 | 6500 | 6500 | 6500 | 6500 | 6500 | s_65 | s_650 | s_6500 | 2175-12-23 11:33:20 | 65 | 650 | 6500 | 6500 | 6500 | 6500 | 6500 + s_66 | s_660 | s_6600 | 2179-02-22 09:20:00 | 66 | 660 | 6600 | 6600 | 6600 | 6600 | 6600 | s_66 | s_660 | s_6600 | 2179-02-22 09:20:00 | 66 | 660 | 6600 | 6600 | 6600 | 6600 | 6600 + s_67 | s_670 | s_6700 | 2182-04-25 07:06:40 | 67 | 670 | 6700 | 6700 | 6700 | 6700 | 6700 | s_67 | s_670 | s_6700 | 2182-04-25 07:06:40 | 67 | 670 | 6700 | 6700 | 6700 | 6700 | 6700 + s_68 | s_680 | s_6800 | 2185-06-25 04:53:20 | 68 | 680 | 6800 | 6800 | 6800 | 6800 | 6800 | s_68 | s_680 | s_6800 | 2185-06-25 04:53:20 | 68 | 680 | 6800 | 6800 | 6800 | 6800 | 6800 + s_69 | s_690 | s_6900 | 2188-08-26 02:40:00 | 69 | 690 | 6900 | 6900 | 6900 | 6900 | 6900 | s_69 | s_690 | s_6900 | 2188-08-26 02:40:00 | 69 | 690 | 6900 | 6900 | 6900 | 6900 | 6900 + s_70 | s_700 | s_7000 | 2191-10-27 12:26:40 | 70 | 700 | 7000 | 7000 | 7000 | 7000 | 7000 | s_70 | s_700 | s_7000 | 2191-10-27 12:26:40 | 70 | 700 | 7000 | 7000 | 7000 | 7000 | 7000 + s_71 | s_710 | s_7100 | 2194-12-27 10:13:20 | 71 | 710 | 7100 | 7100 | 7100 | 7100 | 7100 | s_71 | s_710 | s_7100 | 2194-12-27 10:13:20 | 71 | 710 | 7100 | 7100 | 7100 | 7100 | 7100 + s_72 | s_720 | s_7200 | 2198-02-27 08:00:00 | 72 | 720 | 7200 | 7200 | 7200 | 7200 | 7200 | s_72 | s_720 | s_7200 | 2198-02-27 08:00:00 | 72 | 720 | 7200 | 7200 | 7200 | 7200 | 7200 + s_73 | s_730 | s_7300 | 2201-04-30 05:46:40 | 73 | 730 | 7300 | 7300 | 7300 | 7300 | 7300 | s_73 | s_730 | s_7300 | 2201-04-30 05:46:40 | 73 | 730 | 7300 | 7300 | 7300 | 7300 | 7300 + s_74 | s_740 | s_7400 | 2204-07-01 03:33:20 | 74 | 740 | 7400 | 7400 | 7400 | 7400 | 7400 | s_74 | s_740 | s_7400 | 2204-07-01 03:33:20 | 74 | 740 | 7400 | 7400 | 7400 | 7400 | 7400 + s_75 | s_750 | s_7500 | 2207-09-01 01:20:00 | 75 | 750 | 7500 | 7500 | 7500 | 7500 | 7500 | s_75 | s_750 | s_7500 | 2207-09-01 01:20:00 | 75 | 750 | 7500 | 7500 | 7500 | 7500 | 7500 + s_76 | s_760 | s_7600 | 2210-11-01 11:06:40 | 76 | 760 | 7600 | 7600 | 7600 | 7600 | 7600 | s_76 | s_760 | s_7600 | 2210-11-01 11:06:40 | 76 | 760 | 7600 | 7600 | 7600 | 7600 | 7600 + s_77 | s_770 | s_7700 | 2214-01-02 08:53:20 | 77 | 770 | 7700 | 7700 | 7700 | 7700 | 7700 | s_77 | s_770 | s_7700 | 2214-01-02 08:53:20 | 77 | 770 | 7700 | 7700 | 7700 | 7700 | 7700 + s_78 | s_780 | s_7800 | 2217-03-04 06:40:00 | 78 | 780 | 7800 | 7800 | 7800 | 7800 | 7800 | s_78 | s_780 | s_7800 | 2217-03-04 06:40:00 | 78 | 780 | 7800 | 7800 | 7800 | 7800 | 7800 + s_79 | s_790 | s_7900 | 2220-05-05 04:26:40 | 79 | 790 | 7900 | 7900 | 7900 | 7900 | 7900 | s_79 | s_790 | s_7900 | 2220-05-05 04:26:40 | 79 | 790 | 7900 | 7900 | 7900 | 7900 | 7900 + s_80 | s_800 | s_8000 | 2223-07-06 02:13:20 | 80 | 800 | 8000 | 8000 | 8000 | 8000 | 8000 | s_80 | s_800 | s_8000 | 2223-07-06 02:13:20 | 80 | 800 | 8000 | 8000 | 8000 | 8000 | 8000 + s_81 | s_810 | s_8100 | 2226-09-06 12:00:00 | 81 | 810 | 8100 | 8100 | 8100 | 8100 | 8100 | s_81 | s_810 | s_8100 | 2226-09-06 12:00:00 | 81 | 810 | 8100 | 8100 | 8100 | 8100 | 8100 + s_82 | s_820 | s_8200 | 2229-11-06 09:46:40 | 82 | 820 | 8200 | 8200 | 8200 | 8200 | 8200 | s_82 | s_820 | s_8200 | 2229-11-06 09:46:40 | 82 | 820 | 8200 | 8200 | 8200 | 8200 | 8200 + s_83 | s_830 | s_8300 | 2233-01-06 07:33:20 | 83 | 830 | 8300 | 8300 | 8300 | 8300 | 8300 | s_83 | s_830 | s_8300 | 2233-01-06 07:33:20 | 83 | 830 | 8300 | 8300 | 8300 | 8300 | 8300 + s_84 | s_840 | s_8400 | 2236-03-09 05:20:00 | 84 | 840 | 8400 | 8400 | 8400 | 8400 | 8400 | s_84 | s_840 | s_8400 | 2236-03-09 05:20:00 | 84 | 840 | 8400 | 8400 | 8400 | 8400 | 8400 + s_85 | s_850 | s_8500 | 2239-05-10 03:06:40 | 85 | 850 | 8500 | 8500 | 8500 | 8500 | 8500 | s_85 | s_850 | s_8500 | 2239-05-10 03:06:40 | 85 | 850 | 8500 | 8500 | 8500 | 8500 | 8500 + s_86 | s_860 | s_8600 | 2242-07-11 12:53:20 | 86 | 860 | 8600 | 8600 | 8600 | 8600 | 8600 | s_86 | s_860 | s_8600 | 2242-07-11 12:53:20 | 86 | 860 | 8600 | 8600 | 8600 | 8600 | 8600 + s_87 | s_870 | s_8700 | 2245-09-10 10:40:00 | 87 | 870 | 8700 | 8700 | 8700 | 8700 | 8700 | s_87 | s_870 | s_8700 | 2245-09-10 10:40:00 | 87 | 870 | 8700 | 8700 | 8700 | 8700 | 8700 + s_88 | s_880 | s_8800 | 2248-11-10 08:26:40 | 88 | 880 | 8800 | 8800 | 8800 | 8800 | 8800 | s_88 | s_880 | s_8800 | 2248-11-10 08:26:40 | 88 | 880 | 8800 | 8800 | 8800 | 8800 | 8800 + s_89 | s_890 | s_8900 | 2252-01-12 06:13:20 | 89 | 890 | 8900 | 8900 | 8900 | 8900 | 8900 | s_89 | s_890 | s_8900 | 2252-01-12 06:13:20 | 89 | 890 | 8900 | 8900 | 8900 | 8900 | 8900 + s_90 | s_900 | s_9000 | 2255-03-14 04:00:00 | 90 | 900 | 9000 | 9000 | 9000 | 9000 | 9000 | s_90 | s_900 | s_9000 | 2255-03-14 04:00:00 | 90 | 900 | 9000 | 9000 | 9000 | 9000 | 9000 + s_91 | s_910 | s_9100 | 2258-05-15 01:46:40 | 91 | 910 | 9100 | 9100 | 9100 | 9100 | 9100 | s_91 | s_910 | s_9100 | 2258-05-15 01:46:40 | 91 | 910 | 9100 | 9100 | 9100 | 9100 | 9100 + s_92 | s_920 | s_9200 | 2261-07-15 11:33:20 | 92 | 920 | 9200 | 9200 | 9200 | 9200 | 9200 | s_92 | s_920 | s_9200 | 2261-07-15 11:33:20 | 92 | 920 | 9200 | 9200 | 9200 | 9200 | 9200 + s_93 | s_930 | s_9300 | 2264-09-14 09:20:00 | 93 | 930 | 9300 | 9300 | 9300 | 9300 | 9300 | s_93 | s_930 | s_9300 | 2264-09-14 09:20:00 | 93 | 930 | 9300 | 9300 | 9300 | 9300 | 9300 + s_94 | s_940 | s_9400 | 2267-11-16 07:06:40 | 94 | 940 | 9400 | 9400 | 9400 | 9400 | 9400 | s_94 | s_940 | s_9400 | 2267-11-16 07:06:40 | 94 | 940 | 9400 | 9400 | 9400 | 9400 | 9400 + s_95 | s_950 | s_9500 | 2271-01-16 04:53:20 | 95 | 950 | 9500 | 9500 | 9500 | 9500 | 9500 | s_95 | s_950 | s_9500 | 2271-01-16 04:53:20 | 95 | 950 | 9500 | 9500 | 9500 | 9500 | 9500 + s_96 | s_960 | s_9600 | 2274-03-19 02:40:00 | 96 | 960 | 9600 | 9600 | 9600 | 9600 | 9600 | s_96 | s_960 | s_9600 | 2274-03-19 02:40:00 | 96 | 960 | 9600 | 9600 | 9600 | 9600 | 9600 + s_97 | s_970 | s_9700 | 2277-05-19 12:26:40 | 97 | 970 | 9700 | 9700 | 9700 | 9700 | 9700 | s_97 | s_970 | s_9700 | 2277-05-19 12:26:40 | 97 | 970 | 9700 | 9700 | 9700 | 9700 | 9700 + s_98 | s_980 | s_9800 | 2280-07-19 10:13:20 | 98 | 980 | 9800 | 9800 | 9800 | 9800 | 9800 | s_98 | s_980 | s_9800 | 2280-07-19 10:13:20 | 98 | 980 | 9800 | 9800 | 9800 | 9800 | 9800 + s_99 | s_990 | s_9900 | 2283-09-20 08:00:00 | 99 | 990 | 9900 | 9900 | 9900 | 9900 | 9900 | s_99 | s_990 | s_9900 | 2283-09-20 08:00:00 | 99 | 990 | 9900 | 9900 | 9900 | 9900 | 9900 + s_100 | s_1000 | s_10000 | 2286-11-20 05:46:40 | 100 | 1000 | 10000 | 10000 | 10000 | 10000 | 10000 | s_100 | s_1000 | s_10000 | 2286-11-20 05:46:40 | 100 | 1000 | 10000 | 10000 | 10000 | 10000 | 10000 +(100 rows) diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/one_byte/runTest.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/one_byte/runTest.py new file mode 100755 index 0000000000..1c157338f6 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/one_byte/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfMultibyteOneByteDelimiterData(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/one_byte/sql/query01.sql b/automation/tincrepo/main/pxf/features/multibyte_delimiter/one_byte/sql/query01.sql new file mode 100755 index 0000000000..de267a49f1 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/one_byte/sql/query01.sql @@ -0,0 +1,3 @@ +-- @description query01 for PXF Multibyte delimiter, one byte delimiter option + +SELECT * from pxf_multibyte_onebyte_data ORDER BY n1; diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/one_col/__init__.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/one_col/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/one_col/expected/query01.ans b/automation/tincrepo/main/pxf/features/multibyte_delimiter/one_col/expected/query01.ans new file mode 100755 index 0000000000..246b6501cb --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/one_col/expected/query01.ans @@ -0,0 +1,11 @@ +-- start_ignore +-- end_ignore +-- @description query01 for PXF Multibyte delimiter, one col +SELECT * from pxf_multibyte_onecol_data ORDER BY s1; + s1 +--------------------------------------------- + minden amire szüksége van a szeretet + règles d'automation + tá sé seo le tástáil dea- +(3 rows) + diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/one_col/runTest.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/one_col/runTest.py new file mode 100755 index 0000000000..98f0b77aa7 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/one_col/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfMultibyteOneCol(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/one_col/sql/query01.sql b/automation/tincrepo/main/pxf/features/multibyte_delimiter/one_col/sql/query01.sql new file mode 100755 index 0000000000..288a8a3dad --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/one_col/sql/query01.sql @@ -0,0 +1,3 @@ +-- @description query01 for PXF Multibyte delimiter, one col + +SELECT * from pxf_multibyte_onecol_data ORDER BY s1; diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/one_col_quote/__init__.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/one_col_quote/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/one_col_quote/expected/query01.ans b/automation/tincrepo/main/pxf/features/multibyte_delimiter/one_col_quote/expected/query01.ans new file mode 100755 index 0000000000..ebafea46eb --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/one_col_quote/expected/query01.ans @@ -0,0 +1,11 @@ +-- start_ignore +-- end_ignore +-- @description query01 for PXF Multibyte delimiter, one col with quote +SELECT * from pxf_multibyte_onecol_quote_data ORDER BY s1; + s1 +------------------------------------------- + minden amire szüksége van a szeretet + règles d'automation + tá sé seo le tástáil dea- +(3 rows) + diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/one_col_quote/runTest.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/one_col_quote/runTest.py new file mode 100755 index 0000000000..95876a90ce --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/one_col_quote/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfMultibyteOneColQuote(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/one_col_quote/sql/query01.sql b/automation/tincrepo/main/pxf/features/multibyte_delimiter/one_col_quote/sql/query01.sql new file mode 100755 index 0000000000..810f158242 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/one_col_quote/sql/query01.sql @@ -0,0 +1,3 @@ +-- @description query01 for PXF Multibyte delimiter, one col with quote + +SELECT * from pxf_multibyte_onecol_quote_data ORDER BY s1; diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/quote_escape_newline/__init__.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/quote_escape_newline/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/quote_escape_newline/expected/query01.ans b/automation/tincrepo/main/pxf/features/multibyte_delimiter/quote_escape_newline/expected/query01.ans new file mode 100755 index 0000000000..3422e4d7c9 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/quote_escape_newline/expected/query01.ans @@ -0,0 +1,6 @@ +-- start_ignore +-- end_ignore +-- @description query01 for PXF Multibyte delimiter, 2-byte delim with wrong escape case +SELECT * from pxf_multibyte_quote_escape_newline_data ORDER BY n1; +ERROR: NEWLINE can only be LF, CRLF, or CR (pxfdelimited_formatter.c:xxx) +CONTEXT: External table pxf_multibyte_quote_escape_newline_data diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/quote_escape_newline/runTest.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/quote_escape_newline/runTest.py new file mode 100755 index 0000000000..6400dd3ecd --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/quote_escape_newline/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfMultibyteQuoteEscapeNewlineData(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/quote_escape_newline/sql/query01.sql b/automation/tincrepo/main/pxf/features/multibyte_delimiter/quote_escape_newline/sql/query01.sql new file mode 100755 index 0000000000..0f813f88bb --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/quote_escape_newline/sql/query01.sql @@ -0,0 +1,3 @@ +-- @description query01 for PXF Multibyte delimiter, 2-byte delim with wrong escape case + +SELECT * from pxf_multibyte_quote_escape_newline_data ORDER BY n1; \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/three_byte/__init__.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/three_byte/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/three_byte/expected/query01.ans b/automation/tincrepo/main/pxf/features/multibyte_delimiter/three_byte/expected/query01.ans new file mode 100755 index 0000000000..70c2bd71e4 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/three_byte/expected/query01.ans @@ -0,0 +1,201 @@ +-- @description query01 for PXF Multibyte delimiter, 3-byte delim cases + +SELECT * from pxf_multibyte_threebyte_data ORDER BY n1; + s1 | s2 | s3 | d1 | n1 | n2 | n3 | n4 | n5 | n6 | n7 | s11 | s12 | s13 | d11 | n11 | n12 | n13 | n14 | n15 | n16 | n17 +-------+--------+---------+---------------------+-----+------+-------+-------+-------+-------+-------+-------+--------+---------+---------------------+-----+------+-------+-------+-------+-------+------- + s_1 | s_10 | s_100 | 1973-03-03 09:46:40 | 1 | 10 | 100 | 100 | 100 | 100 | 100 | s_1 | s_10 | s_100 | 1973-03-03 09:46:40 | 1 | 10 | 100 | 100 | 100 | 100 | 100 + s_2 | s_20 | s_200 | 1976-05-03 07:33:20 | 2 | 20 | 200 | 200 | 200 | 200 | 200 | s_2 | s_20 | s_200 | 1976-05-03 07:33:20 | 2 | 20 | 200 | 200 | 200 | 200 | 200 + s_3 | s_30 | s_300 | 1979-07-05 05:20:00 | 3 | 30 | 300 | 300 | 300 | 300 | 300 | s_3 | s_30 | s_300 | 1979-07-05 05:20:00 | 3 | 30 | 300 | 300 | 300 | 300 | 300 + s_4 | s_40 | s_400 | 1982-09-04 03:06:40 | 4 | 40 | 400 | 400 | 400 | 400 | 400 | s_4 | s_40 | s_400 | 1982-09-04 03:06:40 | 4 | 40 | 400 | 400 | 400 | 400 | 400 + s_5 | s_50 | s_500 | 1985-11-05 12:53:20 | 5 | 50 | 500 | 500 | 500 | 500 | 500 | s_5 | s_50 | s_500 | 1985-11-05 12:53:20 | 5 | 50 | 500 | 500 | 500 | 500 | 500 + s_6 | s_60 | s_600 | 1989-01-05 10:40:00 | 6 | 60 | 600 | 600 | 600 | 600 | 600 | s_6 | s_60 | s_600 | 1989-01-05 10:40:00 | 6 | 60 | 600 | 600 | 600 | 600 | 600 + s_7 | s_70 | s_700 | 1992-03-07 08:26:40 | 7 | 70 | 700 | 700 | 700 | 700 | 700 | s_7 | s_70 | s_700 | 1992-03-07 08:26:40 | 7 | 70 | 700 | 700 | 700 | 700 | 700 + s_8 | s_80 | s_800 | 1995-05-09 06:13:20 | 8 | 80 | 800 | 800 | 800 | 800 | 800 | s_8 | s_80 | s_800 | 1995-05-09 06:13:20 | 8 | 80 | 800 | 800 | 800 | 800 | 800 + s_9 | s_90 | s_900 | 1998-07-09 04:00:00 | 9 | 90 | 900 | 900 | 900 | 900 | 900 | s_9 | s_90 | s_900 | 1998-07-09 04:00:00 | 9 | 90 | 900 | 900 | 900 | 900 | 900 + s_10 | s_100 | s_1000 | 2001-09-09 01:46:40 | 10 | 100 | 1000 | 1000 | 1000 | 1000 | 1000 | s_10 | s_100 | s_1000 | 2001-09-09 01:46:40 | 10 | 100 | 1000 | 1000 | 1000 | 1000 | 1000 + s_11 | s_110 | s_1100 | 2004-11-09 11:33:20 | 11 | 110 | 1100 | 1100 | 1100 | 1100 | 1100 | s_11 | s_110 | s_1100 | 2004-11-09 11:33:20 | 11 | 110 | 1100 | 1100 | 1100 | 1100 | 1100 + s_12 | s_120 | s_1200 | 2008-01-10 09:20:00 | 12 | 120 | 1200 | 1200 | 1200 | 1200 | 1200 | s_12 | s_120 | s_1200 | 2008-01-10 09:20:00 | 12 | 120 | 1200 | 1200 | 1200 | 1200 | 1200 + s_13 | s_130 | s_1300 | 2011-03-13 07:06:40 | 13 | 130 | 1300 | 1300 | 1300 | 1300 | 1300 | s_13 | s_130 | s_1300 | 2011-03-13 07:06:40 | 13 | 130 | 1300 | 1300 | 1300 | 1300 | 1300 + s_14 | s_140 | s_1400 | 2014-05-13 04:53:20 | 14 | 140 | 1400 | 1400 | 1400 | 1400 | 1400 | s_14 | s_140 | s_1400 | 2014-05-13 04:53:20 | 14 | 140 | 1400 | 1400 | 1400 | 1400 | 1400 + s_15 | s_150 | s_1500 | 2017-07-14 02:40:00 | 15 | 150 | 1500 | 1500 | 1500 | 1500 | 1500 | s_15 | s_150 | s_1500 | 2017-07-14 02:40:00 | 15 | 150 | 1500 | 1500 | 1500 | 1500 | 1500 + s_16 | s_160 | s_1600 | 2020-09-13 12:26:40 | 16 | 160 | 1600 | 1600 | 1600 | 1600 | 1600 | s_16 | s_160 | s_1600 | 2020-09-13 12:26:40 | 16 | 160 | 1600 | 1600 | 1600 | 1600 | 1600 + s_17 | s_170 | s_1700 | 2023-11-14 10:13:20 | 17 | 170 | 1700 | 1700 | 1700 | 1700 | 1700 | s_17 | s_170 | s_1700 | 2023-11-14 10:13:20 | 17 | 170 | 1700 | 1700 | 1700 | 1700 | 1700 + s_18 | s_180 | s_1800 | 2027-01-15 08:00:00 | 18 | 180 | 1800 | 1800 | 1800 | 1800 | 1800 | s_18 | s_180 | s_1800 | 2027-01-15 08:00:00 | 18 | 180 | 1800 | 1800 | 1800 | 1800 | 1800 + s_19 | s_190 | s_1900 | 2030-03-17 05:46:40 | 19 | 190 | 1900 | 1900 | 1900 | 1900 | 1900 | s_19 | s_190 | s_1900 | 2030-03-17 05:46:40 | 19 | 190 | 1900 | 1900 | 1900 | 1900 | 1900 + s_20 | s_200 | s_2000 | 2033-05-18 03:33:20 | 20 | 200 | 2000 | 2000 | 2000 | 2000 | 2000 | s_20 | s_200 | s_2000 | 2033-05-18 03:33:20 | 20 | 200 | 2000 | 2000 | 2000 | 2000 | 2000 + s_21 | s_210 | s_2100 | 2036-07-18 01:20:00 | 21 | 210 | 2100 | 2100 | 2100 | 2100 | 2100 | s_21 | s_210 | s_2100 | 2036-07-18 01:20:00 | 21 | 210 | 2100 | 2100 | 2100 | 2100 | 2100 + s_22 | s_220 | s_2200 | 2039-09-18 11:06:40 | 22 | 220 | 2200 | 2200 | 2200 | 2200 | 2200 | s_22 | s_220 | s_2200 | 2039-09-18 11:06:40 | 22 | 220 | 2200 | 2200 | 2200 | 2200 | 2200 + s_23 | s_230 | s_2300 | 2042-11-19 08:53:20 | 23 | 230 | 2300 | 2300 | 2300 | 2300 | 2300 | s_23 | s_230 | s_2300 | 2042-11-19 08:53:20 | 23 | 230 | 2300 | 2300 | 2300 | 2300 | 2300 + s_24 | s_240 | s_2400 | 2046-01-19 06:40:00 | 24 | 240 | 2400 | 2400 | 2400 | 2400 | 2400 | s_24 | s_240 | s_2400 | 2046-01-19 06:40:00 | 24 | 240 | 2400 | 2400 | 2400 | 2400 | 2400 + s_25 | s_250 | s_2500 | 2049-03-22 04:26:40 | 25 | 250 | 2500 | 2500 | 2500 | 2500 | 2500 | s_25 | s_250 | s_2500 | 2049-03-22 04:26:40 | 25 | 250 | 2500 | 2500 | 2500 | 2500 | 2500 + s_26 | s_260 | s_2600 | 2052-05-22 02:13:20 | 26 | 260 | 2600 | 2600 | 2600 | 2600 | 2600 | s_26 | s_260 | s_2600 | 2052-05-22 02:13:20 | 26 | 260 | 2600 | 2600 | 2600 | 2600 | 2600 + s_27 | s_270 | s_2700 | 2055-07-24 12:00:00 | 27 | 270 | 2700 | 2700 | 2700 | 2700 | 2700 | s_27 | s_270 | s_2700 | 2055-07-24 12:00:00 | 27 | 270 | 2700 | 2700 | 2700 | 2700 | 2700 + s_28 | s_280 | s_2800 | 2058-09-23 09:46:40 | 28 | 280 | 2800 | 2800 | 2800 | 2800 | 2800 | s_28 | s_280 | s_2800 | 2058-09-23 09:46:40 | 28 | 280 | 2800 | 2800 | 2800 | 2800 | 2800 + s_29 | s_290 | s_2900 | 2061-11-23 07:33:20 | 29 | 290 | 2900 | 2900 | 2900 | 2900 | 2900 | s_29 | s_290 | s_2900 | 2061-11-23 07:33:20 | 29 | 290 | 2900 | 2900 | 2900 | 2900 | 2900 + s_30 | s_300 | s_3000 | 2065-01-24 05:20:00 | 30 | 300 | 3000 | 3000 | 3000 | 3000 | 3000 | s_30 | s_300 | s_3000 | 2065-01-24 05:20:00 | 30 | 300 | 3000 | 3000 | 3000 | 3000 | 3000 + s_31 | s_310 | s_3100 | 2068-03-26 03:06:40 | 31 | 310 | 3100 | 3100 | 3100 | 3100 | 3100 | s_31 | s_310 | s_3100 | 2068-03-26 03:06:40 | 31 | 310 | 3100 | 3100 | 3100 | 3100 | 3100 + s_32 | s_320 | s_3200 | 2071-05-28 12:53:20 | 32 | 320 | 3200 | 3200 | 3200 | 3200 | 3200 | s_32 | s_320 | s_3200 | 2071-05-28 12:53:20 | 32 | 320 | 3200 | 3200 | 3200 | 3200 | 3200 + s_33 | s_330 | s_3300 | 2074-07-28 10:40:00 | 33 | 330 | 3300 | 3300 | 3300 | 3300 | 3300 | s_33 | s_330 | s_3300 | 2074-07-28 10:40:00 | 33 | 330 | 3300 | 3300 | 3300 | 3300 | 3300 + s_34 | s_340 | s_3400 | 2077-09-27 08:26:40 | 34 | 340 | 3400 | 3400 | 3400 | 3400 | 3400 | s_34 | s_340 | s_3400 | 2077-09-27 08:26:40 | 34 | 340 | 3400 | 3400 | 3400 | 3400 | 3400 + s_35 | s_350 | s_3500 | 2080-11-28 06:13:20 | 35 | 350 | 3500 | 3500 | 3500 | 3500 | 3500 | s_35 | s_350 | s_3500 | 2080-11-28 06:13:20 | 35 | 350 | 3500 | 3500 | 3500 | 3500 | 3500 + s_36 | s_360 | s_3600 | 2084-01-29 04:00:00 | 36 | 360 | 3600 | 3600 | 3600 | 3600 | 3600 | s_36 | s_360 | s_3600 | 2084-01-29 04:00:00 | 36 | 360 | 3600 | 3600 | 3600 | 3600 | 3600 + s_37 | s_370 | s_3700 | 2087-04-01 01:46:40 | 37 | 370 | 3700 | 3700 | 3700 | 3700 | 3700 | s_37 | s_370 | s_3700 | 2087-04-01 01:46:40 | 37 | 370 | 3700 | 3700 | 3700 | 3700 | 3700 + s_38 | s_380 | s_3800 | 2090-06-01 11:33:20 | 38 | 380 | 3800 | 3800 | 3800 | 3800 | 3800 | s_38 | s_380 | s_3800 | 2090-06-01 11:33:20 | 38 | 380 | 3800 | 3800 | 3800 | 3800 | 3800 + s_39 | s_390 | s_3900 | 2093-08-01 09:20:00 | 39 | 390 | 3900 | 3900 | 3900 | 3900 | 3900 | s_39 | s_390 | s_3900 | 2093-08-01 09:20:00 | 39 | 390 | 3900 | 3900 | 3900 | 3900 | 3900 + s_40 | s_400 | s_4000 | 2096-10-02 07:06:40 | 40 | 400 | 4000 | 4000 | 4000 | 4000 | 4000 | s_40 | s_400 | s_4000 | 2096-10-02 07:06:40 | 40 | 400 | 4000 | 4000 | 4000 | 4000 | 4000 + s_41 | s_410 | s_4100 | 2099-12-03 04:53:20 | 41 | 410 | 4100 | 4100 | 4100 | 4100 | 4100 | s_41 | s_410 | s_4100 | 2099-12-03 04:53:20 | 41 | 410 | 4100 | 4100 | 4100 | 4100 | 4100 + s_42 | s_420 | s_4200 | 2103-02-04 02:40:00 | 42 | 420 | 4200 | 4200 | 4200 | 4200 | 4200 | s_42 | s_420 | s_4200 | 2103-02-04 02:40:00 | 42 | 420 | 4200 | 4200 | 4200 | 4200 | 4200 + s_43 | s_430 | s_4300 | 2106-04-06 12:26:40 | 43 | 430 | 4300 | 4300 | 4300 | 4300 | 4300 | s_43 | s_430 | s_4300 | 2106-04-06 12:26:40 | 43 | 430 | 4300 | 4300 | 4300 | 4300 | 4300 + s_44 | s_440 | s_4400 | 2109-06-06 10:13:20 | 44 | 440 | 4400 | 4400 | 4400 | 4400 | 4400 | s_44 | s_440 | s_4400 | 2109-06-06 10:13:20 | 44 | 440 | 4400 | 4400 | 4400 | 4400 | 4400 + s_45 | s_450 | s_4500 | 2112-08-07 08:00:00 | 45 | 450 | 4500 | 4500 | 4500 | 4500 | 4500 | s_45 | s_450 | s_4500 | 2112-08-07 08:00:00 | 45 | 450 | 4500 | 4500 | 4500 | 4500 | 4500 + s_46 | s_460 | s_4600 | 2115-10-08 05:46:40 | 46 | 460 | 4600 | 4600 | 4600 | 4600 | 4600 | s_46 | s_460 | s_4600 | 2115-10-08 05:46:40 | 46 | 460 | 4600 | 4600 | 4600 | 4600 | 4600 + s_47 | s_470 | s_4700 | 2118-12-09 03:33:20 | 47 | 470 | 4700 | 4700 | 4700 | 4700 | 4700 | s_47 | s_470 | s_4700 | 2118-12-09 03:33:20 | 47 | 470 | 4700 | 4700 | 4700 | 4700 | 4700 + s_48 | s_480 | s_4800 | 2122-02-08 01:20:00 | 48 | 480 | 4800 | 4800 | 4800 | 4800 | 4800 | s_48 | s_480 | s_4800 | 2122-02-08 01:20:00 | 48 | 480 | 4800 | 4800 | 4800 | 4800 | 4800 + s_49 | s_490 | s_4900 | 2125-04-10 11:06:40 | 49 | 490 | 4900 | 4900 | 4900 | 4900 | 4900 | s_49 | s_490 | s_4900 | 2125-04-10 11:06:40 | 49 | 490 | 4900 | 4900 | 4900 | 4900 | 4900 + s_50 | s_500 | s_5000 | 2128-06-11 08:53:20 | 50 | 500 | 5000 | 5000 | 5000 | 5000 | 5000 | s_50 | s_500 | s_5000 | 2128-06-11 08:53:20 | 50 | 500 | 5000 | 5000 | 5000 | 5000 | 5000 + s_51 | s_510 | s_5100 | 2131-08-12 06:40:00 | 51 | 510 | 5100 | 5100 | 5100 | 5100 | 5100 | s_51 | s_510 | s_5100 | 2131-08-12 06:40:00 | 51 | 510 | 5100 | 5100 | 5100 | 5100 | 5100 + s_52 | s_520 | s_5200 | 2134-10-13 04:26:40 | 52 | 520 | 5200 | 5200 | 5200 | 5200 | 5200 | s_52 | s_520 | s_5200 | 2134-10-13 04:26:40 | 52 | 520 | 5200 | 5200 | 5200 | 5200 | 5200 + s_53 | s_530 | s_5300 | 2137-12-13 02:13:20 | 53 | 530 | 5300 | 5300 | 5300 | 5300 | 5300 | s_53 | s_530 | s_5300 | 2137-12-13 02:13:20 | 53 | 530 | 5300 | 5300 | 5300 | 5300 | 5300 + s_54 | s_540 | s_5400 | 2141-02-13 12:00:00 | 54 | 540 | 5400 | 5400 | 5400 | 5400 | 5400 | s_54 | s_540 | s_5400 | 2141-02-13 12:00:00 | 54 | 540 | 5400 | 5400 | 5400 | 5400 | 5400 + s_55 | s_550 | s_5500 | 2144-04-15 09:46:40 | 55 | 550 | 5500 | 5500 | 5500 | 5500 | 5500 | s_55 | s_550 | s_5500 | 2144-04-15 09:46:40 | 55 | 550 | 5500 | 5500 | 5500 | 5500 | 5500 + s_56 | s_560 | s_5600 | 2147-06-16 07:33:20 | 56 | 560 | 5600 | 5600 | 5600 | 5600 | 5600 | s_56 | s_560 | s_5600 | 2147-06-16 07:33:20 | 56 | 560 | 5600 | 5600 | 5600 | 5600 | 5600 + s_57 | s_570 | s_5700 | 2150-08-17 05:20:00 | 57 | 570 | 5700 | 5700 | 5700 | 5700 | 5700 | s_57 | s_570 | s_5700 | 2150-08-17 05:20:00 | 57 | 570 | 5700 | 5700 | 5700 | 5700 | 5700 + s_58 | s_580 | s_5800 | 2153-10-17 03:06:40 | 58 | 580 | 5800 | 5800 | 5800 | 5800 | 5800 | s_58 | s_580 | s_5800 | 2153-10-17 03:06:40 | 58 | 580 | 5800 | 5800 | 5800 | 5800 | 5800 + s_59 | s_590 | s_5900 | 2156-12-18 12:53:20 | 59 | 590 | 5900 | 5900 | 5900 | 5900 | 5900 | s_59 | s_590 | s_5900 | 2156-12-18 12:53:20 | 59 | 590 | 5900 | 5900 | 5900 | 5900 | 5900 + s_60 | s_600 | s_6000 | 2160-02-18 10:40:00 | 60 | 600 | 6000 | 6000 | 6000 | 6000 | 6000 | s_60 | s_600 | s_6000 | 2160-02-18 10:40:00 | 60 | 600 | 6000 | 6000 | 6000 | 6000 | 6000 + s_61 | s_610 | s_6100 | 2163-04-20 08:26:40 | 61 | 610 | 6100 | 6100 | 6100 | 6100 | 6100 | s_61 | s_610 | s_6100 | 2163-04-20 08:26:40 | 61 | 610 | 6100 | 6100 | 6100 | 6100 | 6100 + s_62 | s_620 | s_6200 | 2166-06-21 06:13:20 | 62 | 620 | 6200 | 6200 | 6200 | 6200 | 6200 | s_62 | s_620 | s_6200 | 2166-06-21 06:13:20 | 62 | 620 | 6200 | 6200 | 6200 | 6200 | 6200 + s_63 | s_630 | s_6300 | 2169-08-21 04:00:00 | 63 | 630 | 6300 | 6300 | 6300 | 6300 | 6300 | s_63 | s_630 | s_6300 | 2169-08-21 04:00:00 | 63 | 630 | 6300 | 6300 | 6300 | 6300 | 6300 + s_64 | s_640 | s_6400 | 2172-10-22 01:46:40 | 64 | 640 | 6400 | 6400 | 6400 | 6400 | 6400 | s_64 | s_640 | s_6400 | 2172-10-22 01:46:40 | 64 | 640 | 6400 | 6400 | 6400 | 6400 | 6400 + s_65 | s_650 | s_6500 | 2175-12-23 11:33:20 | 65 | 650 | 6500 | 6500 | 6500 | 6500 | 6500 | s_65 | s_650 | s_6500 | 2175-12-23 11:33:20 | 65 | 650 | 6500 | 6500 | 6500 | 6500 | 6500 + s_66 | s_660 | s_6600 | 2179-02-22 09:20:00 | 66 | 660 | 6600 | 6600 | 6600 | 6600 | 6600 | s_66 | s_660 | s_6600 | 2179-02-22 09:20:00 | 66 | 660 | 6600 | 6600 | 6600 | 6600 | 6600 + s_67 | s_670 | s_6700 | 2182-04-25 07:06:40 | 67 | 670 | 6700 | 6700 | 6700 | 6700 | 6700 | s_67 | s_670 | s_6700 | 2182-04-25 07:06:40 | 67 | 670 | 6700 | 6700 | 6700 | 6700 | 6700 + s_68 | s_680 | s_6800 | 2185-06-25 04:53:20 | 68 | 680 | 6800 | 6800 | 6800 | 6800 | 6800 | s_68 | s_680 | s_6800 | 2185-06-25 04:53:20 | 68 | 680 | 6800 | 6800 | 6800 | 6800 | 6800 + s_69 | s_690 | s_6900 | 2188-08-26 02:40:00 | 69 | 690 | 6900 | 6900 | 6900 | 6900 | 6900 | s_69 | s_690 | s_6900 | 2188-08-26 02:40:00 | 69 | 690 | 6900 | 6900 | 6900 | 6900 | 6900 + s_70 | s_700 | s_7000 | 2191-10-27 12:26:40 | 70 | 700 | 7000 | 7000 | 7000 | 7000 | 7000 | s_70 | s_700 | s_7000 | 2191-10-27 12:26:40 | 70 | 700 | 7000 | 7000 | 7000 | 7000 | 7000 + s_71 | s_710 | s_7100 | 2194-12-27 10:13:20 | 71 | 710 | 7100 | 7100 | 7100 | 7100 | 7100 | s_71 | s_710 | s_7100 | 2194-12-27 10:13:20 | 71 | 710 | 7100 | 7100 | 7100 | 7100 | 7100 + s_72 | s_720 | s_7200 | 2198-02-27 08:00:00 | 72 | 720 | 7200 | 7200 | 7200 | 7200 | 7200 | s_72 | s_720 | s_7200 | 2198-02-27 08:00:00 | 72 | 720 | 7200 | 7200 | 7200 | 7200 | 7200 + s_73 | s_730 | s_7300 | 2201-04-30 05:46:40 | 73 | 730 | 7300 | 7300 | 7300 | 7300 | 7300 | s_73 | s_730 | s_7300 | 2201-04-30 05:46:40 | 73 | 730 | 7300 | 7300 | 7300 | 7300 | 7300 + s_74 | s_740 | s_7400 | 2204-07-01 03:33:20 | 74 | 740 | 7400 | 7400 | 7400 | 7400 | 7400 | s_74 | s_740 | s_7400 | 2204-07-01 03:33:20 | 74 | 740 | 7400 | 7400 | 7400 | 7400 | 7400 + s_75 | s_750 | s_7500 | 2207-09-01 01:20:00 | 75 | 750 | 7500 | 7500 | 7500 | 7500 | 7500 | s_75 | s_750 | s_7500 | 2207-09-01 01:20:00 | 75 | 750 | 7500 | 7500 | 7500 | 7500 | 7500 + s_76 | s_760 | s_7600 | 2210-11-01 11:06:40 | 76 | 760 | 7600 | 7600 | 7600 | 7600 | 7600 | s_76 | s_760 | s_7600 | 2210-11-01 11:06:40 | 76 | 760 | 7600 | 7600 | 7600 | 7600 | 7600 + s_77 | s_770 | s_7700 | 2214-01-02 08:53:20 | 77 | 770 | 7700 | 7700 | 7700 | 7700 | 7700 | s_77 | s_770 | s_7700 | 2214-01-02 08:53:20 | 77 | 770 | 7700 | 7700 | 7700 | 7700 | 7700 + s_78 | s_780 | s_7800 | 2217-03-04 06:40:00 | 78 | 780 | 7800 | 7800 | 7800 | 7800 | 7800 | s_78 | s_780 | s_7800 | 2217-03-04 06:40:00 | 78 | 780 | 7800 | 7800 | 7800 | 7800 | 7800 + s_79 | s_790 | s_7900 | 2220-05-05 04:26:40 | 79 | 790 | 7900 | 7900 | 7900 | 7900 | 7900 | s_79 | s_790 | s_7900 | 2220-05-05 04:26:40 | 79 | 790 | 7900 | 7900 | 7900 | 7900 | 7900 + s_80 | s_800 | s_8000 | 2223-07-06 02:13:20 | 80 | 800 | 8000 | 8000 | 8000 | 8000 | 8000 | s_80 | s_800 | s_8000 | 2223-07-06 02:13:20 | 80 | 800 | 8000 | 8000 | 8000 | 8000 | 8000 + s_81 | s_810 | s_8100 | 2226-09-06 12:00:00 | 81 | 810 | 8100 | 8100 | 8100 | 8100 | 8100 | s_81 | s_810 | s_8100 | 2226-09-06 12:00:00 | 81 | 810 | 8100 | 8100 | 8100 | 8100 | 8100 + s_82 | s_820 | s_8200 | 2229-11-06 09:46:40 | 82 | 820 | 8200 | 8200 | 8200 | 8200 | 8200 | s_82 | s_820 | s_8200 | 2229-11-06 09:46:40 | 82 | 820 | 8200 | 8200 | 8200 | 8200 | 8200 + s_83 | s_830 | s_8300 | 2233-01-06 07:33:20 | 83 | 830 | 8300 | 8300 | 8300 | 8300 | 8300 | s_83 | s_830 | s_8300 | 2233-01-06 07:33:20 | 83 | 830 | 8300 | 8300 | 8300 | 8300 | 8300 + s_84 | s_840 | s_8400 | 2236-03-09 05:20:00 | 84 | 840 | 8400 | 8400 | 8400 | 8400 | 8400 | s_84 | s_840 | s_8400 | 2236-03-09 05:20:00 | 84 | 840 | 8400 | 8400 | 8400 | 8400 | 8400 + s_85 | s_850 | s_8500 | 2239-05-10 03:06:40 | 85 | 850 | 8500 | 8500 | 8500 | 8500 | 8500 | s_85 | s_850 | s_8500 | 2239-05-10 03:06:40 | 85 | 850 | 8500 | 8500 | 8500 | 8500 | 8500 + s_86 | s_860 | s_8600 | 2242-07-11 12:53:20 | 86 | 860 | 8600 | 8600 | 8600 | 8600 | 8600 | s_86 | s_860 | s_8600 | 2242-07-11 12:53:20 | 86 | 860 | 8600 | 8600 | 8600 | 8600 | 8600 + s_87 | s_870 | s_8700 | 2245-09-10 10:40:00 | 87 | 870 | 8700 | 8700 | 8700 | 8700 | 8700 | s_87 | s_870 | s_8700 | 2245-09-10 10:40:00 | 87 | 870 | 8700 | 8700 | 8700 | 8700 | 8700 + s_88 | s_880 | s_8800 | 2248-11-10 08:26:40 | 88 | 880 | 8800 | 8800 | 8800 | 8800 | 8800 | s_88 | s_880 | s_8800 | 2248-11-10 08:26:40 | 88 | 880 | 8800 | 8800 | 8800 | 8800 | 8800 + s_89 | s_890 | s_8900 | 2252-01-12 06:13:20 | 89 | 890 | 8900 | 8900 | 8900 | 8900 | 8900 | s_89 | s_890 | s_8900 | 2252-01-12 06:13:20 | 89 | 890 | 8900 | 8900 | 8900 | 8900 | 8900 + s_90 | s_900 | s_9000 | 2255-03-14 04:00:00 | 90 | 900 | 9000 | 9000 | 9000 | 9000 | 9000 | s_90 | s_900 | s_9000 | 2255-03-14 04:00:00 | 90 | 900 | 9000 | 9000 | 9000 | 9000 | 9000 + s_91 | s_910 | s_9100 | 2258-05-15 01:46:40 | 91 | 910 | 9100 | 9100 | 9100 | 9100 | 9100 | s_91 | s_910 | s_9100 | 2258-05-15 01:46:40 | 91 | 910 | 9100 | 9100 | 9100 | 9100 | 9100 + s_92 | s_920 | s_9200 | 2261-07-15 11:33:20 | 92 | 920 | 9200 | 9200 | 9200 | 9200 | 9200 | s_92 | s_920 | s_9200 | 2261-07-15 11:33:20 | 92 | 920 | 9200 | 9200 | 9200 | 9200 | 9200 + s_93 | s_930 | s_9300 | 2264-09-14 09:20:00 | 93 | 930 | 9300 | 9300 | 9300 | 9300 | 9300 | s_93 | s_930 | s_9300 | 2264-09-14 09:20:00 | 93 | 930 | 9300 | 9300 | 9300 | 9300 | 9300 + s_94 | s_940 | s_9400 | 2267-11-16 07:06:40 | 94 | 940 | 9400 | 9400 | 9400 | 9400 | 9400 | s_94 | s_940 | s_9400 | 2267-11-16 07:06:40 | 94 | 940 | 9400 | 9400 | 9400 | 9400 | 9400 + s_95 | s_950 | s_9500 | 2271-01-16 04:53:20 | 95 | 950 | 9500 | 9500 | 9500 | 9500 | 9500 | s_95 | s_950 | s_9500 | 2271-01-16 04:53:20 | 95 | 950 | 9500 | 9500 | 9500 | 9500 | 9500 + s_96 | s_960 | s_9600 | 2274-03-19 02:40:00 | 96 | 960 | 9600 | 9600 | 9600 | 9600 | 9600 | s_96 | s_960 | s_9600 | 2274-03-19 02:40:00 | 96 | 960 | 9600 | 9600 | 9600 | 9600 | 9600 + s_97 | s_970 | s_9700 | 2277-05-19 12:26:40 | 97 | 970 | 9700 | 9700 | 9700 | 9700 | 9700 | s_97 | s_970 | s_9700 | 2277-05-19 12:26:40 | 97 | 970 | 9700 | 9700 | 9700 | 9700 | 9700 + s_98 | s_980 | s_9800 | 2280-07-19 10:13:20 | 98 | 980 | 9800 | 9800 | 9800 | 9800 | 9800 | s_98 | s_980 | s_9800 | 2280-07-19 10:13:20 | 98 | 980 | 9800 | 9800 | 9800 | 9800 | 9800 + s_99 | s_990 | s_9900 | 2283-09-20 08:00:00 | 99 | 990 | 9900 | 9900 | 9900 | 9900 | 9900 | s_99 | s_990 | s_9900 | 2283-09-20 08:00:00 | 99 | 990 | 9900 | 9900 | 9900 | 9900 | 9900 + s_100 | s_1000 | s_10000 | 2286-11-20 05:46:40 | 100 | 1000 | 10000 | 10000 | 10000 | 10000 | 10000 | s_100 | s_1000 | s_10000 | 2286-11-20 05:46:40 | 100 | 1000 | 10000 | 10000 | 10000 | 10000 | 10000 +(100 rows) + +SELECT * from pxf_multibyte_threebyte_data_with_skip ORDER BY n1; + s1 | s2 | s3 | d1 | n1 | n2 | n3 | n4 | n5 | n6 | n7 | s11 | s12 | s13 | d11 | n11 | n12 | n13 | n14 | n15 | n16 | n17 +-------+--------+---------+---------------------+-----+------+-------+-------+-------+-------+-------+-------+--------+---------+---------------------+-----+------+-------+-------+-------+-------+------- + s_11 | s_110 | s_1100 | 2004-11-09 11:33:20 | 11 | 110 | 1100 | 1100 | 1100 | 1100 | 1100 | s_11 | s_110 | s_1100 | 2004-11-09 11:33:20 | 11 | 110 | 1100 | 1100 | 1100 | 1100 | 1100 + s_12 | s_120 | s_1200 | 2008-01-10 09:20:00 | 12 | 120 | 1200 | 1200 | 1200 | 1200 | 1200 | s_12 | s_120 | s_1200 | 2008-01-10 09:20:00 | 12 | 120 | 1200 | 1200 | 1200 | 1200 | 1200 + s_13 | s_130 | s_1300 | 2011-03-13 07:06:40 | 13 | 130 | 1300 | 1300 | 1300 | 1300 | 1300 | s_13 | s_130 | s_1300 | 2011-03-13 07:06:40 | 13 | 130 | 1300 | 1300 | 1300 | 1300 | 1300 + s_14 | s_140 | s_1400 | 2014-05-13 04:53:20 | 14 | 140 | 1400 | 1400 | 1400 | 1400 | 1400 | s_14 | s_140 | s_1400 | 2014-05-13 04:53:20 | 14 | 140 | 1400 | 1400 | 1400 | 1400 | 1400 + s_15 | s_150 | s_1500 | 2017-07-14 02:40:00 | 15 | 150 | 1500 | 1500 | 1500 | 1500 | 1500 | s_15 | s_150 | s_1500 | 2017-07-14 02:40:00 | 15 | 150 | 1500 | 1500 | 1500 | 1500 | 1500 + s_16 | s_160 | s_1600 | 2020-09-13 12:26:40 | 16 | 160 | 1600 | 1600 | 1600 | 1600 | 1600 | s_16 | s_160 | s_1600 | 2020-09-13 12:26:40 | 16 | 160 | 1600 | 1600 | 1600 | 1600 | 1600 + s_17 | s_170 | s_1700 | 2023-11-14 10:13:20 | 17 | 170 | 1700 | 1700 | 1700 | 1700 | 1700 | s_17 | s_170 | s_1700 | 2023-11-14 10:13:20 | 17 | 170 | 1700 | 1700 | 1700 | 1700 | 1700 + s_18 | s_180 | s_1800 | 2027-01-15 08:00:00 | 18 | 180 | 1800 | 1800 | 1800 | 1800 | 1800 | s_18 | s_180 | s_1800 | 2027-01-15 08:00:00 | 18 | 180 | 1800 | 1800 | 1800 | 1800 | 1800 + s_19 | s_190 | s_1900 | 2030-03-17 05:46:40 | 19 | 190 | 1900 | 1900 | 1900 | 1900 | 1900 | s_19 | s_190 | s_1900 | 2030-03-17 05:46:40 | 19 | 190 | 1900 | 1900 | 1900 | 1900 | 1900 + s_20 | s_200 | s_2000 | 2033-05-18 03:33:20 | 20 | 200 | 2000 | 2000 | 2000 | 2000 | 2000 | s_20 | s_200 | s_2000 | 2033-05-18 03:33:20 | 20 | 200 | 2000 | 2000 | 2000 | 2000 | 2000 + s_21 | s_210 | s_2100 | 2036-07-18 01:20:00 | 21 | 210 | 2100 | 2100 | 2100 | 2100 | 2100 | s_21 | s_210 | s_2100 | 2036-07-18 01:20:00 | 21 | 210 | 2100 | 2100 | 2100 | 2100 | 2100 + s_22 | s_220 | s_2200 | 2039-09-18 11:06:40 | 22 | 220 | 2200 | 2200 | 2200 | 2200 | 2200 | s_22 | s_220 | s_2200 | 2039-09-18 11:06:40 | 22 | 220 | 2200 | 2200 | 2200 | 2200 | 2200 + s_23 | s_230 | s_2300 | 2042-11-19 08:53:20 | 23 | 230 | 2300 | 2300 | 2300 | 2300 | 2300 | s_23 | s_230 | s_2300 | 2042-11-19 08:53:20 | 23 | 230 | 2300 | 2300 | 2300 | 2300 | 2300 + s_24 | s_240 | s_2400 | 2046-01-19 06:40:00 | 24 | 240 | 2400 | 2400 | 2400 | 2400 | 2400 | s_24 | s_240 | s_2400 | 2046-01-19 06:40:00 | 24 | 240 | 2400 | 2400 | 2400 | 2400 | 2400 + s_25 | s_250 | s_2500 | 2049-03-22 04:26:40 | 25 | 250 | 2500 | 2500 | 2500 | 2500 | 2500 | s_25 | s_250 | s_2500 | 2049-03-22 04:26:40 | 25 | 250 | 2500 | 2500 | 2500 | 2500 | 2500 + s_26 | s_260 | s_2600 | 2052-05-22 02:13:20 | 26 | 260 | 2600 | 2600 | 2600 | 2600 | 2600 | s_26 | s_260 | s_2600 | 2052-05-22 02:13:20 | 26 | 260 | 2600 | 2600 | 2600 | 2600 | 2600 + s_27 | s_270 | s_2700 | 2055-07-24 12:00:00 | 27 | 270 | 2700 | 2700 | 2700 | 2700 | 2700 | s_27 | s_270 | s_2700 | 2055-07-24 12:00:00 | 27 | 270 | 2700 | 2700 | 2700 | 2700 | 2700 + s_28 | s_280 | s_2800 | 2058-09-23 09:46:40 | 28 | 280 | 2800 | 2800 | 2800 | 2800 | 2800 | s_28 | s_280 | s_2800 | 2058-09-23 09:46:40 | 28 | 280 | 2800 | 2800 | 2800 | 2800 | 2800 + s_29 | s_290 | s_2900 | 2061-11-23 07:33:20 | 29 | 290 | 2900 | 2900 | 2900 | 2900 | 2900 | s_29 | s_290 | s_2900 | 2061-11-23 07:33:20 | 29 | 290 | 2900 | 2900 | 2900 | 2900 | 2900 + s_30 | s_300 | s_3000 | 2065-01-24 05:20:00 | 30 | 300 | 3000 | 3000 | 3000 | 3000 | 3000 | s_30 | s_300 | s_3000 | 2065-01-24 05:20:00 | 30 | 300 | 3000 | 3000 | 3000 | 3000 | 3000 + s_31 | s_310 | s_3100 | 2068-03-26 03:06:40 | 31 | 310 | 3100 | 3100 | 3100 | 3100 | 3100 | s_31 | s_310 | s_3100 | 2068-03-26 03:06:40 | 31 | 310 | 3100 | 3100 | 3100 | 3100 | 3100 + s_32 | s_320 | s_3200 | 2071-05-28 12:53:20 | 32 | 320 | 3200 | 3200 | 3200 | 3200 | 3200 | s_32 | s_320 | s_3200 | 2071-05-28 12:53:20 | 32 | 320 | 3200 | 3200 | 3200 | 3200 | 3200 + s_33 | s_330 | s_3300 | 2074-07-28 10:40:00 | 33 | 330 | 3300 | 3300 | 3300 | 3300 | 3300 | s_33 | s_330 | s_3300 | 2074-07-28 10:40:00 | 33 | 330 | 3300 | 3300 | 3300 | 3300 | 3300 + s_34 | s_340 | s_3400 | 2077-09-27 08:26:40 | 34 | 340 | 3400 | 3400 | 3400 | 3400 | 3400 | s_34 | s_340 | s_3400 | 2077-09-27 08:26:40 | 34 | 340 | 3400 | 3400 | 3400 | 3400 | 3400 + s_35 | s_350 | s_3500 | 2080-11-28 06:13:20 | 35 | 350 | 3500 | 3500 | 3500 | 3500 | 3500 | s_35 | s_350 | s_3500 | 2080-11-28 06:13:20 | 35 | 350 | 3500 | 3500 | 3500 | 3500 | 3500 + s_36 | s_360 | s_3600 | 2084-01-29 04:00:00 | 36 | 360 | 3600 | 3600 | 3600 | 3600 | 3600 | s_36 | s_360 | s_3600 | 2084-01-29 04:00:00 | 36 | 360 | 3600 | 3600 | 3600 | 3600 | 3600 + s_37 | s_370 | s_3700 | 2087-04-01 01:46:40 | 37 | 370 | 3700 | 3700 | 3700 | 3700 | 3700 | s_37 | s_370 | s_3700 | 2087-04-01 01:46:40 | 37 | 370 | 3700 | 3700 | 3700 | 3700 | 3700 + s_38 | s_380 | s_3800 | 2090-06-01 11:33:20 | 38 | 380 | 3800 | 3800 | 3800 | 3800 | 3800 | s_38 | s_380 | s_3800 | 2090-06-01 11:33:20 | 38 | 380 | 3800 | 3800 | 3800 | 3800 | 3800 + s_39 | s_390 | s_3900 | 2093-08-01 09:20:00 | 39 | 390 | 3900 | 3900 | 3900 | 3900 | 3900 | s_39 | s_390 | s_3900 | 2093-08-01 09:20:00 | 39 | 390 | 3900 | 3900 | 3900 | 3900 | 3900 + s_40 | s_400 | s_4000 | 2096-10-02 07:06:40 | 40 | 400 | 4000 | 4000 | 4000 | 4000 | 4000 | s_40 | s_400 | s_4000 | 2096-10-02 07:06:40 | 40 | 400 | 4000 | 4000 | 4000 | 4000 | 4000 + s_41 | s_410 | s_4100 | 2099-12-03 04:53:20 | 41 | 410 | 4100 | 4100 | 4100 | 4100 | 4100 | s_41 | s_410 | s_4100 | 2099-12-03 04:53:20 | 41 | 410 | 4100 | 4100 | 4100 | 4100 | 4100 + s_42 | s_420 | s_4200 | 2103-02-04 02:40:00 | 42 | 420 | 4200 | 4200 | 4200 | 4200 | 4200 | s_42 | s_420 | s_4200 | 2103-02-04 02:40:00 | 42 | 420 | 4200 | 4200 | 4200 | 4200 | 4200 + s_43 | s_430 | s_4300 | 2106-04-06 12:26:40 | 43 | 430 | 4300 | 4300 | 4300 | 4300 | 4300 | s_43 | s_430 | s_4300 | 2106-04-06 12:26:40 | 43 | 430 | 4300 | 4300 | 4300 | 4300 | 4300 + s_44 | s_440 | s_4400 | 2109-06-06 10:13:20 | 44 | 440 | 4400 | 4400 | 4400 | 4400 | 4400 | s_44 | s_440 | s_4400 | 2109-06-06 10:13:20 | 44 | 440 | 4400 | 4400 | 4400 | 4400 | 4400 + s_45 | s_450 | s_4500 | 2112-08-07 08:00:00 | 45 | 450 | 4500 | 4500 | 4500 | 4500 | 4500 | s_45 | s_450 | s_4500 | 2112-08-07 08:00:00 | 45 | 450 | 4500 | 4500 | 4500 | 4500 | 4500 + s_46 | s_460 | s_4600 | 2115-10-08 05:46:40 | 46 | 460 | 4600 | 4600 | 4600 | 4600 | 4600 | s_46 | s_460 | s_4600 | 2115-10-08 05:46:40 | 46 | 460 | 4600 | 4600 | 4600 | 4600 | 4600 + s_47 | s_470 | s_4700 | 2118-12-09 03:33:20 | 47 | 470 | 4700 | 4700 | 4700 | 4700 | 4700 | s_47 | s_470 | s_4700 | 2118-12-09 03:33:20 | 47 | 470 | 4700 | 4700 | 4700 | 4700 | 4700 + s_48 | s_480 | s_4800 | 2122-02-08 01:20:00 | 48 | 480 | 4800 | 4800 | 4800 | 4800 | 4800 | s_48 | s_480 | s_4800 | 2122-02-08 01:20:00 | 48 | 480 | 4800 | 4800 | 4800 | 4800 | 4800 + s_49 | s_490 | s_4900 | 2125-04-10 11:06:40 | 49 | 490 | 4900 | 4900 | 4900 | 4900 | 4900 | s_49 | s_490 | s_4900 | 2125-04-10 11:06:40 | 49 | 490 | 4900 | 4900 | 4900 | 4900 | 4900 + s_50 | s_500 | s_5000 | 2128-06-11 08:53:20 | 50 | 500 | 5000 | 5000 | 5000 | 5000 | 5000 | s_50 | s_500 | s_5000 | 2128-06-11 08:53:20 | 50 | 500 | 5000 | 5000 | 5000 | 5000 | 5000 + s_51 | s_510 | s_5100 | 2131-08-12 06:40:00 | 51 | 510 | 5100 | 5100 | 5100 | 5100 | 5100 | s_51 | s_510 | s_5100 | 2131-08-12 06:40:00 | 51 | 510 | 5100 | 5100 | 5100 | 5100 | 5100 + s_52 | s_520 | s_5200 | 2134-10-13 04:26:40 | 52 | 520 | 5200 | 5200 | 5200 | 5200 | 5200 | s_52 | s_520 | s_5200 | 2134-10-13 04:26:40 | 52 | 520 | 5200 | 5200 | 5200 | 5200 | 5200 + s_53 | s_530 | s_5300 | 2137-12-13 02:13:20 | 53 | 530 | 5300 | 5300 | 5300 | 5300 | 5300 | s_53 | s_530 | s_5300 | 2137-12-13 02:13:20 | 53 | 530 | 5300 | 5300 | 5300 | 5300 | 5300 + s_54 | s_540 | s_5400 | 2141-02-13 12:00:00 | 54 | 540 | 5400 | 5400 | 5400 | 5400 | 5400 | s_54 | s_540 | s_5400 | 2141-02-13 12:00:00 | 54 | 540 | 5400 | 5400 | 5400 | 5400 | 5400 + s_55 | s_550 | s_5500 | 2144-04-15 09:46:40 | 55 | 550 | 5500 | 5500 | 5500 | 5500 | 5500 | s_55 | s_550 | s_5500 | 2144-04-15 09:46:40 | 55 | 550 | 5500 | 5500 | 5500 | 5500 | 5500 + s_56 | s_560 | s_5600 | 2147-06-16 07:33:20 | 56 | 560 | 5600 | 5600 | 5600 | 5600 | 5600 | s_56 | s_560 | s_5600 | 2147-06-16 07:33:20 | 56 | 560 | 5600 | 5600 | 5600 | 5600 | 5600 + s_57 | s_570 | s_5700 | 2150-08-17 05:20:00 | 57 | 570 | 5700 | 5700 | 5700 | 5700 | 5700 | s_57 | s_570 | s_5700 | 2150-08-17 05:20:00 | 57 | 570 | 5700 | 5700 | 5700 | 5700 | 5700 + s_58 | s_580 | s_5800 | 2153-10-17 03:06:40 | 58 | 580 | 5800 | 5800 | 5800 | 5800 | 5800 | s_58 | s_580 | s_5800 | 2153-10-17 03:06:40 | 58 | 580 | 5800 | 5800 | 5800 | 5800 | 5800 + s_59 | s_590 | s_5900 | 2156-12-18 12:53:20 | 59 | 590 | 5900 | 5900 | 5900 | 5900 | 5900 | s_59 | s_590 | s_5900 | 2156-12-18 12:53:20 | 59 | 590 | 5900 | 5900 | 5900 | 5900 | 5900 + s_60 | s_600 | s_6000 | 2160-02-18 10:40:00 | 60 | 600 | 6000 | 6000 | 6000 | 6000 | 6000 | s_60 | s_600 | s_6000 | 2160-02-18 10:40:00 | 60 | 600 | 6000 | 6000 | 6000 | 6000 | 6000 + s_61 | s_610 | s_6100 | 2163-04-20 08:26:40 | 61 | 610 | 6100 | 6100 | 6100 | 6100 | 6100 | s_61 | s_610 | s_6100 | 2163-04-20 08:26:40 | 61 | 610 | 6100 | 6100 | 6100 | 6100 | 6100 + s_62 | s_620 | s_6200 | 2166-06-21 06:13:20 | 62 | 620 | 6200 | 6200 | 6200 | 6200 | 6200 | s_62 | s_620 | s_6200 | 2166-06-21 06:13:20 | 62 | 620 | 6200 | 6200 | 6200 | 6200 | 6200 + s_63 | s_630 | s_6300 | 2169-08-21 04:00:00 | 63 | 630 | 6300 | 6300 | 6300 | 6300 | 6300 | s_63 | s_630 | s_6300 | 2169-08-21 04:00:00 | 63 | 630 | 6300 | 6300 | 6300 | 6300 | 6300 + s_64 | s_640 | s_6400 | 2172-10-22 01:46:40 | 64 | 640 | 6400 | 6400 | 6400 | 6400 | 6400 | s_64 | s_640 | s_6400 | 2172-10-22 01:46:40 | 64 | 640 | 6400 | 6400 | 6400 | 6400 | 6400 + s_65 | s_650 | s_6500 | 2175-12-23 11:33:20 | 65 | 650 | 6500 | 6500 | 6500 | 6500 | 6500 | s_65 | s_650 | s_6500 | 2175-12-23 11:33:20 | 65 | 650 | 6500 | 6500 | 6500 | 6500 | 6500 + s_66 | s_660 | s_6600 | 2179-02-22 09:20:00 | 66 | 660 | 6600 | 6600 | 6600 | 6600 | 6600 | s_66 | s_660 | s_6600 | 2179-02-22 09:20:00 | 66 | 660 | 6600 | 6600 | 6600 | 6600 | 6600 + s_67 | s_670 | s_6700 | 2182-04-25 07:06:40 | 67 | 670 | 6700 | 6700 | 6700 | 6700 | 6700 | s_67 | s_670 | s_6700 | 2182-04-25 07:06:40 | 67 | 670 | 6700 | 6700 | 6700 | 6700 | 6700 + s_68 | s_680 | s_6800 | 2185-06-25 04:53:20 | 68 | 680 | 6800 | 6800 | 6800 | 6800 | 6800 | s_68 | s_680 | s_6800 | 2185-06-25 04:53:20 | 68 | 680 | 6800 | 6800 | 6800 | 6800 | 6800 + s_69 | s_690 | s_6900 | 2188-08-26 02:40:00 | 69 | 690 | 6900 | 6900 | 6900 | 6900 | 6900 | s_69 | s_690 | s_6900 | 2188-08-26 02:40:00 | 69 | 690 | 6900 | 6900 | 6900 | 6900 | 6900 + s_70 | s_700 | s_7000 | 2191-10-27 12:26:40 | 70 | 700 | 7000 | 7000 | 7000 | 7000 | 7000 | s_70 | s_700 | s_7000 | 2191-10-27 12:26:40 | 70 | 700 | 7000 | 7000 | 7000 | 7000 | 7000 + s_71 | s_710 | s_7100 | 2194-12-27 10:13:20 | 71 | 710 | 7100 | 7100 | 7100 | 7100 | 7100 | s_71 | s_710 | s_7100 | 2194-12-27 10:13:20 | 71 | 710 | 7100 | 7100 | 7100 | 7100 | 7100 + s_72 | s_720 | s_7200 | 2198-02-27 08:00:00 | 72 | 720 | 7200 | 7200 | 7200 | 7200 | 7200 | s_72 | s_720 | s_7200 | 2198-02-27 08:00:00 | 72 | 720 | 7200 | 7200 | 7200 | 7200 | 7200 + s_73 | s_730 | s_7300 | 2201-04-30 05:46:40 | 73 | 730 | 7300 | 7300 | 7300 | 7300 | 7300 | s_73 | s_730 | s_7300 | 2201-04-30 05:46:40 | 73 | 730 | 7300 | 7300 | 7300 | 7300 | 7300 + s_74 | s_740 | s_7400 | 2204-07-01 03:33:20 | 74 | 740 | 7400 | 7400 | 7400 | 7400 | 7400 | s_74 | s_740 | s_7400 | 2204-07-01 03:33:20 | 74 | 740 | 7400 | 7400 | 7400 | 7400 | 7400 + s_75 | s_750 | s_7500 | 2207-09-01 01:20:00 | 75 | 750 | 7500 | 7500 | 7500 | 7500 | 7500 | s_75 | s_750 | s_7500 | 2207-09-01 01:20:00 | 75 | 750 | 7500 | 7500 | 7500 | 7500 | 7500 + s_76 | s_760 | s_7600 | 2210-11-01 11:06:40 | 76 | 760 | 7600 | 7600 | 7600 | 7600 | 7600 | s_76 | s_760 | s_7600 | 2210-11-01 11:06:40 | 76 | 760 | 7600 | 7600 | 7600 | 7600 | 7600 + s_77 | s_770 | s_7700 | 2214-01-02 08:53:20 | 77 | 770 | 7700 | 7700 | 7700 | 7700 | 7700 | s_77 | s_770 | s_7700 | 2214-01-02 08:53:20 | 77 | 770 | 7700 | 7700 | 7700 | 7700 | 7700 + s_78 | s_780 | s_7800 | 2217-03-04 06:40:00 | 78 | 780 | 7800 | 7800 | 7800 | 7800 | 7800 | s_78 | s_780 | s_7800 | 2217-03-04 06:40:00 | 78 | 780 | 7800 | 7800 | 7800 | 7800 | 7800 + s_79 | s_790 | s_7900 | 2220-05-05 04:26:40 | 79 | 790 | 7900 | 7900 | 7900 | 7900 | 7900 | s_79 | s_790 | s_7900 | 2220-05-05 04:26:40 | 79 | 790 | 7900 | 7900 | 7900 | 7900 | 7900 + s_80 | s_800 | s_8000 | 2223-07-06 02:13:20 | 80 | 800 | 8000 | 8000 | 8000 | 8000 | 8000 | s_80 | s_800 | s_8000 | 2223-07-06 02:13:20 | 80 | 800 | 8000 | 8000 | 8000 | 8000 | 8000 + s_81 | s_810 | s_8100 | 2226-09-06 12:00:00 | 81 | 810 | 8100 | 8100 | 8100 | 8100 | 8100 | s_81 | s_810 | s_8100 | 2226-09-06 12:00:00 | 81 | 810 | 8100 | 8100 | 8100 | 8100 | 8100 + s_82 | s_820 | s_8200 | 2229-11-06 09:46:40 | 82 | 820 | 8200 | 8200 | 8200 | 8200 | 8200 | s_82 | s_820 | s_8200 | 2229-11-06 09:46:40 | 82 | 820 | 8200 | 8200 | 8200 | 8200 | 8200 + s_83 | s_830 | s_8300 | 2233-01-06 07:33:20 | 83 | 830 | 8300 | 8300 | 8300 | 8300 | 8300 | s_83 | s_830 | s_8300 | 2233-01-06 07:33:20 | 83 | 830 | 8300 | 8300 | 8300 | 8300 | 8300 + s_84 | s_840 | s_8400 | 2236-03-09 05:20:00 | 84 | 840 | 8400 | 8400 | 8400 | 8400 | 8400 | s_84 | s_840 | s_8400 | 2236-03-09 05:20:00 | 84 | 840 | 8400 | 8400 | 8400 | 8400 | 8400 + s_85 | s_850 | s_8500 | 2239-05-10 03:06:40 | 85 | 850 | 8500 | 8500 | 8500 | 8500 | 8500 | s_85 | s_850 | s_8500 | 2239-05-10 03:06:40 | 85 | 850 | 8500 | 8500 | 8500 | 8500 | 8500 + s_86 | s_860 | s_8600 | 2242-07-11 12:53:20 | 86 | 860 | 8600 | 8600 | 8600 | 8600 | 8600 | s_86 | s_860 | s_8600 | 2242-07-11 12:53:20 | 86 | 860 | 8600 | 8600 | 8600 | 8600 | 8600 + s_87 | s_870 | s_8700 | 2245-09-10 10:40:00 | 87 | 870 | 8700 | 8700 | 8700 | 8700 | 8700 | s_87 | s_870 | s_8700 | 2245-09-10 10:40:00 | 87 | 870 | 8700 | 8700 | 8700 | 8700 | 8700 + s_88 | s_880 | s_8800 | 2248-11-10 08:26:40 | 88 | 880 | 8800 | 8800 | 8800 | 8800 | 8800 | s_88 | s_880 | s_8800 | 2248-11-10 08:26:40 | 88 | 880 | 8800 | 8800 | 8800 | 8800 | 8800 + s_89 | s_890 | s_8900 | 2252-01-12 06:13:20 | 89 | 890 | 8900 | 8900 | 8900 | 8900 | 8900 | s_89 | s_890 | s_8900 | 2252-01-12 06:13:20 | 89 | 890 | 8900 | 8900 | 8900 | 8900 | 8900 + s_90 | s_900 | s_9000 | 2255-03-14 04:00:00 | 90 | 900 | 9000 | 9000 | 9000 | 9000 | 9000 | s_90 | s_900 | s_9000 | 2255-03-14 04:00:00 | 90 | 900 | 9000 | 9000 | 9000 | 9000 | 9000 + s_91 | s_910 | s_9100 | 2258-05-15 01:46:40 | 91 | 910 | 9100 | 9100 | 9100 | 9100 | 9100 | s_91 | s_910 | s_9100 | 2258-05-15 01:46:40 | 91 | 910 | 9100 | 9100 | 9100 | 9100 | 9100 + s_92 | s_920 | s_9200 | 2261-07-15 11:33:20 | 92 | 920 | 9200 | 9200 | 9200 | 9200 | 9200 | s_92 | s_920 | s_9200 | 2261-07-15 11:33:20 | 92 | 920 | 9200 | 9200 | 9200 | 9200 | 9200 + s_93 | s_930 | s_9300 | 2264-09-14 09:20:00 | 93 | 930 | 9300 | 9300 | 9300 | 9300 | 9300 | s_93 | s_930 | s_9300 | 2264-09-14 09:20:00 | 93 | 930 | 9300 | 9300 | 9300 | 9300 | 9300 + s_94 | s_940 | s_9400 | 2267-11-16 07:06:40 | 94 | 940 | 9400 | 9400 | 9400 | 9400 | 9400 | s_94 | s_940 | s_9400 | 2267-11-16 07:06:40 | 94 | 940 | 9400 | 9400 | 9400 | 9400 | 9400 + s_95 | s_950 | s_9500 | 2271-01-16 04:53:20 | 95 | 950 | 9500 | 9500 | 9500 | 9500 | 9500 | s_95 | s_950 | s_9500 | 2271-01-16 04:53:20 | 95 | 950 | 9500 | 9500 | 9500 | 9500 | 9500 + s_96 | s_960 | s_9600 | 2274-03-19 02:40:00 | 96 | 960 | 9600 | 9600 | 9600 | 9600 | 9600 | s_96 | s_960 | s_9600 | 2274-03-19 02:40:00 | 96 | 960 | 9600 | 9600 | 9600 | 9600 | 9600 + s_97 | s_970 | s_9700 | 2277-05-19 12:26:40 | 97 | 970 | 9700 | 9700 | 9700 | 9700 | 9700 | s_97 | s_970 | s_9700 | 2277-05-19 12:26:40 | 97 | 970 | 9700 | 9700 | 9700 | 9700 | 9700 + s_98 | s_980 | s_9800 | 2280-07-19 10:13:20 | 98 | 980 | 9800 | 9800 | 9800 | 9800 | 9800 | s_98 | s_980 | s_9800 | 2280-07-19 10:13:20 | 98 | 980 | 9800 | 9800 | 9800 | 9800 | 9800 + s_99 | s_990 | s_9900 | 2283-09-20 08:00:00 | 99 | 990 | 9900 | 9900 | 9900 | 9900 | 9900 | s_99 | s_990 | s_9900 | 2283-09-20 08:00:00 | 99 | 990 | 9900 | 9900 | 9900 | 9900 | 9900 + s_100 | s_1000 | s_10000 | 2286-11-20 05:46:40 | 100 | 1000 | 10000 | 10000 | 10000 | 10000 | 10000 | s_100 | s_1000 | s_10000 | 2286-11-20 05:46:40 | 100 | 1000 | 10000 | 10000 | 10000 | 10000 | 10000 +(90 rows) \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/three_byte/runTest.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/three_byte/runTest.py new file mode 100755 index 0000000000..17a06ae190 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/three_byte/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfMultibyteThreeByteDelimiterData(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/three_byte/sql/query01.sql b/automation/tincrepo/main/pxf/features/multibyte_delimiter/three_byte/sql/query01.sql new file mode 100755 index 0000000000..58cd12fea7 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/three_byte/sql/query01.sql @@ -0,0 +1,5 @@ +-- @description query01 for PXF Multibyte delimiter, 3-byte delim cases + +SELECT * from pxf_multibyte_threebyte_data ORDER BY n1; + +SELECT * from pxf_multibyte_threebyte_data_with_skip ORDER BY n1; \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte/__init__.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte/expected/query01.ans b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte/expected/query01.ans new file mode 100755 index 0000000000..eb79289767 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte/expected/query01.ans @@ -0,0 +1,201 @@ +-- @description query01 for PXF Multibyte delimiter, 2-byte delim cases + +SELECT * from pxf_multibyte_twobyte_data ORDER BY n1; + s1 | s2 | s3 | d1 | n1 | n2 | n3 | n4 | n5 | n6 | n7 | s11 | s12 | s13 | d11 | n11 | n12 | n13 | n14 | n15 | n16 | n17 +-------+--------+---------+---------------------+-----+------+-------+-------+-------+-------+-------+-------+--------+---------+---------------------+-----+------+-------+-------+-------+-------+------- + s_1 | s_10 | s_100 | 1973-03-03 09:46:40 | 1 | 10 | 100 | 100 | 100 | 100 | 100 | s_1 | s_10 | s_100 | 1973-03-03 09:46:40 | 1 | 10 | 100 | 100 | 100 | 100 | 100 + s_2 | s_20 | s_200 | 1976-05-03 07:33:20 | 2 | 20 | 200 | 200 | 200 | 200 | 200 | s_2 | s_20 | s_200 | 1976-05-03 07:33:20 | 2 | 20 | 200 | 200 | 200 | 200 | 200 + s_3 | s_30 | s_300 | 1979-07-05 05:20:00 | 3 | 30 | 300 | 300 | 300 | 300 | 300 | s_3 | s_30 | s_300 | 1979-07-05 05:20:00 | 3 | 30 | 300 | 300 | 300 | 300 | 300 + s_4 | s_40 | s_400 | 1982-09-04 03:06:40 | 4 | 40 | 400 | 400 | 400 | 400 | 400 | s_4 | s_40 | s_400 | 1982-09-04 03:06:40 | 4 | 40 | 400 | 400 | 400 | 400 | 400 + s_5 | s_50 | s_500 | 1985-11-05 12:53:20 | 5 | 50 | 500 | 500 | 500 | 500 | 500 | s_5 | s_50 | s_500 | 1985-11-05 12:53:20 | 5 | 50 | 500 | 500 | 500 | 500 | 500 + s_6 | s_60 | s_600 | 1989-01-05 10:40:00 | 6 | 60 | 600 | 600 | 600 | 600 | 600 | s_6 | s_60 | s_600 | 1989-01-05 10:40:00 | 6 | 60 | 600 | 600 | 600 | 600 | 600 + s_7 | s_70 | s_700 | 1992-03-07 08:26:40 | 7 | 70 | 700 | 700 | 700 | 700 | 700 | s_7 | s_70 | s_700 | 1992-03-07 08:26:40 | 7 | 70 | 700 | 700 | 700 | 700 | 700 + s_8 | s_80 | s_800 | 1995-05-09 06:13:20 | 8 | 80 | 800 | 800 | 800 | 800 | 800 | s_8 | s_80 | s_800 | 1995-05-09 06:13:20 | 8 | 80 | 800 | 800 | 800 | 800 | 800 + s_9 | s_90 | s_900 | 1998-07-09 04:00:00 | 9 | 90 | 900 | 900 | 900 | 900 | 900 | s_9 | s_90 | s_900 | 1998-07-09 04:00:00 | 9 | 90 | 900 | 900 | 900 | 900 | 900 + s_10 | s_100 | s_1000 | 2001-09-09 01:46:40 | 10 | 100 | 1000 | 1000 | 1000 | 1000 | 1000 | s_10 | s_100 | s_1000 | 2001-09-09 01:46:40 | 10 | 100 | 1000 | 1000 | 1000 | 1000 | 1000 + s_11 | s_110 | s_1100 | 2004-11-09 11:33:20 | 11 | 110 | 1100 | 1100 | 1100 | 1100 | 1100 | s_11 | s_110 | s_1100 | 2004-11-09 11:33:20 | 11 | 110 | 1100 | 1100 | 1100 | 1100 | 1100 + s_12 | s_120 | s_1200 | 2008-01-10 09:20:00 | 12 | 120 | 1200 | 1200 | 1200 | 1200 | 1200 | s_12 | s_120 | s_1200 | 2008-01-10 09:20:00 | 12 | 120 | 1200 | 1200 | 1200 | 1200 | 1200 + s_13 | s_130 | s_1300 | 2011-03-13 07:06:40 | 13 | 130 | 1300 | 1300 | 1300 | 1300 | 1300 | s_13 | s_130 | s_1300 | 2011-03-13 07:06:40 | 13 | 130 | 1300 | 1300 | 1300 | 1300 | 1300 + s_14 | s_140 | s_1400 | 2014-05-13 04:53:20 | 14 | 140 | 1400 | 1400 | 1400 | 1400 | 1400 | s_14 | s_140 | s_1400 | 2014-05-13 04:53:20 | 14 | 140 | 1400 | 1400 | 1400 | 1400 | 1400 + s_15 | s_150 | s_1500 | 2017-07-14 02:40:00 | 15 | 150 | 1500 | 1500 | 1500 | 1500 | 1500 | s_15 | s_150 | s_1500 | 2017-07-14 02:40:00 | 15 | 150 | 1500 | 1500 | 1500 | 1500 | 1500 + s_16 | s_160 | s_1600 | 2020-09-13 12:26:40 | 16 | 160 | 1600 | 1600 | 1600 | 1600 | 1600 | s_16 | s_160 | s_1600 | 2020-09-13 12:26:40 | 16 | 160 | 1600 | 1600 | 1600 | 1600 | 1600 + s_17 | s_170 | s_1700 | 2023-11-14 10:13:20 | 17 | 170 | 1700 | 1700 | 1700 | 1700 | 1700 | s_17 | s_170 | s_1700 | 2023-11-14 10:13:20 | 17 | 170 | 1700 | 1700 | 1700 | 1700 | 1700 + s_18 | s_180 | s_1800 | 2027-01-15 08:00:00 | 18 | 180 | 1800 | 1800 | 1800 | 1800 | 1800 | s_18 | s_180 | s_1800 | 2027-01-15 08:00:00 | 18 | 180 | 1800 | 1800 | 1800 | 1800 | 1800 + s_19 | s_190 | s_1900 | 2030-03-17 05:46:40 | 19 | 190 | 1900 | 1900 | 1900 | 1900 | 1900 | s_19 | s_190 | s_1900 | 2030-03-17 05:46:40 | 19 | 190 | 1900 | 1900 | 1900 | 1900 | 1900 + s_20 | s_200 | s_2000 | 2033-05-18 03:33:20 | 20 | 200 | 2000 | 2000 | 2000 | 2000 | 2000 | s_20 | s_200 | s_2000 | 2033-05-18 03:33:20 | 20 | 200 | 2000 | 2000 | 2000 | 2000 | 2000 + s_21 | s_210 | s_2100 | 2036-07-18 01:20:00 | 21 | 210 | 2100 | 2100 | 2100 | 2100 | 2100 | s_21 | s_210 | s_2100 | 2036-07-18 01:20:00 | 21 | 210 | 2100 | 2100 | 2100 | 2100 | 2100 + s_22 | s_220 | s_2200 | 2039-09-18 11:06:40 | 22 | 220 | 2200 | 2200 | 2200 | 2200 | 2200 | s_22 | s_220 | s_2200 | 2039-09-18 11:06:40 | 22 | 220 | 2200 | 2200 | 2200 | 2200 | 2200 + s_23 | s_230 | s_2300 | 2042-11-19 08:53:20 | 23 | 230 | 2300 | 2300 | 2300 | 2300 | 2300 | s_23 | s_230 | s_2300 | 2042-11-19 08:53:20 | 23 | 230 | 2300 | 2300 | 2300 | 2300 | 2300 + s_24 | s_240 | s_2400 | 2046-01-19 06:40:00 | 24 | 240 | 2400 | 2400 | 2400 | 2400 | 2400 | s_24 | s_240 | s_2400 | 2046-01-19 06:40:00 | 24 | 240 | 2400 | 2400 | 2400 | 2400 | 2400 + s_25 | s_250 | s_2500 | 2049-03-22 04:26:40 | 25 | 250 | 2500 | 2500 | 2500 | 2500 | 2500 | s_25 | s_250 | s_2500 | 2049-03-22 04:26:40 | 25 | 250 | 2500 | 2500 | 2500 | 2500 | 2500 + s_26 | s_260 | s_2600 | 2052-05-22 02:13:20 | 26 | 260 | 2600 | 2600 | 2600 | 2600 | 2600 | s_26 | s_260 | s_2600 | 2052-05-22 02:13:20 | 26 | 260 | 2600 | 2600 | 2600 | 2600 | 2600 + s_27 | s_270 | s_2700 | 2055-07-24 12:00:00 | 27 | 270 | 2700 | 2700 | 2700 | 2700 | 2700 | s_27 | s_270 | s_2700 | 2055-07-24 12:00:00 | 27 | 270 | 2700 | 2700 | 2700 | 2700 | 2700 + s_28 | s_280 | s_2800 | 2058-09-23 09:46:40 | 28 | 280 | 2800 | 2800 | 2800 | 2800 | 2800 | s_28 | s_280 | s_2800 | 2058-09-23 09:46:40 | 28 | 280 | 2800 | 2800 | 2800 | 2800 | 2800 + s_29 | s_290 | s_2900 | 2061-11-23 07:33:20 | 29 | 290 | 2900 | 2900 | 2900 | 2900 | 2900 | s_29 | s_290 | s_2900 | 2061-11-23 07:33:20 | 29 | 290 | 2900 | 2900 | 2900 | 2900 | 2900 + s_30 | s_300 | s_3000 | 2065-01-24 05:20:00 | 30 | 300 | 3000 | 3000 | 3000 | 3000 | 3000 | s_30 | s_300 | s_3000 | 2065-01-24 05:20:00 | 30 | 300 | 3000 | 3000 | 3000 | 3000 | 3000 + s_31 | s_310 | s_3100 | 2068-03-26 03:06:40 | 31 | 310 | 3100 | 3100 | 3100 | 3100 | 3100 | s_31 | s_310 | s_3100 | 2068-03-26 03:06:40 | 31 | 310 | 3100 | 3100 | 3100 | 3100 | 3100 + s_32 | s_320 | s_3200 | 2071-05-28 12:53:20 | 32 | 320 | 3200 | 3200 | 3200 | 3200 | 3200 | s_32 | s_320 | s_3200 | 2071-05-28 12:53:20 | 32 | 320 | 3200 | 3200 | 3200 | 3200 | 3200 + s_33 | s_330 | s_3300 | 2074-07-28 10:40:00 | 33 | 330 | 3300 | 3300 | 3300 | 3300 | 3300 | s_33 | s_330 | s_3300 | 2074-07-28 10:40:00 | 33 | 330 | 3300 | 3300 | 3300 | 3300 | 3300 + s_34 | s_340 | s_3400 | 2077-09-27 08:26:40 | 34 | 340 | 3400 | 3400 | 3400 | 3400 | 3400 | s_34 | s_340 | s_3400 | 2077-09-27 08:26:40 | 34 | 340 | 3400 | 3400 | 3400 | 3400 | 3400 + s_35 | s_350 | s_3500 | 2080-11-28 06:13:20 | 35 | 350 | 3500 | 3500 | 3500 | 3500 | 3500 | s_35 | s_350 | s_3500 | 2080-11-28 06:13:20 | 35 | 350 | 3500 | 3500 | 3500 | 3500 | 3500 + s_36 | s_360 | s_3600 | 2084-01-29 04:00:00 | 36 | 360 | 3600 | 3600 | 3600 | 3600 | 3600 | s_36 | s_360 | s_3600 | 2084-01-29 04:00:00 | 36 | 360 | 3600 | 3600 | 3600 | 3600 | 3600 + s_37 | s_370 | s_3700 | 2087-04-01 01:46:40 | 37 | 370 | 3700 | 3700 | 3700 | 3700 | 3700 | s_37 | s_370 | s_3700 | 2087-04-01 01:46:40 | 37 | 370 | 3700 | 3700 | 3700 | 3700 | 3700 + s_38 | s_380 | s_3800 | 2090-06-01 11:33:20 | 38 | 380 | 3800 | 3800 | 3800 | 3800 | 3800 | s_38 | s_380 | s_3800 | 2090-06-01 11:33:20 | 38 | 380 | 3800 | 3800 | 3800 | 3800 | 3800 + s_39 | s_390 | s_3900 | 2093-08-01 09:20:00 | 39 | 390 | 3900 | 3900 | 3900 | 3900 | 3900 | s_39 | s_390 | s_3900 | 2093-08-01 09:20:00 | 39 | 390 | 3900 | 3900 | 3900 | 3900 | 3900 + s_40 | s_400 | s_4000 | 2096-10-02 07:06:40 | 40 | 400 | 4000 | 4000 | 4000 | 4000 | 4000 | s_40 | s_400 | s_4000 | 2096-10-02 07:06:40 | 40 | 400 | 4000 | 4000 | 4000 | 4000 | 4000 + s_41 | s_410 | s_4100 | 2099-12-03 04:53:20 | 41 | 410 | 4100 | 4100 | 4100 | 4100 | 4100 | s_41 | s_410 | s_4100 | 2099-12-03 04:53:20 | 41 | 410 | 4100 | 4100 | 4100 | 4100 | 4100 + s_42 | s_420 | s_4200 | 2103-02-04 02:40:00 | 42 | 420 | 4200 | 4200 | 4200 | 4200 | 4200 | s_42 | s_420 | s_4200 | 2103-02-04 02:40:00 | 42 | 420 | 4200 | 4200 | 4200 | 4200 | 4200 + s_43 | s_430 | s_4300 | 2106-04-06 12:26:40 | 43 | 430 | 4300 | 4300 | 4300 | 4300 | 4300 | s_43 | s_430 | s_4300 | 2106-04-06 12:26:40 | 43 | 430 | 4300 | 4300 | 4300 | 4300 | 4300 + s_44 | s_440 | s_4400 | 2109-06-06 10:13:20 | 44 | 440 | 4400 | 4400 | 4400 | 4400 | 4400 | s_44 | s_440 | s_4400 | 2109-06-06 10:13:20 | 44 | 440 | 4400 | 4400 | 4400 | 4400 | 4400 + s_45 | s_450 | s_4500 | 2112-08-07 08:00:00 | 45 | 450 | 4500 | 4500 | 4500 | 4500 | 4500 | s_45 | s_450 | s_4500 | 2112-08-07 08:00:00 | 45 | 450 | 4500 | 4500 | 4500 | 4500 | 4500 + s_46 | s_460 | s_4600 | 2115-10-08 05:46:40 | 46 | 460 | 4600 | 4600 | 4600 | 4600 | 4600 | s_46 | s_460 | s_4600 | 2115-10-08 05:46:40 | 46 | 460 | 4600 | 4600 | 4600 | 4600 | 4600 + s_47 | s_470 | s_4700 | 2118-12-09 03:33:20 | 47 | 470 | 4700 | 4700 | 4700 | 4700 | 4700 | s_47 | s_470 | s_4700 | 2118-12-09 03:33:20 | 47 | 470 | 4700 | 4700 | 4700 | 4700 | 4700 + s_48 | s_480 | s_4800 | 2122-02-08 01:20:00 | 48 | 480 | 4800 | 4800 | 4800 | 4800 | 4800 | s_48 | s_480 | s_4800 | 2122-02-08 01:20:00 | 48 | 480 | 4800 | 4800 | 4800 | 4800 | 4800 + s_49 | s_490 | s_4900 | 2125-04-10 11:06:40 | 49 | 490 | 4900 | 4900 | 4900 | 4900 | 4900 | s_49 | s_490 | s_4900 | 2125-04-10 11:06:40 | 49 | 490 | 4900 | 4900 | 4900 | 4900 | 4900 + s_50 | s_500 | s_5000 | 2128-06-11 08:53:20 | 50 | 500 | 5000 | 5000 | 5000 | 5000 | 5000 | s_50 | s_500 | s_5000 | 2128-06-11 08:53:20 | 50 | 500 | 5000 | 5000 | 5000 | 5000 | 5000 + s_51 | s_510 | s_5100 | 2131-08-12 06:40:00 | 51 | 510 | 5100 | 5100 | 5100 | 5100 | 5100 | s_51 | s_510 | s_5100 | 2131-08-12 06:40:00 | 51 | 510 | 5100 | 5100 | 5100 | 5100 | 5100 + s_52 | s_520 | s_5200 | 2134-10-13 04:26:40 | 52 | 520 | 5200 | 5200 | 5200 | 5200 | 5200 | s_52 | s_520 | s_5200 | 2134-10-13 04:26:40 | 52 | 520 | 5200 | 5200 | 5200 | 5200 | 5200 + s_53 | s_530 | s_5300 | 2137-12-13 02:13:20 | 53 | 530 | 5300 | 5300 | 5300 | 5300 | 5300 | s_53 | s_530 | s_5300 | 2137-12-13 02:13:20 | 53 | 530 | 5300 | 5300 | 5300 | 5300 | 5300 + s_54 | s_540 | s_5400 | 2141-02-13 12:00:00 | 54 | 540 | 5400 | 5400 | 5400 | 5400 | 5400 | s_54 | s_540 | s_5400 | 2141-02-13 12:00:00 | 54 | 540 | 5400 | 5400 | 5400 | 5400 | 5400 + s_55 | s_550 | s_5500 | 2144-04-15 09:46:40 | 55 | 550 | 5500 | 5500 | 5500 | 5500 | 5500 | s_55 | s_550 | s_5500 | 2144-04-15 09:46:40 | 55 | 550 | 5500 | 5500 | 5500 | 5500 | 5500 + s_56 | s_560 | s_5600 | 2147-06-16 07:33:20 | 56 | 560 | 5600 | 5600 | 5600 | 5600 | 5600 | s_56 | s_560 | s_5600 | 2147-06-16 07:33:20 | 56 | 560 | 5600 | 5600 | 5600 | 5600 | 5600 + s_57 | s_570 | s_5700 | 2150-08-17 05:20:00 | 57 | 570 | 5700 | 5700 | 5700 | 5700 | 5700 | s_57 | s_570 | s_5700 | 2150-08-17 05:20:00 | 57 | 570 | 5700 | 5700 | 5700 | 5700 | 5700 + s_58 | s_580 | s_5800 | 2153-10-17 03:06:40 | 58 | 580 | 5800 | 5800 | 5800 | 5800 | 5800 | s_58 | s_580 | s_5800 | 2153-10-17 03:06:40 | 58 | 580 | 5800 | 5800 | 5800 | 5800 | 5800 + s_59 | s_590 | s_5900 | 2156-12-18 12:53:20 | 59 | 590 | 5900 | 5900 | 5900 | 5900 | 5900 | s_59 | s_590 | s_5900 | 2156-12-18 12:53:20 | 59 | 590 | 5900 | 5900 | 5900 | 5900 | 5900 + s_60 | s_600 | s_6000 | 2160-02-18 10:40:00 | 60 | 600 | 6000 | 6000 | 6000 | 6000 | 6000 | s_60 | s_600 | s_6000 | 2160-02-18 10:40:00 | 60 | 600 | 6000 | 6000 | 6000 | 6000 | 6000 + s_61 | s_610 | s_6100 | 2163-04-20 08:26:40 | 61 | 610 | 6100 | 6100 | 6100 | 6100 | 6100 | s_61 | s_610 | s_6100 | 2163-04-20 08:26:40 | 61 | 610 | 6100 | 6100 | 6100 | 6100 | 6100 + s_62 | s_620 | s_6200 | 2166-06-21 06:13:20 | 62 | 620 | 6200 | 6200 | 6200 | 6200 | 6200 | s_62 | s_620 | s_6200 | 2166-06-21 06:13:20 | 62 | 620 | 6200 | 6200 | 6200 | 6200 | 6200 + s_63 | s_630 | s_6300 | 2169-08-21 04:00:00 | 63 | 630 | 6300 | 6300 | 6300 | 6300 | 6300 | s_63 | s_630 | s_6300 | 2169-08-21 04:00:00 | 63 | 630 | 6300 | 6300 | 6300 | 6300 | 6300 + s_64 | s_640 | s_6400 | 2172-10-22 01:46:40 | 64 | 640 | 6400 | 6400 | 6400 | 6400 | 6400 | s_64 | s_640 | s_6400 | 2172-10-22 01:46:40 | 64 | 640 | 6400 | 6400 | 6400 | 6400 | 6400 + s_65 | s_650 | s_6500 | 2175-12-23 11:33:20 | 65 | 650 | 6500 | 6500 | 6500 | 6500 | 6500 | s_65 | s_650 | s_6500 | 2175-12-23 11:33:20 | 65 | 650 | 6500 | 6500 | 6500 | 6500 | 6500 + s_66 | s_660 | s_6600 | 2179-02-22 09:20:00 | 66 | 660 | 6600 | 6600 | 6600 | 6600 | 6600 | s_66 | s_660 | s_6600 | 2179-02-22 09:20:00 | 66 | 660 | 6600 | 6600 | 6600 | 6600 | 6600 + s_67 | s_670 | s_6700 | 2182-04-25 07:06:40 | 67 | 670 | 6700 | 6700 | 6700 | 6700 | 6700 | s_67 | s_670 | s_6700 | 2182-04-25 07:06:40 | 67 | 670 | 6700 | 6700 | 6700 | 6700 | 6700 + s_68 | s_680 | s_6800 | 2185-06-25 04:53:20 | 68 | 680 | 6800 | 6800 | 6800 | 6800 | 6800 | s_68 | s_680 | s_6800 | 2185-06-25 04:53:20 | 68 | 680 | 6800 | 6800 | 6800 | 6800 | 6800 + s_69 | s_690 | s_6900 | 2188-08-26 02:40:00 | 69 | 690 | 6900 | 6900 | 6900 | 6900 | 6900 | s_69 | s_690 | s_6900 | 2188-08-26 02:40:00 | 69 | 690 | 6900 | 6900 | 6900 | 6900 | 6900 + s_70 | s_700 | s_7000 | 2191-10-27 12:26:40 | 70 | 700 | 7000 | 7000 | 7000 | 7000 | 7000 | s_70 | s_700 | s_7000 | 2191-10-27 12:26:40 | 70 | 700 | 7000 | 7000 | 7000 | 7000 | 7000 + s_71 | s_710 | s_7100 | 2194-12-27 10:13:20 | 71 | 710 | 7100 | 7100 | 7100 | 7100 | 7100 | s_71 | s_710 | s_7100 | 2194-12-27 10:13:20 | 71 | 710 | 7100 | 7100 | 7100 | 7100 | 7100 + s_72 | s_720 | s_7200 | 2198-02-27 08:00:00 | 72 | 720 | 7200 | 7200 | 7200 | 7200 | 7200 | s_72 | s_720 | s_7200 | 2198-02-27 08:00:00 | 72 | 720 | 7200 | 7200 | 7200 | 7200 | 7200 + s_73 | s_730 | s_7300 | 2201-04-30 05:46:40 | 73 | 730 | 7300 | 7300 | 7300 | 7300 | 7300 | s_73 | s_730 | s_7300 | 2201-04-30 05:46:40 | 73 | 730 | 7300 | 7300 | 7300 | 7300 | 7300 + s_74 | s_740 | s_7400 | 2204-07-01 03:33:20 | 74 | 740 | 7400 | 7400 | 7400 | 7400 | 7400 | s_74 | s_740 | s_7400 | 2204-07-01 03:33:20 | 74 | 740 | 7400 | 7400 | 7400 | 7400 | 7400 + s_75 | s_750 | s_7500 | 2207-09-01 01:20:00 | 75 | 750 | 7500 | 7500 | 7500 | 7500 | 7500 | s_75 | s_750 | s_7500 | 2207-09-01 01:20:00 | 75 | 750 | 7500 | 7500 | 7500 | 7500 | 7500 + s_76 | s_760 | s_7600 | 2210-11-01 11:06:40 | 76 | 760 | 7600 | 7600 | 7600 | 7600 | 7600 | s_76 | s_760 | s_7600 | 2210-11-01 11:06:40 | 76 | 760 | 7600 | 7600 | 7600 | 7600 | 7600 + s_77 | s_770 | s_7700 | 2214-01-02 08:53:20 | 77 | 770 | 7700 | 7700 | 7700 | 7700 | 7700 | s_77 | s_770 | s_7700 | 2214-01-02 08:53:20 | 77 | 770 | 7700 | 7700 | 7700 | 7700 | 7700 + s_78 | s_780 | s_7800 | 2217-03-04 06:40:00 | 78 | 780 | 7800 | 7800 | 7800 | 7800 | 7800 | s_78 | s_780 | s_7800 | 2217-03-04 06:40:00 | 78 | 780 | 7800 | 7800 | 7800 | 7800 | 7800 + s_79 | s_790 | s_7900 | 2220-05-05 04:26:40 | 79 | 790 | 7900 | 7900 | 7900 | 7900 | 7900 | s_79 | s_790 | s_7900 | 2220-05-05 04:26:40 | 79 | 790 | 7900 | 7900 | 7900 | 7900 | 7900 + s_80 | s_800 | s_8000 | 2223-07-06 02:13:20 | 80 | 800 | 8000 | 8000 | 8000 | 8000 | 8000 | s_80 | s_800 | s_8000 | 2223-07-06 02:13:20 | 80 | 800 | 8000 | 8000 | 8000 | 8000 | 8000 + s_81 | s_810 | s_8100 | 2226-09-06 12:00:00 | 81 | 810 | 8100 | 8100 | 8100 | 8100 | 8100 | s_81 | s_810 | s_8100 | 2226-09-06 12:00:00 | 81 | 810 | 8100 | 8100 | 8100 | 8100 | 8100 + s_82 | s_820 | s_8200 | 2229-11-06 09:46:40 | 82 | 820 | 8200 | 8200 | 8200 | 8200 | 8200 | s_82 | s_820 | s_8200 | 2229-11-06 09:46:40 | 82 | 820 | 8200 | 8200 | 8200 | 8200 | 8200 + s_83 | s_830 | s_8300 | 2233-01-06 07:33:20 | 83 | 830 | 8300 | 8300 | 8300 | 8300 | 8300 | s_83 | s_830 | s_8300 | 2233-01-06 07:33:20 | 83 | 830 | 8300 | 8300 | 8300 | 8300 | 8300 + s_84 | s_840 | s_8400 | 2236-03-09 05:20:00 | 84 | 840 | 8400 | 8400 | 8400 | 8400 | 8400 | s_84 | s_840 | s_8400 | 2236-03-09 05:20:00 | 84 | 840 | 8400 | 8400 | 8400 | 8400 | 8400 + s_85 | s_850 | s_8500 | 2239-05-10 03:06:40 | 85 | 850 | 8500 | 8500 | 8500 | 8500 | 8500 | s_85 | s_850 | s_8500 | 2239-05-10 03:06:40 | 85 | 850 | 8500 | 8500 | 8500 | 8500 | 8500 + s_86 | s_860 | s_8600 | 2242-07-11 12:53:20 | 86 | 860 | 8600 | 8600 | 8600 | 8600 | 8600 | s_86 | s_860 | s_8600 | 2242-07-11 12:53:20 | 86 | 860 | 8600 | 8600 | 8600 | 8600 | 8600 + s_87 | s_870 | s_8700 | 2245-09-10 10:40:00 | 87 | 870 | 8700 | 8700 | 8700 | 8700 | 8700 | s_87 | s_870 | s_8700 | 2245-09-10 10:40:00 | 87 | 870 | 8700 | 8700 | 8700 | 8700 | 8700 + s_88 | s_880 | s_8800 | 2248-11-10 08:26:40 | 88 | 880 | 8800 | 8800 | 8800 | 8800 | 8800 | s_88 | s_880 | s_8800 | 2248-11-10 08:26:40 | 88 | 880 | 8800 | 8800 | 8800 | 8800 | 8800 + s_89 | s_890 | s_8900 | 2252-01-12 06:13:20 | 89 | 890 | 8900 | 8900 | 8900 | 8900 | 8900 | s_89 | s_890 | s_8900 | 2252-01-12 06:13:20 | 89 | 890 | 8900 | 8900 | 8900 | 8900 | 8900 + s_90 | s_900 | s_9000 | 2255-03-14 04:00:00 | 90 | 900 | 9000 | 9000 | 9000 | 9000 | 9000 | s_90 | s_900 | s_9000 | 2255-03-14 04:00:00 | 90 | 900 | 9000 | 9000 | 9000 | 9000 | 9000 + s_91 | s_910 | s_9100 | 2258-05-15 01:46:40 | 91 | 910 | 9100 | 9100 | 9100 | 9100 | 9100 | s_91 | s_910 | s_9100 | 2258-05-15 01:46:40 | 91 | 910 | 9100 | 9100 | 9100 | 9100 | 9100 + s_92 | s_920 | s_9200 | 2261-07-15 11:33:20 | 92 | 920 | 9200 | 9200 | 9200 | 9200 | 9200 | s_92 | s_920 | s_9200 | 2261-07-15 11:33:20 | 92 | 920 | 9200 | 9200 | 9200 | 9200 | 9200 + s_93 | s_930 | s_9300 | 2264-09-14 09:20:00 | 93 | 930 | 9300 | 9300 | 9300 | 9300 | 9300 | s_93 | s_930 | s_9300 | 2264-09-14 09:20:00 | 93 | 930 | 9300 | 9300 | 9300 | 9300 | 9300 + s_94 | s_940 | s_9400 | 2267-11-16 07:06:40 | 94 | 940 | 9400 | 9400 | 9400 | 9400 | 9400 | s_94 | s_940 | s_9400 | 2267-11-16 07:06:40 | 94 | 940 | 9400 | 9400 | 9400 | 9400 | 9400 + s_95 | s_950 | s_9500 | 2271-01-16 04:53:20 | 95 | 950 | 9500 | 9500 | 9500 | 9500 | 9500 | s_95 | s_950 | s_9500 | 2271-01-16 04:53:20 | 95 | 950 | 9500 | 9500 | 9500 | 9500 | 9500 + s_96 | s_960 | s_9600 | 2274-03-19 02:40:00 | 96 | 960 | 9600 | 9600 | 9600 | 9600 | 9600 | s_96 | s_960 | s_9600 | 2274-03-19 02:40:00 | 96 | 960 | 9600 | 9600 | 9600 | 9600 | 9600 + s_97 | s_970 | s_9700 | 2277-05-19 12:26:40 | 97 | 970 | 9700 | 9700 | 9700 | 9700 | 9700 | s_97 | s_970 | s_9700 | 2277-05-19 12:26:40 | 97 | 970 | 9700 | 9700 | 9700 | 9700 | 9700 + s_98 | s_980 | s_9800 | 2280-07-19 10:13:20 | 98 | 980 | 9800 | 9800 | 9800 | 9800 | 9800 | s_98 | s_980 | s_9800 | 2280-07-19 10:13:20 | 98 | 980 | 9800 | 9800 | 9800 | 9800 | 9800 + s_99 | s_990 | s_9900 | 2283-09-20 08:00:00 | 99 | 990 | 9900 | 9900 | 9900 | 9900 | 9900 | s_99 | s_990 | s_9900 | 2283-09-20 08:00:00 | 99 | 990 | 9900 | 9900 | 9900 | 9900 | 9900 + s_100 | s_1000 | s_10000 | 2286-11-20 05:46:40 | 100 | 1000 | 10000 | 10000 | 10000 | 10000 | 10000 | s_100 | s_1000 | s_10000 | 2286-11-20 05:46:40 | 100 | 1000 | 10000 | 10000 | 10000 | 10000 | 10000 +(100 rows) + +SELECT * from pxf_multibyte_twobyte_data_with_skip ORDER BY n1; + s1 | s2 | s3 | d1 | n1 | n2 | n3 | n4 | n5 | n6 | n7 | s11 | s12 | s13 | d11 | n11 | n12 | n13 | n14 | n15 | n16 | n17 +-------+--------+---------+---------------------+-----+------+-------+-------+-------+-------+-------+-------+--------+---------+---------------------+-----+------+-------+-------+-------+-------+------- + s_11 | s_110 | s_1100 | 2004-11-09 11:33:20 | 11 | 110 | 1100 | 1100 | 1100 | 1100 | 1100 | s_11 | s_110 | s_1100 | 2004-11-09 11:33:20 | 11 | 110 | 1100 | 1100 | 1100 | 1100 | 1100 + s_12 | s_120 | s_1200 | 2008-01-10 09:20:00 | 12 | 120 | 1200 | 1200 | 1200 | 1200 | 1200 | s_12 | s_120 | s_1200 | 2008-01-10 09:20:00 | 12 | 120 | 1200 | 1200 | 1200 | 1200 | 1200 + s_13 | s_130 | s_1300 | 2011-03-13 07:06:40 | 13 | 130 | 1300 | 1300 | 1300 | 1300 | 1300 | s_13 | s_130 | s_1300 | 2011-03-13 07:06:40 | 13 | 130 | 1300 | 1300 | 1300 | 1300 | 1300 + s_14 | s_140 | s_1400 | 2014-05-13 04:53:20 | 14 | 140 | 1400 | 1400 | 1400 | 1400 | 1400 | s_14 | s_140 | s_1400 | 2014-05-13 04:53:20 | 14 | 140 | 1400 | 1400 | 1400 | 1400 | 1400 + s_15 | s_150 | s_1500 | 2017-07-14 02:40:00 | 15 | 150 | 1500 | 1500 | 1500 | 1500 | 1500 | s_15 | s_150 | s_1500 | 2017-07-14 02:40:00 | 15 | 150 | 1500 | 1500 | 1500 | 1500 | 1500 + s_16 | s_160 | s_1600 | 2020-09-13 12:26:40 | 16 | 160 | 1600 | 1600 | 1600 | 1600 | 1600 | s_16 | s_160 | s_1600 | 2020-09-13 12:26:40 | 16 | 160 | 1600 | 1600 | 1600 | 1600 | 1600 + s_17 | s_170 | s_1700 | 2023-11-14 10:13:20 | 17 | 170 | 1700 | 1700 | 1700 | 1700 | 1700 | s_17 | s_170 | s_1700 | 2023-11-14 10:13:20 | 17 | 170 | 1700 | 1700 | 1700 | 1700 | 1700 + s_18 | s_180 | s_1800 | 2027-01-15 08:00:00 | 18 | 180 | 1800 | 1800 | 1800 | 1800 | 1800 | s_18 | s_180 | s_1800 | 2027-01-15 08:00:00 | 18 | 180 | 1800 | 1800 | 1800 | 1800 | 1800 + s_19 | s_190 | s_1900 | 2030-03-17 05:46:40 | 19 | 190 | 1900 | 1900 | 1900 | 1900 | 1900 | s_19 | s_190 | s_1900 | 2030-03-17 05:46:40 | 19 | 190 | 1900 | 1900 | 1900 | 1900 | 1900 + s_20 | s_200 | s_2000 | 2033-05-18 03:33:20 | 20 | 200 | 2000 | 2000 | 2000 | 2000 | 2000 | s_20 | s_200 | s_2000 | 2033-05-18 03:33:20 | 20 | 200 | 2000 | 2000 | 2000 | 2000 | 2000 + s_21 | s_210 | s_2100 | 2036-07-18 01:20:00 | 21 | 210 | 2100 | 2100 | 2100 | 2100 | 2100 | s_21 | s_210 | s_2100 | 2036-07-18 01:20:00 | 21 | 210 | 2100 | 2100 | 2100 | 2100 | 2100 + s_22 | s_220 | s_2200 | 2039-09-18 11:06:40 | 22 | 220 | 2200 | 2200 | 2200 | 2200 | 2200 | s_22 | s_220 | s_2200 | 2039-09-18 11:06:40 | 22 | 220 | 2200 | 2200 | 2200 | 2200 | 2200 + s_23 | s_230 | s_2300 | 2042-11-19 08:53:20 | 23 | 230 | 2300 | 2300 | 2300 | 2300 | 2300 | s_23 | s_230 | s_2300 | 2042-11-19 08:53:20 | 23 | 230 | 2300 | 2300 | 2300 | 2300 | 2300 + s_24 | s_240 | s_2400 | 2046-01-19 06:40:00 | 24 | 240 | 2400 | 2400 | 2400 | 2400 | 2400 | s_24 | s_240 | s_2400 | 2046-01-19 06:40:00 | 24 | 240 | 2400 | 2400 | 2400 | 2400 | 2400 + s_25 | s_250 | s_2500 | 2049-03-22 04:26:40 | 25 | 250 | 2500 | 2500 | 2500 | 2500 | 2500 | s_25 | s_250 | s_2500 | 2049-03-22 04:26:40 | 25 | 250 | 2500 | 2500 | 2500 | 2500 | 2500 + s_26 | s_260 | s_2600 | 2052-05-22 02:13:20 | 26 | 260 | 2600 | 2600 | 2600 | 2600 | 2600 | s_26 | s_260 | s_2600 | 2052-05-22 02:13:20 | 26 | 260 | 2600 | 2600 | 2600 | 2600 | 2600 + s_27 | s_270 | s_2700 | 2055-07-24 12:00:00 | 27 | 270 | 2700 | 2700 | 2700 | 2700 | 2700 | s_27 | s_270 | s_2700 | 2055-07-24 12:00:00 | 27 | 270 | 2700 | 2700 | 2700 | 2700 | 2700 + s_28 | s_280 | s_2800 | 2058-09-23 09:46:40 | 28 | 280 | 2800 | 2800 | 2800 | 2800 | 2800 | s_28 | s_280 | s_2800 | 2058-09-23 09:46:40 | 28 | 280 | 2800 | 2800 | 2800 | 2800 | 2800 + s_29 | s_290 | s_2900 | 2061-11-23 07:33:20 | 29 | 290 | 2900 | 2900 | 2900 | 2900 | 2900 | s_29 | s_290 | s_2900 | 2061-11-23 07:33:20 | 29 | 290 | 2900 | 2900 | 2900 | 2900 | 2900 + s_30 | s_300 | s_3000 | 2065-01-24 05:20:00 | 30 | 300 | 3000 | 3000 | 3000 | 3000 | 3000 | s_30 | s_300 | s_3000 | 2065-01-24 05:20:00 | 30 | 300 | 3000 | 3000 | 3000 | 3000 | 3000 + s_31 | s_310 | s_3100 | 2068-03-26 03:06:40 | 31 | 310 | 3100 | 3100 | 3100 | 3100 | 3100 | s_31 | s_310 | s_3100 | 2068-03-26 03:06:40 | 31 | 310 | 3100 | 3100 | 3100 | 3100 | 3100 + s_32 | s_320 | s_3200 | 2071-05-28 12:53:20 | 32 | 320 | 3200 | 3200 | 3200 | 3200 | 3200 | s_32 | s_320 | s_3200 | 2071-05-28 12:53:20 | 32 | 320 | 3200 | 3200 | 3200 | 3200 | 3200 + s_33 | s_330 | s_3300 | 2074-07-28 10:40:00 | 33 | 330 | 3300 | 3300 | 3300 | 3300 | 3300 | s_33 | s_330 | s_3300 | 2074-07-28 10:40:00 | 33 | 330 | 3300 | 3300 | 3300 | 3300 | 3300 + s_34 | s_340 | s_3400 | 2077-09-27 08:26:40 | 34 | 340 | 3400 | 3400 | 3400 | 3400 | 3400 | s_34 | s_340 | s_3400 | 2077-09-27 08:26:40 | 34 | 340 | 3400 | 3400 | 3400 | 3400 | 3400 + s_35 | s_350 | s_3500 | 2080-11-28 06:13:20 | 35 | 350 | 3500 | 3500 | 3500 | 3500 | 3500 | s_35 | s_350 | s_3500 | 2080-11-28 06:13:20 | 35 | 350 | 3500 | 3500 | 3500 | 3500 | 3500 + s_36 | s_360 | s_3600 | 2084-01-29 04:00:00 | 36 | 360 | 3600 | 3600 | 3600 | 3600 | 3600 | s_36 | s_360 | s_3600 | 2084-01-29 04:00:00 | 36 | 360 | 3600 | 3600 | 3600 | 3600 | 3600 + s_37 | s_370 | s_3700 | 2087-04-01 01:46:40 | 37 | 370 | 3700 | 3700 | 3700 | 3700 | 3700 | s_37 | s_370 | s_3700 | 2087-04-01 01:46:40 | 37 | 370 | 3700 | 3700 | 3700 | 3700 | 3700 + s_38 | s_380 | s_3800 | 2090-06-01 11:33:20 | 38 | 380 | 3800 | 3800 | 3800 | 3800 | 3800 | s_38 | s_380 | s_3800 | 2090-06-01 11:33:20 | 38 | 380 | 3800 | 3800 | 3800 | 3800 | 3800 + s_39 | s_390 | s_3900 | 2093-08-01 09:20:00 | 39 | 390 | 3900 | 3900 | 3900 | 3900 | 3900 | s_39 | s_390 | s_3900 | 2093-08-01 09:20:00 | 39 | 390 | 3900 | 3900 | 3900 | 3900 | 3900 + s_40 | s_400 | s_4000 | 2096-10-02 07:06:40 | 40 | 400 | 4000 | 4000 | 4000 | 4000 | 4000 | s_40 | s_400 | s_4000 | 2096-10-02 07:06:40 | 40 | 400 | 4000 | 4000 | 4000 | 4000 | 4000 + s_41 | s_410 | s_4100 | 2099-12-03 04:53:20 | 41 | 410 | 4100 | 4100 | 4100 | 4100 | 4100 | s_41 | s_410 | s_4100 | 2099-12-03 04:53:20 | 41 | 410 | 4100 | 4100 | 4100 | 4100 | 4100 + s_42 | s_420 | s_4200 | 2103-02-04 02:40:00 | 42 | 420 | 4200 | 4200 | 4200 | 4200 | 4200 | s_42 | s_420 | s_4200 | 2103-02-04 02:40:00 | 42 | 420 | 4200 | 4200 | 4200 | 4200 | 4200 + s_43 | s_430 | s_4300 | 2106-04-06 12:26:40 | 43 | 430 | 4300 | 4300 | 4300 | 4300 | 4300 | s_43 | s_430 | s_4300 | 2106-04-06 12:26:40 | 43 | 430 | 4300 | 4300 | 4300 | 4300 | 4300 + s_44 | s_440 | s_4400 | 2109-06-06 10:13:20 | 44 | 440 | 4400 | 4400 | 4400 | 4400 | 4400 | s_44 | s_440 | s_4400 | 2109-06-06 10:13:20 | 44 | 440 | 4400 | 4400 | 4400 | 4400 | 4400 + s_45 | s_450 | s_4500 | 2112-08-07 08:00:00 | 45 | 450 | 4500 | 4500 | 4500 | 4500 | 4500 | s_45 | s_450 | s_4500 | 2112-08-07 08:00:00 | 45 | 450 | 4500 | 4500 | 4500 | 4500 | 4500 + s_46 | s_460 | s_4600 | 2115-10-08 05:46:40 | 46 | 460 | 4600 | 4600 | 4600 | 4600 | 4600 | s_46 | s_460 | s_4600 | 2115-10-08 05:46:40 | 46 | 460 | 4600 | 4600 | 4600 | 4600 | 4600 + s_47 | s_470 | s_4700 | 2118-12-09 03:33:20 | 47 | 470 | 4700 | 4700 | 4700 | 4700 | 4700 | s_47 | s_470 | s_4700 | 2118-12-09 03:33:20 | 47 | 470 | 4700 | 4700 | 4700 | 4700 | 4700 + s_48 | s_480 | s_4800 | 2122-02-08 01:20:00 | 48 | 480 | 4800 | 4800 | 4800 | 4800 | 4800 | s_48 | s_480 | s_4800 | 2122-02-08 01:20:00 | 48 | 480 | 4800 | 4800 | 4800 | 4800 | 4800 + s_49 | s_490 | s_4900 | 2125-04-10 11:06:40 | 49 | 490 | 4900 | 4900 | 4900 | 4900 | 4900 | s_49 | s_490 | s_4900 | 2125-04-10 11:06:40 | 49 | 490 | 4900 | 4900 | 4900 | 4900 | 4900 + s_50 | s_500 | s_5000 | 2128-06-11 08:53:20 | 50 | 500 | 5000 | 5000 | 5000 | 5000 | 5000 | s_50 | s_500 | s_5000 | 2128-06-11 08:53:20 | 50 | 500 | 5000 | 5000 | 5000 | 5000 | 5000 + s_51 | s_510 | s_5100 | 2131-08-12 06:40:00 | 51 | 510 | 5100 | 5100 | 5100 | 5100 | 5100 | s_51 | s_510 | s_5100 | 2131-08-12 06:40:00 | 51 | 510 | 5100 | 5100 | 5100 | 5100 | 5100 + s_52 | s_520 | s_5200 | 2134-10-13 04:26:40 | 52 | 520 | 5200 | 5200 | 5200 | 5200 | 5200 | s_52 | s_520 | s_5200 | 2134-10-13 04:26:40 | 52 | 520 | 5200 | 5200 | 5200 | 5200 | 5200 + s_53 | s_530 | s_5300 | 2137-12-13 02:13:20 | 53 | 530 | 5300 | 5300 | 5300 | 5300 | 5300 | s_53 | s_530 | s_5300 | 2137-12-13 02:13:20 | 53 | 530 | 5300 | 5300 | 5300 | 5300 | 5300 + s_54 | s_540 | s_5400 | 2141-02-13 12:00:00 | 54 | 540 | 5400 | 5400 | 5400 | 5400 | 5400 | s_54 | s_540 | s_5400 | 2141-02-13 12:00:00 | 54 | 540 | 5400 | 5400 | 5400 | 5400 | 5400 + s_55 | s_550 | s_5500 | 2144-04-15 09:46:40 | 55 | 550 | 5500 | 5500 | 5500 | 5500 | 5500 | s_55 | s_550 | s_5500 | 2144-04-15 09:46:40 | 55 | 550 | 5500 | 5500 | 5500 | 5500 | 5500 + s_56 | s_560 | s_5600 | 2147-06-16 07:33:20 | 56 | 560 | 5600 | 5600 | 5600 | 5600 | 5600 | s_56 | s_560 | s_5600 | 2147-06-16 07:33:20 | 56 | 560 | 5600 | 5600 | 5600 | 5600 | 5600 + s_57 | s_570 | s_5700 | 2150-08-17 05:20:00 | 57 | 570 | 5700 | 5700 | 5700 | 5700 | 5700 | s_57 | s_570 | s_5700 | 2150-08-17 05:20:00 | 57 | 570 | 5700 | 5700 | 5700 | 5700 | 5700 + s_58 | s_580 | s_5800 | 2153-10-17 03:06:40 | 58 | 580 | 5800 | 5800 | 5800 | 5800 | 5800 | s_58 | s_580 | s_5800 | 2153-10-17 03:06:40 | 58 | 580 | 5800 | 5800 | 5800 | 5800 | 5800 + s_59 | s_590 | s_5900 | 2156-12-18 12:53:20 | 59 | 590 | 5900 | 5900 | 5900 | 5900 | 5900 | s_59 | s_590 | s_5900 | 2156-12-18 12:53:20 | 59 | 590 | 5900 | 5900 | 5900 | 5900 | 5900 + s_60 | s_600 | s_6000 | 2160-02-18 10:40:00 | 60 | 600 | 6000 | 6000 | 6000 | 6000 | 6000 | s_60 | s_600 | s_6000 | 2160-02-18 10:40:00 | 60 | 600 | 6000 | 6000 | 6000 | 6000 | 6000 + s_61 | s_610 | s_6100 | 2163-04-20 08:26:40 | 61 | 610 | 6100 | 6100 | 6100 | 6100 | 6100 | s_61 | s_610 | s_6100 | 2163-04-20 08:26:40 | 61 | 610 | 6100 | 6100 | 6100 | 6100 | 6100 + s_62 | s_620 | s_6200 | 2166-06-21 06:13:20 | 62 | 620 | 6200 | 6200 | 6200 | 6200 | 6200 | s_62 | s_620 | s_6200 | 2166-06-21 06:13:20 | 62 | 620 | 6200 | 6200 | 6200 | 6200 | 6200 + s_63 | s_630 | s_6300 | 2169-08-21 04:00:00 | 63 | 630 | 6300 | 6300 | 6300 | 6300 | 6300 | s_63 | s_630 | s_6300 | 2169-08-21 04:00:00 | 63 | 630 | 6300 | 6300 | 6300 | 6300 | 6300 + s_64 | s_640 | s_6400 | 2172-10-22 01:46:40 | 64 | 640 | 6400 | 6400 | 6400 | 6400 | 6400 | s_64 | s_640 | s_6400 | 2172-10-22 01:46:40 | 64 | 640 | 6400 | 6400 | 6400 | 6400 | 6400 + s_65 | s_650 | s_6500 | 2175-12-23 11:33:20 | 65 | 650 | 6500 | 6500 | 6500 | 6500 | 6500 | s_65 | s_650 | s_6500 | 2175-12-23 11:33:20 | 65 | 650 | 6500 | 6500 | 6500 | 6500 | 6500 + s_66 | s_660 | s_6600 | 2179-02-22 09:20:00 | 66 | 660 | 6600 | 6600 | 6600 | 6600 | 6600 | s_66 | s_660 | s_6600 | 2179-02-22 09:20:00 | 66 | 660 | 6600 | 6600 | 6600 | 6600 | 6600 + s_67 | s_670 | s_6700 | 2182-04-25 07:06:40 | 67 | 670 | 6700 | 6700 | 6700 | 6700 | 6700 | s_67 | s_670 | s_6700 | 2182-04-25 07:06:40 | 67 | 670 | 6700 | 6700 | 6700 | 6700 | 6700 + s_68 | s_680 | s_6800 | 2185-06-25 04:53:20 | 68 | 680 | 6800 | 6800 | 6800 | 6800 | 6800 | s_68 | s_680 | s_6800 | 2185-06-25 04:53:20 | 68 | 680 | 6800 | 6800 | 6800 | 6800 | 6800 + s_69 | s_690 | s_6900 | 2188-08-26 02:40:00 | 69 | 690 | 6900 | 6900 | 6900 | 6900 | 6900 | s_69 | s_690 | s_6900 | 2188-08-26 02:40:00 | 69 | 690 | 6900 | 6900 | 6900 | 6900 | 6900 + s_70 | s_700 | s_7000 | 2191-10-27 12:26:40 | 70 | 700 | 7000 | 7000 | 7000 | 7000 | 7000 | s_70 | s_700 | s_7000 | 2191-10-27 12:26:40 | 70 | 700 | 7000 | 7000 | 7000 | 7000 | 7000 + s_71 | s_710 | s_7100 | 2194-12-27 10:13:20 | 71 | 710 | 7100 | 7100 | 7100 | 7100 | 7100 | s_71 | s_710 | s_7100 | 2194-12-27 10:13:20 | 71 | 710 | 7100 | 7100 | 7100 | 7100 | 7100 + s_72 | s_720 | s_7200 | 2198-02-27 08:00:00 | 72 | 720 | 7200 | 7200 | 7200 | 7200 | 7200 | s_72 | s_720 | s_7200 | 2198-02-27 08:00:00 | 72 | 720 | 7200 | 7200 | 7200 | 7200 | 7200 + s_73 | s_730 | s_7300 | 2201-04-30 05:46:40 | 73 | 730 | 7300 | 7300 | 7300 | 7300 | 7300 | s_73 | s_730 | s_7300 | 2201-04-30 05:46:40 | 73 | 730 | 7300 | 7300 | 7300 | 7300 | 7300 + s_74 | s_740 | s_7400 | 2204-07-01 03:33:20 | 74 | 740 | 7400 | 7400 | 7400 | 7400 | 7400 | s_74 | s_740 | s_7400 | 2204-07-01 03:33:20 | 74 | 740 | 7400 | 7400 | 7400 | 7400 | 7400 + s_75 | s_750 | s_7500 | 2207-09-01 01:20:00 | 75 | 750 | 7500 | 7500 | 7500 | 7500 | 7500 | s_75 | s_750 | s_7500 | 2207-09-01 01:20:00 | 75 | 750 | 7500 | 7500 | 7500 | 7500 | 7500 + s_76 | s_760 | s_7600 | 2210-11-01 11:06:40 | 76 | 760 | 7600 | 7600 | 7600 | 7600 | 7600 | s_76 | s_760 | s_7600 | 2210-11-01 11:06:40 | 76 | 760 | 7600 | 7600 | 7600 | 7600 | 7600 + s_77 | s_770 | s_7700 | 2214-01-02 08:53:20 | 77 | 770 | 7700 | 7700 | 7700 | 7700 | 7700 | s_77 | s_770 | s_7700 | 2214-01-02 08:53:20 | 77 | 770 | 7700 | 7700 | 7700 | 7700 | 7700 + s_78 | s_780 | s_7800 | 2217-03-04 06:40:00 | 78 | 780 | 7800 | 7800 | 7800 | 7800 | 7800 | s_78 | s_780 | s_7800 | 2217-03-04 06:40:00 | 78 | 780 | 7800 | 7800 | 7800 | 7800 | 7800 + s_79 | s_790 | s_7900 | 2220-05-05 04:26:40 | 79 | 790 | 7900 | 7900 | 7900 | 7900 | 7900 | s_79 | s_790 | s_7900 | 2220-05-05 04:26:40 | 79 | 790 | 7900 | 7900 | 7900 | 7900 | 7900 + s_80 | s_800 | s_8000 | 2223-07-06 02:13:20 | 80 | 800 | 8000 | 8000 | 8000 | 8000 | 8000 | s_80 | s_800 | s_8000 | 2223-07-06 02:13:20 | 80 | 800 | 8000 | 8000 | 8000 | 8000 | 8000 + s_81 | s_810 | s_8100 | 2226-09-06 12:00:00 | 81 | 810 | 8100 | 8100 | 8100 | 8100 | 8100 | s_81 | s_810 | s_8100 | 2226-09-06 12:00:00 | 81 | 810 | 8100 | 8100 | 8100 | 8100 | 8100 + s_82 | s_820 | s_8200 | 2229-11-06 09:46:40 | 82 | 820 | 8200 | 8200 | 8200 | 8200 | 8200 | s_82 | s_820 | s_8200 | 2229-11-06 09:46:40 | 82 | 820 | 8200 | 8200 | 8200 | 8200 | 8200 + s_83 | s_830 | s_8300 | 2233-01-06 07:33:20 | 83 | 830 | 8300 | 8300 | 8300 | 8300 | 8300 | s_83 | s_830 | s_8300 | 2233-01-06 07:33:20 | 83 | 830 | 8300 | 8300 | 8300 | 8300 | 8300 + s_84 | s_840 | s_8400 | 2236-03-09 05:20:00 | 84 | 840 | 8400 | 8400 | 8400 | 8400 | 8400 | s_84 | s_840 | s_8400 | 2236-03-09 05:20:00 | 84 | 840 | 8400 | 8400 | 8400 | 8400 | 8400 + s_85 | s_850 | s_8500 | 2239-05-10 03:06:40 | 85 | 850 | 8500 | 8500 | 8500 | 8500 | 8500 | s_85 | s_850 | s_8500 | 2239-05-10 03:06:40 | 85 | 850 | 8500 | 8500 | 8500 | 8500 | 8500 + s_86 | s_860 | s_8600 | 2242-07-11 12:53:20 | 86 | 860 | 8600 | 8600 | 8600 | 8600 | 8600 | s_86 | s_860 | s_8600 | 2242-07-11 12:53:20 | 86 | 860 | 8600 | 8600 | 8600 | 8600 | 8600 + s_87 | s_870 | s_8700 | 2245-09-10 10:40:00 | 87 | 870 | 8700 | 8700 | 8700 | 8700 | 8700 | s_87 | s_870 | s_8700 | 2245-09-10 10:40:00 | 87 | 870 | 8700 | 8700 | 8700 | 8700 | 8700 + s_88 | s_880 | s_8800 | 2248-11-10 08:26:40 | 88 | 880 | 8800 | 8800 | 8800 | 8800 | 8800 | s_88 | s_880 | s_8800 | 2248-11-10 08:26:40 | 88 | 880 | 8800 | 8800 | 8800 | 8800 | 8800 + s_89 | s_890 | s_8900 | 2252-01-12 06:13:20 | 89 | 890 | 8900 | 8900 | 8900 | 8900 | 8900 | s_89 | s_890 | s_8900 | 2252-01-12 06:13:20 | 89 | 890 | 8900 | 8900 | 8900 | 8900 | 8900 + s_90 | s_900 | s_9000 | 2255-03-14 04:00:00 | 90 | 900 | 9000 | 9000 | 9000 | 9000 | 9000 | s_90 | s_900 | s_9000 | 2255-03-14 04:00:00 | 90 | 900 | 9000 | 9000 | 9000 | 9000 | 9000 + s_91 | s_910 | s_9100 | 2258-05-15 01:46:40 | 91 | 910 | 9100 | 9100 | 9100 | 9100 | 9100 | s_91 | s_910 | s_9100 | 2258-05-15 01:46:40 | 91 | 910 | 9100 | 9100 | 9100 | 9100 | 9100 + s_92 | s_920 | s_9200 | 2261-07-15 11:33:20 | 92 | 920 | 9200 | 9200 | 9200 | 9200 | 9200 | s_92 | s_920 | s_9200 | 2261-07-15 11:33:20 | 92 | 920 | 9200 | 9200 | 9200 | 9200 | 9200 + s_93 | s_930 | s_9300 | 2264-09-14 09:20:00 | 93 | 930 | 9300 | 9300 | 9300 | 9300 | 9300 | s_93 | s_930 | s_9300 | 2264-09-14 09:20:00 | 93 | 930 | 9300 | 9300 | 9300 | 9300 | 9300 + s_94 | s_940 | s_9400 | 2267-11-16 07:06:40 | 94 | 940 | 9400 | 9400 | 9400 | 9400 | 9400 | s_94 | s_940 | s_9400 | 2267-11-16 07:06:40 | 94 | 940 | 9400 | 9400 | 9400 | 9400 | 9400 + s_95 | s_950 | s_9500 | 2271-01-16 04:53:20 | 95 | 950 | 9500 | 9500 | 9500 | 9500 | 9500 | s_95 | s_950 | s_9500 | 2271-01-16 04:53:20 | 95 | 950 | 9500 | 9500 | 9500 | 9500 | 9500 + s_96 | s_960 | s_9600 | 2274-03-19 02:40:00 | 96 | 960 | 9600 | 9600 | 9600 | 9600 | 9600 | s_96 | s_960 | s_9600 | 2274-03-19 02:40:00 | 96 | 960 | 9600 | 9600 | 9600 | 9600 | 9600 + s_97 | s_970 | s_9700 | 2277-05-19 12:26:40 | 97 | 970 | 9700 | 9700 | 9700 | 9700 | 9700 | s_97 | s_970 | s_9700 | 2277-05-19 12:26:40 | 97 | 970 | 9700 | 9700 | 9700 | 9700 | 9700 + s_98 | s_980 | s_9800 | 2280-07-19 10:13:20 | 98 | 980 | 9800 | 9800 | 9800 | 9800 | 9800 | s_98 | s_980 | s_9800 | 2280-07-19 10:13:20 | 98 | 980 | 9800 | 9800 | 9800 | 9800 | 9800 + s_99 | s_990 | s_9900 | 2283-09-20 08:00:00 | 99 | 990 | 9900 | 9900 | 9900 | 9900 | 9900 | s_99 | s_990 | s_9900 | 2283-09-20 08:00:00 | 99 | 990 | 9900 | 9900 | 9900 | 9900 | 9900 + s_100 | s_1000 | s_10000 | 2286-11-20 05:46:40 | 100 | 1000 | 10000 | 10000 | 10000 | 10000 | 10000 | s_100 | s_1000 | s_10000 | 2286-11-20 05:46:40 | 100 | 1000 | 10000 | 10000 | 10000 | 10000 | 10000 +(90 rows) \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte/runTest.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte/runTest.py new file mode 100755 index 0000000000..9d317200ad --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfMultibyteTwoByteDelimiterData(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte/sql/query01.sql b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte/sql/query01.sql new file mode 100755 index 0000000000..c62387b7c0 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte/sql/query01.sql @@ -0,0 +1,5 @@ +-- @description query01 for PXF Multibyte delimiter, 2-byte delim cases + +SELECT * from pxf_multibyte_twobyte_data ORDER BY n1; + +SELECT * from pxf_multibyte_twobyte_data_with_skip ORDER BY n1; \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_no_delim/__init__.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_no_delim/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_no_delim/expected/query01.ans b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_no_delim/expected/query01.ans new file mode 100755 index 0000000000..8ffcdc6345 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_no_delim/expected/query01.ans @@ -0,0 +1,8 @@ +-- start_ignore +-- m/DETAIL/ +-- s/DETAIL/CONTEXT/ +-- end_ignore +-- @description query01 for PXF Multibyte delimiter, 2-byte delim cases with no delim provided +SELECT * from pxf_multibyte_twobyte_nodelim_data ORDER BY n1; +ERROR: missing delimiter option (pxfdelimited_formatter.c:xxx) +CONTEXT: External table pxf_multibyte_twobyte_nodelim_data diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_no_delim/runTest.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_no_delim/runTest.py new file mode 100755 index 0000000000..42b48c068f --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_no_delim/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfMultibyteTwoByteNoDelimiter(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_no_delim/sql/query01.sql b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_no_delim/sql/query01.sql new file mode 100755 index 0000000000..551d66cd17 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_no_delim/sql/query01.sql @@ -0,0 +1,3 @@ +-- @description query01 for PXF Multibyte delimiter, 2-byte delim cases with no formatter provided + +SELECT * from pxf_multibyte_twobyte_nodelim_data ORDER BY n1; \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_bzip2/__init__.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_bzip2/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_bzip2/expected/query01.ans b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_bzip2/expected/query01.ans new file mode 100755 index 0000000000..926843adcf --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_bzip2/expected/query01.ans @@ -0,0 +1,108 @@ +-- start_ignore +-- end_ignore +-- @description query01 for PXF Multibyte delimiter, 2-byte delim cases with compressed bzip2 file +SELECT * from pxf_multibyte_twobyte_withbzip2_data ORDER BY name; + name | num | dub | longnum | bool +-----------+-----+-----+---------------+------ + aa_row_1 | 1 | 1 | 100000000000 | f + aa_row_10 | 10 | 10 | 1000000000000 | t + aa_row_2 | 2 | 2 | 200000000000 | t + aa_row_3 | 3 | 3 | 300000000000 | f + aa_row_4 | 4 | 4 | 400000000000 | t + aa_row_5 | 5 | 5 | 500000000000 | f + aa_row_6 | 6 | 6 | 600000000000 | t + aa_row_7 | 7 | 7 | 700000000000 | f + aa_row_8 | 8 | 8 | 800000000000 | t + aa_row_9 | 9 | 9 | 900000000000 | f + bb_row_1 | 1 | 1 | 100000000000 | f + bb_row_10 | 10 | 10 | 1000000000000 | t + bb_row_2 | 2 | 2 | 200000000000 | t + bb_row_3 | 3 | 3 | 300000000000 | f + bb_row_4 | 4 | 4 | 400000000000 | t + bb_row_5 | 5 | 5 | 500000000000 | f + bb_row_6 | 6 | 6 | 600000000000 | t + bb_row_7 | 7 | 7 | 700000000000 | f + bb_row_8 | 8 | 8 | 800000000000 | t + bb_row_9 | 9 | 9 | 900000000000 | f + cc_row_1 | 1 | 1 | 100000000000 | f + cc_row_10 | 10 | 10 | 1000000000000 | t + cc_row_2 | 2 | 2 | 200000000000 | t + cc_row_3 | 3 | 3 | 300000000000 | f + cc_row_4 | 4 | 4 | 400000000000 | t + cc_row_5 | 5 | 5 | 500000000000 | f + cc_row_6 | 6 | 6 | 600000000000 | t + cc_row_7 | 7 | 7 | 700000000000 | f + cc_row_8 | 8 | 8 | 800000000000 | t + cc_row_9 | 9 | 9 | 900000000000 | f + dd_row_1 | 1 | 1 | 100000000000 | f + dd_row_10 | 10 | 10 | 1000000000000 | t + dd_row_2 | 2 | 2 | 200000000000 | t + dd_row_3 | 3 | 3 | 300000000000 | f + dd_row_4 | 4 | 4 | 400000000000 | t + dd_row_5 | 5 | 5 | 500000000000 | f + dd_row_6 | 6 | 6 | 600000000000 | t + dd_row_7 | 7 | 7 | 700000000000 | f + dd_row_8 | 8 | 8 | 800000000000 | t + dd_row_9 | 9 | 9 | 900000000000 | f + ee_row_1 | 1 | 1 | 100000000000 | f + ee_row_10 | 10 | 10 | 1000000000000 | t + ee_row_2 | 2 | 2 | 200000000000 | t + ee_row_3 | 3 | 3 | 300000000000 | f + ee_row_4 | 4 | 4 | 400000000000 | t + ee_row_5 | 5 | 5 | 500000000000 | f + ee_row_6 | 6 | 6 | 600000000000 | t + ee_row_7 | 7 | 7 | 700000000000 | f + ee_row_8 | 8 | 8 | 800000000000 | t + ee_row_9 | 9 | 9 | 900000000000 | f + ff_row_1 | 1 | 1 | 100000000000 | f + ff_row_10 | 10 | 10 | 1000000000000 | t + ff_row_2 | 2 | 2 | 200000000000 | t + ff_row_3 | 3 | 3 | 300000000000 | f + ff_row_4 | 4 | 4 | 400000000000 | t + ff_row_5 | 5 | 5 | 500000000000 | f + ff_row_6 | 6 | 6 | 600000000000 | t + ff_row_7 | 7 | 7 | 700000000000 | f + ff_row_8 | 8 | 8 | 800000000000 | t + ff_row_9 | 9 | 9 | 900000000000 | f + gg_row_1 | 1 | 1 | 100000000000 | f + gg_row_10 | 10 | 10 | 1000000000000 | t + gg_row_2 | 2 | 2 | 200000000000 | t + gg_row_3 | 3 | 3 | 300000000000 | f + gg_row_4 | 4 | 4 | 400000000000 | t + gg_row_5 | 5 | 5 | 500000000000 | f + gg_row_6 | 6 | 6 | 600000000000 | t + gg_row_7 | 7 | 7 | 700000000000 | f + gg_row_8 | 8 | 8 | 800000000000 | t + gg_row_9 | 9 | 9 | 900000000000 | f + hh_row_1 | 1 | 1 | 100000000000 | f + hh_row_10 | 10 | 10 | 1000000000000 | t + hh_row_2 | 2 | 2 | 200000000000 | t + hh_row_3 | 3 | 3 | 300000000000 | f + hh_row_4 | 4 | 4 | 400000000000 | t + hh_row_5 | 5 | 5 | 500000000000 | f + hh_row_6 | 6 | 6 | 600000000000 | t + hh_row_7 | 7 | 7 | 700000000000 | f + hh_row_8 | 8 | 8 | 800000000000 | t + hh_row_9 | 9 | 9 | 900000000000 | f + ii_row_1 | 1 | 1 | 100000000000 | f + ii_row_10 | 10 | 10 | 1000000000000 | t + ii_row_2 | 2 | 2 | 200000000000 | t + ii_row_3 | 3 | 3 | 300000000000 | f + ii_row_4 | 4 | 4 | 400000000000 | t + ii_row_5 | 5 | 5 | 500000000000 | f + ii_row_6 | 6 | 6 | 600000000000 | t + ii_row_7 | 7 | 7 | 700000000000 | f + ii_row_8 | 8 | 8 | 800000000000 | t + ii_row_9 | 9 | 9 | 900000000000 | f + jj_row_1 | 1 | 1 | 100000000000 | f + jj_row_10 | 10 | 10 | 1000000000000 | t + jj_row_2 | 2 | 2 | 200000000000 | t + jj_row_3 | 3 | 3 | 300000000000 | f + jj_row_4 | 4 | 4 | 400000000000 | t + jj_row_5 | 5 | 5 | 500000000000 | f + jj_row_6 | 6 | 6 | 600000000000 | t + jj_row_7 | 7 | 7 | 700000000000 | f + jj_row_8 | 8 | 8 | 800000000000 | t + jj_row_9 | 9 | 9 | 900000000000 | f +(100 rows) + diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_bzip2/runTest.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_bzip2/runTest.py new file mode 100755 index 0000000000..2bb72d2f64 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_bzip2/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfMultibyteTwoByteWithBzip2(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_bzip2/sql/query01.sql b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_bzip2/sql/query01.sql new file mode 100755 index 0000000000..f5d31c7f7b --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_bzip2/sql/query01.sql @@ -0,0 +1,3 @@ +-- @description query01 for PXF Multibyte delimiter, 2-byte delim cases with compressed bzip2 file + +SELECT * from pxf_multibyte_twobyte_withbzip2_data ORDER BY name; \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_cr/__init__.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_cr/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_cr/expected/query01.ans b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_cr/expected/query01.ans new file mode 100755 index 0000000000..b14366a004 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_cr/expected/query01.ans @@ -0,0 +1,106 @@ +-- @description query01 for PXF Multibyte delimiter, 2-byte delim with CR case + +SELECT * from pxf_multibyte_twobyte_withcr_data ORDER BY n1; + s1 | s2 | s3 | d1 | n1 | n2 | n3 | n4 | n5 | n6 | n7 | s11 | s12 | s13 | d11 | n11 | n12 | n13 | n14 | n15 | n16 | n17 +-------+--------+---------+---------------------+-----+------+-------+-------+-------+-------+-------+-------+--------+---------+---------------------+-----+------+-------+-------+-------+-------+------- + s_1 | s_10 | s_100 | 1973-03-03 09:46:40 | 1 | 10 | 100 | 100 | 100 | 100 | 100 | s_1 | s_10 | s_100 | 1973-03-03 09:46:40 | 1 | 10 | 100 | 100 | 100 | 100 | 100 + s_2 | s_20 | s_200 | 1976-05-03 07:33:20 | 2 | 20 | 200 | 200 | 200 | 200 | 200 | s_2 | s_20 | s_200 | 1976-05-03 07:33:20 | 2 | 20 | 200 | 200 | 200 | 200 | 200 + s_3 | s_30 | s_300 | 1979-07-05 05:20:00 | 3 | 30 | 300 | 300 | 300 | 300 | 300 | s_3 | s_30 | s_300 | 1979-07-05 05:20:00 | 3 | 30 | 300 | 300 | 300 | 300 | 300 + s_4 | s_40 | s_400 | 1982-09-04 03:06:40 | 4 | 40 | 400 | 400 | 400 | 400 | 400 | s_4 | s_40 | s_400 | 1982-09-04 03:06:40 | 4 | 40 | 400 | 400 | 400 | 400 | 400 + s_5 | s_50 | s_500 | 1985-11-05 12:53:20 | 5 | 50 | 500 | 500 | 500 | 500 | 500 | s_5 | s_50 | s_500 | 1985-11-05 12:53:20 | 5 | 50 | 500 | 500 | 500 | 500 | 500 + s_6 | s_60 | s_600 | 1989-01-05 10:40:00 | 6 | 60 | 600 | 600 | 600 | 600 | 600 | s_6 | s_60 | s_600 | 1989-01-05 10:40:00 | 6 | 60 | 600 | 600 | 600 | 600 | 600 + s_7 | s_70 | s_700 | 1992-03-07 08:26:40 | 7 | 70 | 700 | 700 | 700 | 700 | 700 | s_7 | s_70 | s_700 | 1992-03-07 08:26:40 | 7 | 70 | 700 | 700 | 700 | 700 | 700 + s_8 | s_80 | s_800 | 1995-05-09 06:13:20 | 8 | 80 | 800 | 800 | 800 | 800 | 800 | s_8 | s_80 | s_800 | 1995-05-09 06:13:20 | 8 | 80 | 800 | 800 | 800 | 800 | 800 + s_9 | s_90 | s_900 | 1998-07-09 04:00:00 | 9 | 90 | 900 | 900 | 900 | 900 | 900 | s_9 | s_90 | s_900 | 1998-07-09 04:00:00 | 9 | 90 | 900 | 900 | 900 | 900 | 900 + s_10 | s_100 | s_1000 | 2001-09-09 01:46:40 | 10 | 100 | 1000 | 1000 | 1000 | 1000 | 1000 | s_10 | s_100 | s_1000 | 2001-09-09 01:46:40 | 10 | 100 | 1000 | 1000 | 1000 | 1000 | 1000 + s_11 | s_110 | s_1100 | 2004-11-09 11:33:20 | 11 | 110 | 1100 | 1100 | 1100 | 1100 | 1100 | s_11 | s_110 | s_1100 | 2004-11-09 11:33:20 | 11 | 110 | 1100 | 1100 | 1100 | 1100 | 1100 + s_12 | s_120 | s_1200 | 2008-01-10 09:20:00 | 12 | 120 | 1200 | 1200 | 1200 | 1200 | 1200 | s_12 | s_120 | s_1200 | 2008-01-10 09:20:00 | 12 | 120 | 1200 | 1200 | 1200 | 1200 | 1200 + s_13 | s_130 | s_1300 | 2011-03-13 07:06:40 | 13 | 130 | 1300 | 1300 | 1300 | 1300 | 1300 | s_13 | s_130 | s_1300 | 2011-03-13 07:06:40 | 13 | 130 | 1300 | 1300 | 1300 | 1300 | 1300 + s_14 | s_140 | s_1400 | 2014-05-13 04:53:20 | 14 | 140 | 1400 | 1400 | 1400 | 1400 | 1400 | s_14 | s_140 | s_1400 | 2014-05-13 04:53:20 | 14 | 140 | 1400 | 1400 | 1400 | 1400 | 1400 + s_15 | s_150 | s_1500 | 2017-07-14 02:40:00 | 15 | 150 | 1500 | 1500 | 1500 | 1500 | 1500 | s_15 | s_150 | s_1500 | 2017-07-14 02:40:00 | 15 | 150 | 1500 | 1500 | 1500 | 1500 | 1500 + s_16 | s_160 | s_1600 | 2020-09-13 12:26:40 | 16 | 160 | 1600 | 1600 | 1600 | 1600 | 1600 | s_16 | s_160 | s_1600 | 2020-09-13 12:26:40 | 16 | 160 | 1600 | 1600 | 1600 | 1600 | 1600 + s_17 | s_170 | s_1700 | 2023-11-14 10:13:20 | 17 | 170 | 1700 | 1700 | 1700 | 1700 | 1700 | s_17 | s_170 | s_1700 | 2023-11-14 10:13:20 | 17 | 170 | 1700 | 1700 | 1700 | 1700 | 1700 + s_18 | s_180 | s_1800 | 2027-01-15 08:00:00 | 18 | 180 | 1800 | 1800 | 1800 | 1800 | 1800 | s_18 | s_180 | s_1800 | 2027-01-15 08:00:00 | 18 | 180 | 1800 | 1800 | 1800 | 1800 | 1800 + s_19 | s_190 | s_1900 | 2030-03-17 05:46:40 | 19 | 190 | 1900 | 1900 | 1900 | 1900 | 1900 | s_19 | s_190 | s_1900 | 2030-03-17 05:46:40 | 19 | 190 | 1900 | 1900 | 1900 | 1900 | 1900 + s_20 | s_200 | s_2000 | 2033-05-18 03:33:20 | 20 | 200 | 2000 | 2000 | 2000 | 2000 | 2000 | s_20 | s_200 | s_2000 | 2033-05-18 03:33:20 | 20 | 200 | 2000 | 2000 | 2000 | 2000 | 2000 + s_21 | s_210 | s_2100 | 2036-07-18 01:20:00 | 21 | 210 | 2100 | 2100 | 2100 | 2100 | 2100 | s_21 | s_210 | s_2100 | 2036-07-18 01:20:00 | 21 | 210 | 2100 | 2100 | 2100 | 2100 | 2100 + s_22 | s_220 | s_2200 | 2039-09-18 11:06:40 | 22 | 220 | 2200 | 2200 | 2200 | 2200 | 2200 | s_22 | s_220 | s_2200 | 2039-09-18 11:06:40 | 22 | 220 | 2200 | 2200 | 2200 | 2200 | 2200 + s_23 | s_230 | s_2300 | 2042-11-19 08:53:20 | 23 | 230 | 2300 | 2300 | 2300 | 2300 | 2300 | s_23 | s_230 | s_2300 | 2042-11-19 08:53:20 | 23 | 230 | 2300 | 2300 | 2300 | 2300 | 2300 + s_24 | s_240 | s_2400 | 2046-01-19 06:40:00 | 24 | 240 | 2400 | 2400 | 2400 | 2400 | 2400 | s_24 | s_240 | s_2400 | 2046-01-19 06:40:00 | 24 | 240 | 2400 | 2400 | 2400 | 2400 | 2400 + s_25 | s_250 | s_2500 | 2049-03-22 04:26:40 | 25 | 250 | 2500 | 2500 | 2500 | 2500 | 2500 | s_25 | s_250 | s_2500 | 2049-03-22 04:26:40 | 25 | 250 | 2500 | 2500 | 2500 | 2500 | 2500 + s_26 | s_260 | s_2600 | 2052-05-22 02:13:20 | 26 | 260 | 2600 | 2600 | 2600 | 2600 | 2600 | s_26 | s_260 | s_2600 | 2052-05-22 02:13:20 | 26 | 260 | 2600 | 2600 | 2600 | 2600 | 2600 + s_27 | s_270 | s_2700 | 2055-07-24 12:00:00 | 27 | 270 | 2700 | 2700 | 2700 | 2700 | 2700 | s_27 | s_270 | s_2700 | 2055-07-24 12:00:00 | 27 | 270 | 2700 | 2700 | 2700 | 2700 | 2700 + s_28 | s_280 | s_2800 | 2058-09-23 09:46:40 | 28 | 280 | 2800 | 2800 | 2800 | 2800 | 2800 | s_28 | s_280 | s_2800 | 2058-09-23 09:46:40 | 28 | 280 | 2800 | 2800 | 2800 | 2800 | 2800 + s_29 | s_290 | s_2900 | 2061-11-23 07:33:20 | 29 | 290 | 2900 | 2900 | 2900 | 2900 | 2900 | s_29 | s_290 | s_2900 | 2061-11-23 07:33:20 | 29 | 290 | 2900 | 2900 | 2900 | 2900 | 2900 + s_30 | s_300 | s_3000 | 2065-01-24 05:20:00 | 30 | 300 | 3000 | 3000 | 3000 | 3000 | 3000 | s_30 | s_300 | s_3000 | 2065-01-24 05:20:00 | 30 | 300 | 3000 | 3000 | 3000 | 3000 | 3000 + s_31 | s_310 | s_3100 | 2068-03-26 03:06:40 | 31 | 310 | 3100 | 3100 | 3100 | 3100 | 3100 | s_31 | s_310 | s_3100 | 2068-03-26 03:06:40 | 31 | 310 | 3100 | 3100 | 3100 | 3100 | 3100 + s_32 | s_320 | s_3200 | 2071-05-28 12:53:20 | 32 | 320 | 3200 | 3200 | 3200 | 3200 | 3200 | s_32 | s_320 | s_3200 | 2071-05-28 12:53:20 | 32 | 320 | 3200 | 3200 | 3200 | 3200 | 3200 + s_33 | s_330 | s_3300 | 2074-07-28 10:40:00 | 33 | 330 | 3300 | 3300 | 3300 | 3300 | 3300 | s_33 | s_330 | s_3300 | 2074-07-28 10:40:00 | 33 | 330 | 3300 | 3300 | 3300 | 3300 | 3300 + s_34 | s_340 | s_3400 | 2077-09-27 08:26:40 | 34 | 340 | 3400 | 3400 | 3400 | 3400 | 3400 | s_34 | s_340 | s_3400 | 2077-09-27 08:26:40 | 34 | 340 | 3400 | 3400 | 3400 | 3400 | 3400 + s_35 | s_350 | s_3500 | 2080-11-28 06:13:20 | 35 | 350 | 3500 | 3500 | 3500 | 3500 | 3500 | s_35 | s_350 | s_3500 | 2080-11-28 06:13:20 | 35 | 350 | 3500 | 3500 | 3500 | 3500 | 3500 + s_36 | s_360 | s_3600 | 2084-01-29 04:00:00 | 36 | 360 | 3600 | 3600 | 3600 | 3600 | 3600 | s_36 | s_360 | s_3600 | 2084-01-29 04:00:00 | 36 | 360 | 3600 | 3600 | 3600 | 3600 | 3600 + s_37 | s_370 | s_3700 | 2087-04-01 01:46:40 | 37 | 370 | 3700 | 3700 | 3700 | 3700 | 3700 | s_37 | s_370 | s_3700 | 2087-04-01 01:46:40 | 37 | 370 | 3700 | 3700 | 3700 | 3700 | 3700 + s_38 | s_380 | s_3800 | 2090-06-01 11:33:20 | 38 | 380 | 3800 | 3800 | 3800 | 3800 | 3800 | s_38 | s_380 | s_3800 | 2090-06-01 11:33:20 | 38 | 380 | 3800 | 3800 | 3800 | 3800 | 3800 + s_39 | s_390 | s_3900 | 2093-08-01 09:20:00 | 39 | 390 | 3900 | 3900 | 3900 | 3900 | 3900 | s_39 | s_390 | s_3900 | 2093-08-01 09:20:00 | 39 | 390 | 3900 | 3900 | 3900 | 3900 | 3900 + s_40 | s_400 | s_4000 | 2096-10-02 07:06:40 | 40 | 400 | 4000 | 4000 | 4000 | 4000 | 4000 | s_40 | s_400 | s_4000 | 2096-10-02 07:06:40 | 40 | 400 | 4000 | 4000 | 4000 | 4000 | 4000 + s_41 | s_410 | s_4100 | 2099-12-03 04:53:20 | 41 | 410 | 4100 | 4100 | 4100 | 4100 | 4100 | s_41 | s_410 | s_4100 | 2099-12-03 04:53:20 | 41 | 410 | 4100 | 4100 | 4100 | 4100 | 4100 + s_42 | s_420 | s_4200 | 2103-02-04 02:40:00 | 42 | 420 | 4200 | 4200 | 4200 | 4200 | 4200 | s_42 | s_420 | s_4200 | 2103-02-04 02:40:00 | 42 | 420 | 4200 | 4200 | 4200 | 4200 | 4200 + s_43 | s_430 | s_4300 | 2106-04-06 12:26:40 | 43 | 430 | 4300 | 4300 | 4300 | 4300 | 4300 | s_43 | s_430 | s_4300 | 2106-04-06 12:26:40 | 43 | 430 | 4300 | 4300 | 4300 | 4300 | 4300 + s_44 | s_440 | s_4400 | 2109-06-06 10:13:20 | 44 | 440 | 4400 | 4400 | 4400 | 4400 | 4400 | s_44 | s_440 | s_4400 | 2109-06-06 10:13:20 | 44 | 440 | 4400 | 4400 | 4400 | 4400 | 4400 + s_45 | s_450 | s_4500 | 2112-08-07 08:00:00 | 45 | 450 | 4500 | 4500 | 4500 | 4500 | 4500 | s_45 | s_450 | s_4500 | 2112-08-07 08:00:00 | 45 | 450 | 4500 | 4500 | 4500 | 4500 | 4500 + s_46 | s_460 | s_4600 | 2115-10-08 05:46:40 | 46 | 460 | 4600 | 4600 | 4600 | 4600 | 4600 | s_46 | s_460 | s_4600 | 2115-10-08 05:46:40 | 46 | 460 | 4600 | 4600 | 4600 | 4600 | 4600 + s_47 | s_470 | s_4700 | 2118-12-09 03:33:20 | 47 | 470 | 4700 | 4700 | 4700 | 4700 | 4700 | s_47 | s_470 | s_4700 | 2118-12-09 03:33:20 | 47 | 470 | 4700 | 4700 | 4700 | 4700 | 4700 + s_48 | s_480 | s_4800 | 2122-02-08 01:20:00 | 48 | 480 | 4800 | 4800 | 4800 | 4800 | 4800 | s_48 | s_480 | s_4800 | 2122-02-08 01:20:00 | 48 | 480 | 4800 | 4800 | 4800 | 4800 | 4800 + s_49 | s_490 | s_4900 | 2125-04-10 11:06:40 | 49 | 490 | 4900 | 4900 | 4900 | 4900 | 4900 | s_49 | s_490 | s_4900 | 2125-04-10 11:06:40 | 49 | 490 | 4900 | 4900 | 4900 | 4900 | 4900 + s_50 | s_500 | s_5000 | 2128-06-11 08:53:20 | 50 | 500 | 5000 | 5000 | 5000 | 5000 | 5000 | s_50 | s_500 | s_5000 | 2128-06-11 08:53:20 | 50 | 500 | 5000 | 5000 | 5000 | 5000 | 5000 + s_51 | s_510 | s_5100 | 2131-08-12 06:40:00 | 51 | 510 | 5100 | 5100 | 5100 | 5100 | 5100 | s_51 | s_510 | s_5100 | 2131-08-12 06:40:00 | 51 | 510 | 5100 | 5100 | 5100 | 5100 | 5100 + s_52 | s_520 | s_5200 | 2134-10-13 04:26:40 | 52 | 520 | 5200 | 5200 | 5200 | 5200 | 5200 | s_52 | s_520 | s_5200 | 2134-10-13 04:26:40 | 52 | 520 | 5200 | 5200 | 5200 | 5200 | 5200 + s_53 | s_530 | s_5300 | 2137-12-13 02:13:20 | 53 | 530 | 5300 | 5300 | 5300 | 5300 | 5300 | s_53 | s_530 | s_5300 | 2137-12-13 02:13:20 | 53 | 530 | 5300 | 5300 | 5300 | 5300 | 5300 + s_54 | s_540 | s_5400 | 2141-02-13 12:00:00 | 54 | 540 | 5400 | 5400 | 5400 | 5400 | 5400 | s_54 | s_540 | s_5400 | 2141-02-13 12:00:00 | 54 | 540 | 5400 | 5400 | 5400 | 5400 | 5400 + s_55 | s_550 | s_5500 | 2144-04-15 09:46:40 | 55 | 550 | 5500 | 5500 | 5500 | 5500 | 5500 | s_55 | s_550 | s_5500 | 2144-04-15 09:46:40 | 55 | 550 | 5500 | 5500 | 5500 | 5500 | 5500 + s_56 | s_560 | s_5600 | 2147-06-16 07:33:20 | 56 | 560 | 5600 | 5600 | 5600 | 5600 | 5600 | s_56 | s_560 | s_5600 | 2147-06-16 07:33:20 | 56 | 560 | 5600 | 5600 | 5600 | 5600 | 5600 + s_57 | s_570 | s_5700 | 2150-08-17 05:20:00 | 57 | 570 | 5700 | 5700 | 5700 | 5700 | 5700 | s_57 | s_570 | s_5700 | 2150-08-17 05:20:00 | 57 | 570 | 5700 | 5700 | 5700 | 5700 | 5700 + s_58 | s_580 | s_5800 | 2153-10-17 03:06:40 | 58 | 580 | 5800 | 5800 | 5800 | 5800 | 5800 | s_58 | s_580 | s_5800 | 2153-10-17 03:06:40 | 58 | 580 | 5800 | 5800 | 5800 | 5800 | 5800 + s_59 | s_590 | s_5900 | 2156-12-18 12:53:20 | 59 | 590 | 5900 | 5900 | 5900 | 5900 | 5900 | s_59 | s_590 | s_5900 | 2156-12-18 12:53:20 | 59 | 590 | 5900 | 5900 | 5900 | 5900 | 5900 + s_60 | s_600 | s_6000 | 2160-02-18 10:40:00 | 60 | 600 | 6000 | 6000 | 6000 | 6000 | 6000 | s_60 | s_600 | s_6000 | 2160-02-18 10:40:00 | 60 | 600 | 6000 | 6000 | 6000 | 6000 | 6000 + s_61 | s_610 | s_6100 | 2163-04-20 08:26:40 | 61 | 610 | 6100 | 6100 | 6100 | 6100 | 6100 | s_61 | s_610 | s_6100 | 2163-04-20 08:26:40 | 61 | 610 | 6100 | 6100 | 6100 | 6100 | 6100 + s_62 | s_620 | s_6200 | 2166-06-21 06:13:20 | 62 | 620 | 6200 | 6200 | 6200 | 6200 | 6200 | s_62 | s_620 | s_6200 | 2166-06-21 06:13:20 | 62 | 620 | 6200 | 6200 | 6200 | 6200 | 6200 + s_63 | s_630 | s_6300 | 2169-08-21 04:00:00 | 63 | 630 | 6300 | 6300 | 6300 | 6300 | 6300 | s_63 | s_630 | s_6300 | 2169-08-21 04:00:00 | 63 | 630 | 6300 | 6300 | 6300 | 6300 | 6300 + s_64 | s_640 | s_6400 | 2172-10-22 01:46:40 | 64 | 640 | 6400 | 6400 | 6400 | 6400 | 6400 | s_64 | s_640 | s_6400 | 2172-10-22 01:46:40 | 64 | 640 | 6400 | 6400 | 6400 | 6400 | 6400 + s_65 | s_650 | s_6500 | 2175-12-23 11:33:20 | 65 | 650 | 6500 | 6500 | 6500 | 6500 | 6500 | s_65 | s_650 | s_6500 | 2175-12-23 11:33:20 | 65 | 650 | 6500 | 6500 | 6500 | 6500 | 6500 + s_66 | s_660 | s_6600 | 2179-02-22 09:20:00 | 66 | 660 | 6600 | 6600 | 6600 | 6600 | 6600 | s_66 | s_660 | s_6600 | 2179-02-22 09:20:00 | 66 | 660 | 6600 | 6600 | 6600 | 6600 | 6600 + s_67 | s_670 | s_6700 | 2182-04-25 07:06:40 | 67 | 670 | 6700 | 6700 | 6700 | 6700 | 6700 | s_67 | s_670 | s_6700 | 2182-04-25 07:06:40 | 67 | 670 | 6700 | 6700 | 6700 | 6700 | 6700 + s_68 | s_680 | s_6800 | 2185-06-25 04:53:20 | 68 | 680 | 6800 | 6800 | 6800 | 6800 | 6800 | s_68 | s_680 | s_6800 | 2185-06-25 04:53:20 | 68 | 680 | 6800 | 6800 | 6800 | 6800 | 6800 + s_69 | s_690 | s_6900 | 2188-08-26 02:40:00 | 69 | 690 | 6900 | 6900 | 6900 | 6900 | 6900 | s_69 | s_690 | s_6900 | 2188-08-26 02:40:00 | 69 | 690 | 6900 | 6900 | 6900 | 6900 | 6900 + s_70 | s_700 | s_7000 | 2191-10-27 12:26:40 | 70 | 700 | 7000 | 7000 | 7000 | 7000 | 7000 | s_70 | s_700 | s_7000 | 2191-10-27 12:26:40 | 70 | 700 | 7000 | 7000 | 7000 | 7000 | 7000 + s_71 | s_710 | s_7100 | 2194-12-27 10:13:20 | 71 | 710 | 7100 | 7100 | 7100 | 7100 | 7100 | s_71 | s_710 | s_7100 | 2194-12-27 10:13:20 | 71 | 710 | 7100 | 7100 | 7100 | 7100 | 7100 + s_72 | s_720 | s_7200 | 2198-02-27 08:00:00 | 72 | 720 | 7200 | 7200 | 7200 | 7200 | 7200 | s_72 | s_720 | s_7200 | 2198-02-27 08:00:00 | 72 | 720 | 7200 | 7200 | 7200 | 7200 | 7200 + s_73 | s_730 | s_7300 | 2201-04-30 05:46:40 | 73 | 730 | 7300 | 7300 | 7300 | 7300 | 7300 | s_73 | s_730 | s_7300 | 2201-04-30 05:46:40 | 73 | 730 | 7300 | 7300 | 7300 | 7300 | 7300 + s_74 | s_740 | s_7400 | 2204-07-01 03:33:20 | 74 | 740 | 7400 | 7400 | 7400 | 7400 | 7400 | s_74 | s_740 | s_7400 | 2204-07-01 03:33:20 | 74 | 740 | 7400 | 7400 | 7400 | 7400 | 7400 + s_75 | s_750 | s_7500 | 2207-09-01 01:20:00 | 75 | 750 | 7500 | 7500 | 7500 | 7500 | 7500 | s_75 | s_750 | s_7500 | 2207-09-01 01:20:00 | 75 | 750 | 7500 | 7500 | 7500 | 7500 | 7500 + s_76 | s_760 | s_7600 | 2210-11-01 11:06:40 | 76 | 760 | 7600 | 7600 | 7600 | 7600 | 7600 | s_76 | s_760 | s_7600 | 2210-11-01 11:06:40 | 76 | 760 | 7600 | 7600 | 7600 | 7600 | 7600 + s_77 | s_770 | s_7700 | 2214-01-02 08:53:20 | 77 | 770 | 7700 | 7700 | 7700 | 7700 | 7700 | s_77 | s_770 | s_7700 | 2214-01-02 08:53:20 | 77 | 770 | 7700 | 7700 | 7700 | 7700 | 7700 + s_78 | s_780 | s_7800 | 2217-03-04 06:40:00 | 78 | 780 | 7800 | 7800 | 7800 | 7800 | 7800 | s_78 | s_780 | s_7800 | 2217-03-04 06:40:00 | 78 | 780 | 7800 | 7800 | 7800 | 7800 | 7800 + s_79 | s_790 | s_7900 | 2220-05-05 04:26:40 | 79 | 790 | 7900 | 7900 | 7900 | 7900 | 7900 | s_79 | s_790 | s_7900 | 2220-05-05 04:26:40 | 79 | 790 | 7900 | 7900 | 7900 | 7900 | 7900 + s_80 | s_800 | s_8000 | 2223-07-06 02:13:20 | 80 | 800 | 8000 | 8000 | 8000 | 8000 | 8000 | s_80 | s_800 | s_8000 | 2223-07-06 02:13:20 | 80 | 800 | 8000 | 8000 | 8000 | 8000 | 8000 + s_81 | s_810 | s_8100 | 2226-09-06 12:00:00 | 81 | 810 | 8100 | 8100 | 8100 | 8100 | 8100 | s_81 | s_810 | s_8100 | 2226-09-06 12:00:00 | 81 | 810 | 8100 | 8100 | 8100 | 8100 | 8100 + s_82 | s_820 | s_8200 | 2229-11-06 09:46:40 | 82 | 820 | 8200 | 8200 | 8200 | 8200 | 8200 | s_82 | s_820 | s_8200 | 2229-11-06 09:46:40 | 82 | 820 | 8200 | 8200 | 8200 | 8200 | 8200 + s_83 | s_830 | s_8300 | 2233-01-06 07:33:20 | 83 | 830 | 8300 | 8300 | 8300 | 8300 | 8300 | s_83 | s_830 | s_8300 | 2233-01-06 07:33:20 | 83 | 830 | 8300 | 8300 | 8300 | 8300 | 8300 + s_84 | s_840 | s_8400 | 2236-03-09 05:20:00 | 84 | 840 | 8400 | 8400 | 8400 | 8400 | 8400 | s_84 | s_840 | s_8400 | 2236-03-09 05:20:00 | 84 | 840 | 8400 | 8400 | 8400 | 8400 | 8400 + s_85 | s_850 | s_8500 | 2239-05-10 03:06:40 | 85 | 850 | 8500 | 8500 | 8500 | 8500 | 8500 | s_85 | s_850 | s_8500 | 2239-05-10 03:06:40 | 85 | 850 | 8500 | 8500 | 8500 | 8500 | 8500 + s_86 | s_860 | s_8600 | 2242-07-11 12:53:20 | 86 | 860 | 8600 | 8600 | 8600 | 8600 | 8600 | s_86 | s_860 | s_8600 | 2242-07-11 12:53:20 | 86 | 860 | 8600 | 8600 | 8600 | 8600 | 8600 + s_87 | s_870 | s_8700 | 2245-09-10 10:40:00 | 87 | 870 | 8700 | 8700 | 8700 | 8700 | 8700 | s_87 | s_870 | s_8700 | 2245-09-10 10:40:00 | 87 | 870 | 8700 | 8700 | 8700 | 8700 | 8700 + s_88 | s_880 | s_8800 | 2248-11-10 08:26:40 | 88 | 880 | 8800 | 8800 | 8800 | 8800 | 8800 | s_88 | s_880 | s_8800 | 2248-11-10 08:26:40 | 88 | 880 | 8800 | 8800 | 8800 | 8800 | 8800 + s_89 | s_890 | s_8900 | 2252-01-12 06:13:20 | 89 | 890 | 8900 | 8900 | 8900 | 8900 | 8900 | s_89 | s_890 | s_8900 | 2252-01-12 06:13:20 | 89 | 890 | 8900 | 8900 | 8900 | 8900 | 8900 + s_90 | s_900 | s_9000 | 2255-03-14 04:00:00 | 90 | 900 | 9000 | 9000 | 9000 | 9000 | 9000 | s_90 | s_900 | s_9000 | 2255-03-14 04:00:00 | 90 | 900 | 9000 | 9000 | 9000 | 9000 | 9000 + s_91 | s_910 | s_9100 | 2258-05-15 01:46:40 | 91 | 910 | 9100 | 9100 | 9100 | 9100 | 9100 | s_91 | s_910 | s_9100 | 2258-05-15 01:46:40 | 91 | 910 | 9100 | 9100 | 9100 | 9100 | 9100 + s_92 | s_920 | s_9200 | 2261-07-15 11:33:20 | 92 | 920 | 9200 | 9200 | 9200 | 9200 | 9200 | s_92 | s_920 | s_9200 | 2261-07-15 11:33:20 | 92 | 920 | 9200 | 9200 | 9200 | 9200 | 9200 + s_93 | s_930 | s_9300 | 2264-09-14 09:20:00 | 93 | 930 | 9300 | 9300 | 9300 | 9300 | 9300 | s_93 | s_930 | s_9300 | 2264-09-14 09:20:00 | 93 | 930 | 9300 | 9300 | 9300 | 9300 | 9300 + s_94 | s_940 | s_9400 | 2267-11-16 07:06:40 | 94 | 940 | 9400 | 9400 | 9400 | 9400 | 9400 | s_94 | s_940 | s_9400 | 2267-11-16 07:06:40 | 94 | 940 | 9400 | 9400 | 9400 | 9400 | 9400 + s_95 | s_950 | s_9500 | 2271-01-16 04:53:20 | 95 | 950 | 9500 | 9500 | 9500 | 9500 | 9500 | s_95 | s_950 | s_9500 | 2271-01-16 04:53:20 | 95 | 950 | 9500 | 9500 | 9500 | 9500 | 9500 + s_96 | s_960 | s_9600 | 2274-03-19 02:40:00 | 96 | 960 | 9600 | 9600 | 9600 | 9600 | 9600 | s_96 | s_960 | s_9600 | 2274-03-19 02:40:00 | 96 | 960 | 9600 | 9600 | 9600 | 9600 | 9600 + s_97 | s_970 | s_9700 | 2277-05-19 12:26:40 | 97 | 970 | 9700 | 9700 | 9700 | 9700 | 9700 | s_97 | s_970 | s_9700 | 2277-05-19 12:26:40 | 97 | 970 | 9700 | 9700 | 9700 | 9700 | 9700 + s_98 | s_980 | s_9800 | 2280-07-19 10:13:20 | 98 | 980 | 9800 | 9800 | 9800 | 9800 | 9800 | s_98 | s_980 | s_9800 | 2280-07-19 10:13:20 | 98 | 980 | 9800 | 9800 | 9800 | 9800 | 9800 + s_99 | s_990 | s_9900 | 2283-09-20 08:00:00 | 99 | 990 | 9900 | 9900 | 9900 | 9900 | 9900 | s_99 | s_990 | s_9900 | 2283-09-20 08:00:00 | 99 | 990 | 9900 | 9900 | 9900 | 9900 | 9900 + s_100 | s_1000 | s_10000 | 2286-11-20 05:46:40 | 100 | 1000 | 10000 | 10000 | 10000 | 10000 | 10000 | s_100 | s_1000 | s_10000 | 2286-11-20 05:46:40 | 100 | 1000 | 10000 | 10000 | 10000 | 10000 | 10000 +(100 rows) diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_cr/runTest.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_cr/runTest.py new file mode 100755 index 0000000000..fbcc948d4a --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_cr/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfMultibyteTwoByteWithCRDelimiterData(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_cr/sql/query01.sql b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_cr/sql/query01.sql new file mode 100755 index 0000000000..45f1a82fe8 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_cr/sql/query01.sql @@ -0,0 +1,3 @@ +-- @description query01 for PXF Multibyte delimiter, 2-byte delim with CR case + +SELECT * from pxf_multibyte_twobyte_withcr_data ORDER BY n1; diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_crlf/__init__.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_crlf/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_crlf/expected/query01.ans b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_crlf/expected/query01.ans new file mode 100755 index 0000000000..641e450f83 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_crlf/expected/query01.ans @@ -0,0 +1,106 @@ +-- @description query01 for PXF Multibyte delimiter, 2-byte delim with CRLF case + +SELECT * from pxf_multibyte_twobyte_withcrlf_data ORDER BY n1; + s1 | s2 | s3 | d1 | n1 | n2 | n3 | n4 | n5 | n6 | n7 | s11 | s12 | s13 | d11 | n11 | n12 | n13 | n14 | n15 | n16 | n17 +-------+--------+---------+---------------------+-----+------+-------+-------+-------+-------+-------+-------+--------+---------+---------------------+-----+------+-------+-------+-------+-------+------- + s_1 | s_10 | s_100 | 1973-03-03 09:46:40 | 1 | 10 | 100 | 100 | 100 | 100 | 100 | s_1 | s_10 | s_100 | 1973-03-03 09:46:40 | 1 | 10 | 100 | 100 | 100 | 100 | 100 + s_2 | s_20 | s_200 | 1976-05-03 07:33:20 | 2 | 20 | 200 | 200 | 200 | 200 | 200 | s_2 | s_20 | s_200 | 1976-05-03 07:33:20 | 2 | 20 | 200 | 200 | 200 | 200 | 200 + s_3 | s_30 | s_300 | 1979-07-05 05:20:00 | 3 | 30 | 300 | 300 | 300 | 300 | 300 | s_3 | s_30 | s_300 | 1979-07-05 05:20:00 | 3 | 30 | 300 | 300 | 300 | 300 | 300 + s_4 | s_40 | s_400 | 1982-09-04 03:06:40 | 4 | 40 | 400 | 400 | 400 | 400 | 400 | s_4 | s_40 | s_400 | 1982-09-04 03:06:40 | 4 | 40 | 400 | 400 | 400 | 400 | 400 + s_5 | s_50 | s_500 | 1985-11-05 12:53:20 | 5 | 50 | 500 | 500 | 500 | 500 | 500 | s_5 | s_50 | s_500 | 1985-11-05 12:53:20 | 5 | 50 | 500 | 500 | 500 | 500 | 500 + s_6 | s_60 | s_600 | 1989-01-05 10:40:00 | 6 | 60 | 600 | 600 | 600 | 600 | 600 | s_6 | s_60 | s_600 | 1989-01-05 10:40:00 | 6 | 60 | 600 | 600 | 600 | 600 | 600 + s_7 | s_70 | s_700 | 1992-03-07 08:26:40 | 7 | 70 | 700 | 700 | 700 | 700 | 700 | s_7 | s_70 | s_700 | 1992-03-07 08:26:40 | 7 | 70 | 700 | 700 | 700 | 700 | 700 + s_8 | s_80 | s_800 | 1995-05-09 06:13:20 | 8 | 80 | 800 | 800 | 800 | 800 | 800 | s_8 | s_80 | s_800 | 1995-05-09 06:13:20 | 8 | 80 | 800 | 800 | 800 | 800 | 800 + s_9 | s_90 | s_900 | 1998-07-09 04:00:00 | 9 | 90 | 900 | 900 | 900 | 900 | 900 | s_9 | s_90 | s_900 | 1998-07-09 04:00:00 | 9 | 90 | 900 | 900 | 900 | 900 | 900 + s_10 | s_100 | s_1000 | 2001-09-09 01:46:40 | 10 | 100 | 1000 | 1000 | 1000 | 1000 | 1000 | s_10 | s_100 | s_1000 | 2001-09-09 01:46:40 | 10 | 100 | 1000 | 1000 | 1000 | 1000 | 1000 + s_11 | s_110 | s_1100 | 2004-11-09 11:33:20 | 11 | 110 | 1100 | 1100 | 1100 | 1100 | 1100 | s_11 | s_110 | s_1100 | 2004-11-09 11:33:20 | 11 | 110 | 1100 | 1100 | 1100 | 1100 | 1100 + s_12 | s_120 | s_1200 | 2008-01-10 09:20:00 | 12 | 120 | 1200 | 1200 | 1200 | 1200 | 1200 | s_12 | s_120 | s_1200 | 2008-01-10 09:20:00 | 12 | 120 | 1200 | 1200 | 1200 | 1200 | 1200 + s_13 | s_130 | s_1300 | 2011-03-13 07:06:40 | 13 | 130 | 1300 | 1300 | 1300 | 1300 | 1300 | s_13 | s_130 | s_1300 | 2011-03-13 07:06:40 | 13 | 130 | 1300 | 1300 | 1300 | 1300 | 1300 + s_14 | s_140 | s_1400 | 2014-05-13 04:53:20 | 14 | 140 | 1400 | 1400 | 1400 | 1400 | 1400 | s_14 | s_140 | s_1400 | 2014-05-13 04:53:20 | 14 | 140 | 1400 | 1400 | 1400 | 1400 | 1400 + s_15 | s_150 | s_1500 | 2017-07-14 02:40:00 | 15 | 150 | 1500 | 1500 | 1500 | 1500 | 1500 | s_15 | s_150 | s_1500 | 2017-07-14 02:40:00 | 15 | 150 | 1500 | 1500 | 1500 | 1500 | 1500 + s_16 | s_160 | s_1600 | 2020-09-13 12:26:40 | 16 | 160 | 1600 | 1600 | 1600 | 1600 | 1600 | s_16 | s_160 | s_1600 | 2020-09-13 12:26:40 | 16 | 160 | 1600 | 1600 | 1600 | 1600 | 1600 + s_17 | s_170 | s_1700 | 2023-11-14 10:13:20 | 17 | 170 | 1700 | 1700 | 1700 | 1700 | 1700 | s_17 | s_170 | s_1700 | 2023-11-14 10:13:20 | 17 | 170 | 1700 | 1700 | 1700 | 1700 | 1700 + s_18 | s_180 | s_1800 | 2027-01-15 08:00:00 | 18 | 180 | 1800 | 1800 | 1800 | 1800 | 1800 | s_18 | s_180 | s_1800 | 2027-01-15 08:00:00 | 18 | 180 | 1800 | 1800 | 1800 | 1800 | 1800 + s_19 | s_190 | s_1900 | 2030-03-17 05:46:40 | 19 | 190 | 1900 | 1900 | 1900 | 1900 | 1900 | s_19 | s_190 | s_1900 | 2030-03-17 05:46:40 | 19 | 190 | 1900 | 1900 | 1900 | 1900 | 1900 + s_20 | s_200 | s_2000 | 2033-05-18 03:33:20 | 20 | 200 | 2000 | 2000 | 2000 | 2000 | 2000 | s_20 | s_200 | s_2000 | 2033-05-18 03:33:20 | 20 | 200 | 2000 | 2000 | 2000 | 2000 | 2000 + s_21 | s_210 | s_2100 | 2036-07-18 01:20:00 | 21 | 210 | 2100 | 2100 | 2100 | 2100 | 2100 | s_21 | s_210 | s_2100 | 2036-07-18 01:20:00 | 21 | 210 | 2100 | 2100 | 2100 | 2100 | 2100 + s_22 | s_220 | s_2200 | 2039-09-18 11:06:40 | 22 | 220 | 2200 | 2200 | 2200 | 2200 | 2200 | s_22 | s_220 | s_2200 | 2039-09-18 11:06:40 | 22 | 220 | 2200 | 2200 | 2200 | 2200 | 2200 + s_23 | s_230 | s_2300 | 2042-11-19 08:53:20 | 23 | 230 | 2300 | 2300 | 2300 | 2300 | 2300 | s_23 | s_230 | s_2300 | 2042-11-19 08:53:20 | 23 | 230 | 2300 | 2300 | 2300 | 2300 | 2300 + s_24 | s_240 | s_2400 | 2046-01-19 06:40:00 | 24 | 240 | 2400 | 2400 | 2400 | 2400 | 2400 | s_24 | s_240 | s_2400 | 2046-01-19 06:40:00 | 24 | 240 | 2400 | 2400 | 2400 | 2400 | 2400 + s_25 | s_250 | s_2500 | 2049-03-22 04:26:40 | 25 | 250 | 2500 | 2500 | 2500 | 2500 | 2500 | s_25 | s_250 | s_2500 | 2049-03-22 04:26:40 | 25 | 250 | 2500 | 2500 | 2500 | 2500 | 2500 + s_26 | s_260 | s_2600 | 2052-05-22 02:13:20 | 26 | 260 | 2600 | 2600 | 2600 | 2600 | 2600 | s_26 | s_260 | s_2600 | 2052-05-22 02:13:20 | 26 | 260 | 2600 | 2600 | 2600 | 2600 | 2600 + s_27 | s_270 | s_2700 | 2055-07-24 12:00:00 | 27 | 270 | 2700 | 2700 | 2700 | 2700 | 2700 | s_27 | s_270 | s_2700 | 2055-07-24 12:00:00 | 27 | 270 | 2700 | 2700 | 2700 | 2700 | 2700 + s_28 | s_280 | s_2800 | 2058-09-23 09:46:40 | 28 | 280 | 2800 | 2800 | 2800 | 2800 | 2800 | s_28 | s_280 | s_2800 | 2058-09-23 09:46:40 | 28 | 280 | 2800 | 2800 | 2800 | 2800 | 2800 + s_29 | s_290 | s_2900 | 2061-11-23 07:33:20 | 29 | 290 | 2900 | 2900 | 2900 | 2900 | 2900 | s_29 | s_290 | s_2900 | 2061-11-23 07:33:20 | 29 | 290 | 2900 | 2900 | 2900 | 2900 | 2900 + s_30 | s_300 | s_3000 | 2065-01-24 05:20:00 | 30 | 300 | 3000 | 3000 | 3000 | 3000 | 3000 | s_30 | s_300 | s_3000 | 2065-01-24 05:20:00 | 30 | 300 | 3000 | 3000 | 3000 | 3000 | 3000 + s_31 | s_310 | s_3100 | 2068-03-26 03:06:40 | 31 | 310 | 3100 | 3100 | 3100 | 3100 | 3100 | s_31 | s_310 | s_3100 | 2068-03-26 03:06:40 | 31 | 310 | 3100 | 3100 | 3100 | 3100 | 3100 + s_32 | s_320 | s_3200 | 2071-05-28 12:53:20 | 32 | 320 | 3200 | 3200 | 3200 | 3200 | 3200 | s_32 | s_320 | s_3200 | 2071-05-28 12:53:20 | 32 | 320 | 3200 | 3200 | 3200 | 3200 | 3200 + s_33 | s_330 | s_3300 | 2074-07-28 10:40:00 | 33 | 330 | 3300 | 3300 | 3300 | 3300 | 3300 | s_33 | s_330 | s_3300 | 2074-07-28 10:40:00 | 33 | 330 | 3300 | 3300 | 3300 | 3300 | 3300 + s_34 | s_340 | s_3400 | 2077-09-27 08:26:40 | 34 | 340 | 3400 | 3400 | 3400 | 3400 | 3400 | s_34 | s_340 | s_3400 | 2077-09-27 08:26:40 | 34 | 340 | 3400 | 3400 | 3400 | 3400 | 3400 + s_35 | s_350 | s_3500 | 2080-11-28 06:13:20 | 35 | 350 | 3500 | 3500 | 3500 | 3500 | 3500 | s_35 | s_350 | s_3500 | 2080-11-28 06:13:20 | 35 | 350 | 3500 | 3500 | 3500 | 3500 | 3500 + s_36 | s_360 | s_3600 | 2084-01-29 04:00:00 | 36 | 360 | 3600 | 3600 | 3600 | 3600 | 3600 | s_36 | s_360 | s_3600 | 2084-01-29 04:00:00 | 36 | 360 | 3600 | 3600 | 3600 | 3600 | 3600 + s_37 | s_370 | s_3700 | 2087-04-01 01:46:40 | 37 | 370 | 3700 | 3700 | 3700 | 3700 | 3700 | s_37 | s_370 | s_3700 | 2087-04-01 01:46:40 | 37 | 370 | 3700 | 3700 | 3700 | 3700 | 3700 + s_38 | s_380 | s_3800 | 2090-06-01 11:33:20 | 38 | 380 | 3800 | 3800 | 3800 | 3800 | 3800 | s_38 | s_380 | s_3800 | 2090-06-01 11:33:20 | 38 | 380 | 3800 | 3800 | 3800 | 3800 | 3800 + s_39 | s_390 | s_3900 | 2093-08-01 09:20:00 | 39 | 390 | 3900 | 3900 | 3900 | 3900 | 3900 | s_39 | s_390 | s_3900 | 2093-08-01 09:20:00 | 39 | 390 | 3900 | 3900 | 3900 | 3900 | 3900 + s_40 | s_400 | s_4000 | 2096-10-02 07:06:40 | 40 | 400 | 4000 | 4000 | 4000 | 4000 | 4000 | s_40 | s_400 | s_4000 | 2096-10-02 07:06:40 | 40 | 400 | 4000 | 4000 | 4000 | 4000 | 4000 + s_41 | s_410 | s_4100 | 2099-12-03 04:53:20 | 41 | 410 | 4100 | 4100 | 4100 | 4100 | 4100 | s_41 | s_410 | s_4100 | 2099-12-03 04:53:20 | 41 | 410 | 4100 | 4100 | 4100 | 4100 | 4100 + s_42 | s_420 | s_4200 | 2103-02-04 02:40:00 | 42 | 420 | 4200 | 4200 | 4200 | 4200 | 4200 | s_42 | s_420 | s_4200 | 2103-02-04 02:40:00 | 42 | 420 | 4200 | 4200 | 4200 | 4200 | 4200 + s_43 | s_430 | s_4300 | 2106-04-06 12:26:40 | 43 | 430 | 4300 | 4300 | 4300 | 4300 | 4300 | s_43 | s_430 | s_4300 | 2106-04-06 12:26:40 | 43 | 430 | 4300 | 4300 | 4300 | 4300 | 4300 + s_44 | s_440 | s_4400 | 2109-06-06 10:13:20 | 44 | 440 | 4400 | 4400 | 4400 | 4400 | 4400 | s_44 | s_440 | s_4400 | 2109-06-06 10:13:20 | 44 | 440 | 4400 | 4400 | 4400 | 4400 | 4400 + s_45 | s_450 | s_4500 | 2112-08-07 08:00:00 | 45 | 450 | 4500 | 4500 | 4500 | 4500 | 4500 | s_45 | s_450 | s_4500 | 2112-08-07 08:00:00 | 45 | 450 | 4500 | 4500 | 4500 | 4500 | 4500 + s_46 | s_460 | s_4600 | 2115-10-08 05:46:40 | 46 | 460 | 4600 | 4600 | 4600 | 4600 | 4600 | s_46 | s_460 | s_4600 | 2115-10-08 05:46:40 | 46 | 460 | 4600 | 4600 | 4600 | 4600 | 4600 + s_47 | s_470 | s_4700 | 2118-12-09 03:33:20 | 47 | 470 | 4700 | 4700 | 4700 | 4700 | 4700 | s_47 | s_470 | s_4700 | 2118-12-09 03:33:20 | 47 | 470 | 4700 | 4700 | 4700 | 4700 | 4700 + s_48 | s_480 | s_4800 | 2122-02-08 01:20:00 | 48 | 480 | 4800 | 4800 | 4800 | 4800 | 4800 | s_48 | s_480 | s_4800 | 2122-02-08 01:20:00 | 48 | 480 | 4800 | 4800 | 4800 | 4800 | 4800 + s_49 | s_490 | s_4900 | 2125-04-10 11:06:40 | 49 | 490 | 4900 | 4900 | 4900 | 4900 | 4900 | s_49 | s_490 | s_4900 | 2125-04-10 11:06:40 | 49 | 490 | 4900 | 4900 | 4900 | 4900 | 4900 + s_50 | s_500 | s_5000 | 2128-06-11 08:53:20 | 50 | 500 | 5000 | 5000 | 5000 | 5000 | 5000 | s_50 | s_500 | s_5000 | 2128-06-11 08:53:20 | 50 | 500 | 5000 | 5000 | 5000 | 5000 | 5000 + s_51 | s_510 | s_5100 | 2131-08-12 06:40:00 | 51 | 510 | 5100 | 5100 | 5100 | 5100 | 5100 | s_51 | s_510 | s_5100 | 2131-08-12 06:40:00 | 51 | 510 | 5100 | 5100 | 5100 | 5100 | 5100 + s_52 | s_520 | s_5200 | 2134-10-13 04:26:40 | 52 | 520 | 5200 | 5200 | 5200 | 5200 | 5200 | s_52 | s_520 | s_5200 | 2134-10-13 04:26:40 | 52 | 520 | 5200 | 5200 | 5200 | 5200 | 5200 + s_53 | s_530 | s_5300 | 2137-12-13 02:13:20 | 53 | 530 | 5300 | 5300 | 5300 | 5300 | 5300 | s_53 | s_530 | s_5300 | 2137-12-13 02:13:20 | 53 | 530 | 5300 | 5300 | 5300 | 5300 | 5300 + s_54 | s_540 | s_5400 | 2141-02-13 12:00:00 | 54 | 540 | 5400 | 5400 | 5400 | 5400 | 5400 | s_54 | s_540 | s_5400 | 2141-02-13 12:00:00 | 54 | 540 | 5400 | 5400 | 5400 | 5400 | 5400 + s_55 | s_550 | s_5500 | 2144-04-15 09:46:40 | 55 | 550 | 5500 | 5500 | 5500 | 5500 | 5500 | s_55 | s_550 | s_5500 | 2144-04-15 09:46:40 | 55 | 550 | 5500 | 5500 | 5500 | 5500 | 5500 + s_56 | s_560 | s_5600 | 2147-06-16 07:33:20 | 56 | 560 | 5600 | 5600 | 5600 | 5600 | 5600 | s_56 | s_560 | s_5600 | 2147-06-16 07:33:20 | 56 | 560 | 5600 | 5600 | 5600 | 5600 | 5600 + s_57 | s_570 | s_5700 | 2150-08-17 05:20:00 | 57 | 570 | 5700 | 5700 | 5700 | 5700 | 5700 | s_57 | s_570 | s_5700 | 2150-08-17 05:20:00 | 57 | 570 | 5700 | 5700 | 5700 | 5700 | 5700 + s_58 | s_580 | s_5800 | 2153-10-17 03:06:40 | 58 | 580 | 5800 | 5800 | 5800 | 5800 | 5800 | s_58 | s_580 | s_5800 | 2153-10-17 03:06:40 | 58 | 580 | 5800 | 5800 | 5800 | 5800 | 5800 + s_59 | s_590 | s_5900 | 2156-12-18 12:53:20 | 59 | 590 | 5900 | 5900 | 5900 | 5900 | 5900 | s_59 | s_590 | s_5900 | 2156-12-18 12:53:20 | 59 | 590 | 5900 | 5900 | 5900 | 5900 | 5900 + s_60 | s_600 | s_6000 | 2160-02-18 10:40:00 | 60 | 600 | 6000 | 6000 | 6000 | 6000 | 6000 | s_60 | s_600 | s_6000 | 2160-02-18 10:40:00 | 60 | 600 | 6000 | 6000 | 6000 | 6000 | 6000 + s_61 | s_610 | s_6100 | 2163-04-20 08:26:40 | 61 | 610 | 6100 | 6100 | 6100 | 6100 | 6100 | s_61 | s_610 | s_6100 | 2163-04-20 08:26:40 | 61 | 610 | 6100 | 6100 | 6100 | 6100 | 6100 + s_62 | s_620 | s_6200 | 2166-06-21 06:13:20 | 62 | 620 | 6200 | 6200 | 6200 | 6200 | 6200 | s_62 | s_620 | s_6200 | 2166-06-21 06:13:20 | 62 | 620 | 6200 | 6200 | 6200 | 6200 | 6200 + s_63 | s_630 | s_6300 | 2169-08-21 04:00:00 | 63 | 630 | 6300 | 6300 | 6300 | 6300 | 6300 | s_63 | s_630 | s_6300 | 2169-08-21 04:00:00 | 63 | 630 | 6300 | 6300 | 6300 | 6300 | 6300 + s_64 | s_640 | s_6400 | 2172-10-22 01:46:40 | 64 | 640 | 6400 | 6400 | 6400 | 6400 | 6400 | s_64 | s_640 | s_6400 | 2172-10-22 01:46:40 | 64 | 640 | 6400 | 6400 | 6400 | 6400 | 6400 + s_65 | s_650 | s_6500 | 2175-12-23 11:33:20 | 65 | 650 | 6500 | 6500 | 6500 | 6500 | 6500 | s_65 | s_650 | s_6500 | 2175-12-23 11:33:20 | 65 | 650 | 6500 | 6500 | 6500 | 6500 | 6500 + s_66 | s_660 | s_6600 | 2179-02-22 09:20:00 | 66 | 660 | 6600 | 6600 | 6600 | 6600 | 6600 | s_66 | s_660 | s_6600 | 2179-02-22 09:20:00 | 66 | 660 | 6600 | 6600 | 6600 | 6600 | 6600 + s_67 | s_670 | s_6700 | 2182-04-25 07:06:40 | 67 | 670 | 6700 | 6700 | 6700 | 6700 | 6700 | s_67 | s_670 | s_6700 | 2182-04-25 07:06:40 | 67 | 670 | 6700 | 6700 | 6700 | 6700 | 6700 + s_68 | s_680 | s_6800 | 2185-06-25 04:53:20 | 68 | 680 | 6800 | 6800 | 6800 | 6800 | 6800 | s_68 | s_680 | s_6800 | 2185-06-25 04:53:20 | 68 | 680 | 6800 | 6800 | 6800 | 6800 | 6800 + s_69 | s_690 | s_6900 | 2188-08-26 02:40:00 | 69 | 690 | 6900 | 6900 | 6900 | 6900 | 6900 | s_69 | s_690 | s_6900 | 2188-08-26 02:40:00 | 69 | 690 | 6900 | 6900 | 6900 | 6900 | 6900 + s_70 | s_700 | s_7000 | 2191-10-27 12:26:40 | 70 | 700 | 7000 | 7000 | 7000 | 7000 | 7000 | s_70 | s_700 | s_7000 | 2191-10-27 12:26:40 | 70 | 700 | 7000 | 7000 | 7000 | 7000 | 7000 + s_71 | s_710 | s_7100 | 2194-12-27 10:13:20 | 71 | 710 | 7100 | 7100 | 7100 | 7100 | 7100 | s_71 | s_710 | s_7100 | 2194-12-27 10:13:20 | 71 | 710 | 7100 | 7100 | 7100 | 7100 | 7100 + s_72 | s_720 | s_7200 | 2198-02-27 08:00:00 | 72 | 720 | 7200 | 7200 | 7200 | 7200 | 7200 | s_72 | s_720 | s_7200 | 2198-02-27 08:00:00 | 72 | 720 | 7200 | 7200 | 7200 | 7200 | 7200 + s_73 | s_730 | s_7300 | 2201-04-30 05:46:40 | 73 | 730 | 7300 | 7300 | 7300 | 7300 | 7300 | s_73 | s_730 | s_7300 | 2201-04-30 05:46:40 | 73 | 730 | 7300 | 7300 | 7300 | 7300 | 7300 + s_74 | s_740 | s_7400 | 2204-07-01 03:33:20 | 74 | 740 | 7400 | 7400 | 7400 | 7400 | 7400 | s_74 | s_740 | s_7400 | 2204-07-01 03:33:20 | 74 | 740 | 7400 | 7400 | 7400 | 7400 | 7400 + s_75 | s_750 | s_7500 | 2207-09-01 01:20:00 | 75 | 750 | 7500 | 7500 | 7500 | 7500 | 7500 | s_75 | s_750 | s_7500 | 2207-09-01 01:20:00 | 75 | 750 | 7500 | 7500 | 7500 | 7500 | 7500 + s_76 | s_760 | s_7600 | 2210-11-01 11:06:40 | 76 | 760 | 7600 | 7600 | 7600 | 7600 | 7600 | s_76 | s_760 | s_7600 | 2210-11-01 11:06:40 | 76 | 760 | 7600 | 7600 | 7600 | 7600 | 7600 + s_77 | s_770 | s_7700 | 2214-01-02 08:53:20 | 77 | 770 | 7700 | 7700 | 7700 | 7700 | 7700 | s_77 | s_770 | s_7700 | 2214-01-02 08:53:20 | 77 | 770 | 7700 | 7700 | 7700 | 7700 | 7700 + s_78 | s_780 | s_7800 | 2217-03-04 06:40:00 | 78 | 780 | 7800 | 7800 | 7800 | 7800 | 7800 | s_78 | s_780 | s_7800 | 2217-03-04 06:40:00 | 78 | 780 | 7800 | 7800 | 7800 | 7800 | 7800 + s_79 | s_790 | s_7900 | 2220-05-05 04:26:40 | 79 | 790 | 7900 | 7900 | 7900 | 7900 | 7900 | s_79 | s_790 | s_7900 | 2220-05-05 04:26:40 | 79 | 790 | 7900 | 7900 | 7900 | 7900 | 7900 + s_80 | s_800 | s_8000 | 2223-07-06 02:13:20 | 80 | 800 | 8000 | 8000 | 8000 | 8000 | 8000 | s_80 | s_800 | s_8000 | 2223-07-06 02:13:20 | 80 | 800 | 8000 | 8000 | 8000 | 8000 | 8000 + s_81 | s_810 | s_8100 | 2226-09-06 12:00:00 | 81 | 810 | 8100 | 8100 | 8100 | 8100 | 8100 | s_81 | s_810 | s_8100 | 2226-09-06 12:00:00 | 81 | 810 | 8100 | 8100 | 8100 | 8100 | 8100 + s_82 | s_820 | s_8200 | 2229-11-06 09:46:40 | 82 | 820 | 8200 | 8200 | 8200 | 8200 | 8200 | s_82 | s_820 | s_8200 | 2229-11-06 09:46:40 | 82 | 820 | 8200 | 8200 | 8200 | 8200 | 8200 + s_83 | s_830 | s_8300 | 2233-01-06 07:33:20 | 83 | 830 | 8300 | 8300 | 8300 | 8300 | 8300 | s_83 | s_830 | s_8300 | 2233-01-06 07:33:20 | 83 | 830 | 8300 | 8300 | 8300 | 8300 | 8300 + s_84 | s_840 | s_8400 | 2236-03-09 05:20:00 | 84 | 840 | 8400 | 8400 | 8400 | 8400 | 8400 | s_84 | s_840 | s_8400 | 2236-03-09 05:20:00 | 84 | 840 | 8400 | 8400 | 8400 | 8400 | 8400 + s_85 | s_850 | s_8500 | 2239-05-10 03:06:40 | 85 | 850 | 8500 | 8500 | 8500 | 8500 | 8500 | s_85 | s_850 | s_8500 | 2239-05-10 03:06:40 | 85 | 850 | 8500 | 8500 | 8500 | 8500 | 8500 + s_86 | s_860 | s_8600 | 2242-07-11 12:53:20 | 86 | 860 | 8600 | 8600 | 8600 | 8600 | 8600 | s_86 | s_860 | s_8600 | 2242-07-11 12:53:20 | 86 | 860 | 8600 | 8600 | 8600 | 8600 | 8600 + s_87 | s_870 | s_8700 | 2245-09-10 10:40:00 | 87 | 870 | 8700 | 8700 | 8700 | 8700 | 8700 | s_87 | s_870 | s_8700 | 2245-09-10 10:40:00 | 87 | 870 | 8700 | 8700 | 8700 | 8700 | 8700 + s_88 | s_880 | s_8800 | 2248-11-10 08:26:40 | 88 | 880 | 8800 | 8800 | 8800 | 8800 | 8800 | s_88 | s_880 | s_8800 | 2248-11-10 08:26:40 | 88 | 880 | 8800 | 8800 | 8800 | 8800 | 8800 + s_89 | s_890 | s_8900 | 2252-01-12 06:13:20 | 89 | 890 | 8900 | 8900 | 8900 | 8900 | 8900 | s_89 | s_890 | s_8900 | 2252-01-12 06:13:20 | 89 | 890 | 8900 | 8900 | 8900 | 8900 | 8900 + s_90 | s_900 | s_9000 | 2255-03-14 04:00:00 | 90 | 900 | 9000 | 9000 | 9000 | 9000 | 9000 | s_90 | s_900 | s_9000 | 2255-03-14 04:00:00 | 90 | 900 | 9000 | 9000 | 9000 | 9000 | 9000 + s_91 | s_910 | s_9100 | 2258-05-15 01:46:40 | 91 | 910 | 9100 | 9100 | 9100 | 9100 | 9100 | s_91 | s_910 | s_9100 | 2258-05-15 01:46:40 | 91 | 910 | 9100 | 9100 | 9100 | 9100 | 9100 + s_92 | s_920 | s_9200 | 2261-07-15 11:33:20 | 92 | 920 | 9200 | 9200 | 9200 | 9200 | 9200 | s_92 | s_920 | s_9200 | 2261-07-15 11:33:20 | 92 | 920 | 9200 | 9200 | 9200 | 9200 | 9200 + s_93 | s_930 | s_9300 | 2264-09-14 09:20:00 | 93 | 930 | 9300 | 9300 | 9300 | 9300 | 9300 | s_93 | s_930 | s_9300 | 2264-09-14 09:20:00 | 93 | 930 | 9300 | 9300 | 9300 | 9300 | 9300 + s_94 | s_940 | s_9400 | 2267-11-16 07:06:40 | 94 | 940 | 9400 | 9400 | 9400 | 9400 | 9400 | s_94 | s_940 | s_9400 | 2267-11-16 07:06:40 | 94 | 940 | 9400 | 9400 | 9400 | 9400 | 9400 + s_95 | s_950 | s_9500 | 2271-01-16 04:53:20 | 95 | 950 | 9500 | 9500 | 9500 | 9500 | 9500 | s_95 | s_950 | s_9500 | 2271-01-16 04:53:20 | 95 | 950 | 9500 | 9500 | 9500 | 9500 | 9500 + s_96 | s_960 | s_9600 | 2274-03-19 02:40:00 | 96 | 960 | 9600 | 9600 | 9600 | 9600 | 9600 | s_96 | s_960 | s_9600 | 2274-03-19 02:40:00 | 96 | 960 | 9600 | 9600 | 9600 | 9600 | 9600 + s_97 | s_970 | s_9700 | 2277-05-19 12:26:40 | 97 | 970 | 9700 | 9700 | 9700 | 9700 | 9700 | s_97 | s_970 | s_9700 | 2277-05-19 12:26:40 | 97 | 970 | 9700 | 9700 | 9700 | 9700 | 9700 + s_98 | s_980 | s_9800 | 2280-07-19 10:13:20 | 98 | 980 | 9800 | 9800 | 9800 | 9800 | 9800 | s_98 | s_980 | s_9800 | 2280-07-19 10:13:20 | 98 | 980 | 9800 | 9800 | 9800 | 9800 | 9800 + s_99 | s_990 | s_9900 | 2283-09-20 08:00:00 | 99 | 990 | 9900 | 9900 | 9900 | 9900 | 9900 | s_99 | s_990 | s_9900 | 2283-09-20 08:00:00 | 99 | 990 | 9900 | 9900 | 9900 | 9900 | 9900 + s_100 | s_1000 | s_10000 | 2286-11-20 05:46:40 | 100 | 1000 | 10000 | 10000 | 10000 | 10000 | 10000 | s_100 | s_1000 | s_10000 | 2286-11-20 05:46:40 | 100 | 1000 | 10000 | 10000 | 10000 | 10000 | 10000 +(100 rows) diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_crlf/runTest.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_crlf/runTest.py new file mode 100755 index 0000000000..b9e7de4f73 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_crlf/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfMultibyteTwoByteWithCRLFDelimiterData(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_crlf/sql/query01.sql b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_crlf/sql/query01.sql new file mode 100755 index 0000000000..806fdf6715 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_crlf/sql/query01.sql @@ -0,0 +1,3 @@ +-- @description query01 for PXF Multibyte delimiter, 2-byte delim with CRLF case + +SELECT * from pxf_multibyte_twobyte_withcrlf_data ORDER BY n1; diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_quote/__init__.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_quote/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_quote/expected/query01.ans b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_quote/expected/query01.ans new file mode 100755 index 0000000000..18d1709d50 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_quote/expected/query01.ans @@ -0,0 +1,106 @@ +-- @description query01 for PXF Multibyte delimiter, 2-byte delim with quote cases + +SELECT * from pxf_multibyte_twobyte_withquote_data ORDER BY n1; + s1 | s2 | s3 | d1 | n1 | n2 | n3 | n4 | n5 | n6 | n7 | s11 | s12 | s13 | d11 | n11 | n12 | n13 | n14 | n15 | n16 | n17 +-------+--------+---------+---------------------+-----+------+-------+-------+-------+-------+-------+-------+--------+---------+---------------------+-----+------+-------+-------+-------+-------+------- + s_1 | s_10 | s_100 | 1973-03-03 09:46:40 | 1 | 10 | 100 | 100 | 100 | 100 | 100 | s_1 | s_10 | s_100 | 1973-03-03 09:46:40 | 1 | 10 | 100 | 100 | 100 | 100 | 100 + s_2 | s_20 | s_200 | 1976-05-03 07:33:20 | 2 | 20 | 200 | 200 | 200 | 200 | 200 | s_2 | s_20 | s_200 | 1976-05-03 07:33:20 | 2 | 20 | 200 | 200 | 200 | 200 | 200 + s_3 | s_30 | s_300 | 1979-07-05 05:20:00 | 3 | 30 | 300 | 300 | 300 | 300 | 300 | s_3 | s_30 | s_300 | 1979-07-05 05:20:00 | 3 | 30 | 300 | 300 | 300 | 300 | 300 + s_4 | s_40 | s_400 | 1982-09-04 03:06:40 | 4 | 40 | 400 | 400 | 400 | 400 | 400 | s_4 | s_40 | s_400 | 1982-09-04 03:06:40 | 4 | 40 | 400 | 400 | 400 | 400 | 400 + s_5 | s_50 | s_500 | 1985-11-05 12:53:20 | 5 | 50 | 500 | 500 | 500 | 500 | 500 | s_5 | s_50 | s_500 | 1985-11-05 12:53:20 | 5 | 50 | 500 | 500 | 500 | 500 | 500 + s_6 | s_60 | s_600 | 1989-01-05 10:40:00 | 6 | 60 | 600 | 600 | 600 | 600 | 600 | s_6 | s_60 | s_600 | 1989-01-05 10:40:00 | 6 | 60 | 600 | 600 | 600 | 600 | 600 + s_7 | s_70 | s_700 | 1992-03-07 08:26:40 | 7 | 70 | 700 | 700 | 700 | 700 | 700 | s_7 | s_70 | s_700 | 1992-03-07 08:26:40 | 7 | 70 | 700 | 700 | 700 | 700 | 700 + s_8 | s_80 | s_800 | 1995-05-09 06:13:20 | 8 | 80 | 800 | 800 | 800 | 800 | 800 | s_8 | s_80 | s_800 | 1995-05-09 06:13:20 | 8 | 80 | 800 | 800 | 800 | 800 | 800 + s_9 | s_90 | s_900 | 1998-07-09 04:00:00 | 9 | 90 | 900 | 900 | 900 | 900 | 900 | s_9 | s_90 | s_900 | 1998-07-09 04:00:00 | 9 | 90 | 900 | 900 | 900 | 900 | 900 + s_10 | s_100 | s_1000 | 2001-09-09 01:46:40 | 10 | 100 | 1000 | 1000 | 1000 | 1000 | 1000 | s_10 | s_100 | s_1000 | 2001-09-09 01:46:40 | 10 | 100 | 1000 | 1000 | 1000 | 1000 | 1000 + s_11 | s_110 | s_1100 | 2004-11-09 11:33:20 | 11 | 110 | 1100 | 1100 | 1100 | 1100 | 1100 | s_11 | s_110 | s_1100 | 2004-11-09 11:33:20 | 11 | 110 | 1100 | 1100 | 1100 | 1100 | 1100 + s_12 | s_120 | s_1200 | 2008-01-10 09:20:00 | 12 | 120 | 1200 | 1200 | 1200 | 1200 | 1200 | s_12 | s_120 | s_1200 | 2008-01-10 09:20:00 | 12 | 120 | 1200 | 1200 | 1200 | 1200 | 1200 + s_13 | s_130 | s_1300 | 2011-03-13 07:06:40 | 13 | 130 | 1300 | 1300 | 1300 | 1300 | 1300 | s_13 | s_130 | s_1300 | 2011-03-13 07:06:40 | 13 | 130 | 1300 | 1300 | 1300 | 1300 | 1300 + s_14 | s_140 | s_1400 | 2014-05-13 04:53:20 | 14 | 140 | 1400 | 1400 | 1400 | 1400 | 1400 | s_14 | s_140 | s_1400 | 2014-05-13 04:53:20 | 14 | 140 | 1400 | 1400 | 1400 | 1400 | 1400 + s_15 | s_150 | s_1500 | 2017-07-14 02:40:00 | 15 | 150 | 1500 | 1500 | 1500 | 1500 | 1500 | s_15 | s_150 | s_1500 | 2017-07-14 02:40:00 | 15 | 150 | 1500 | 1500 | 1500 | 1500 | 1500 + s_16 | s_160 | s_1600 | 2020-09-13 12:26:40 | 16 | 160 | 1600 | 1600 | 1600 | 1600 | 1600 | s_16 | s_160 | s_1600 | 2020-09-13 12:26:40 | 16 | 160 | 1600 | 1600 | 1600 | 1600 | 1600 + s_17 | s_170 | s_1700 | 2023-11-14 10:13:20 | 17 | 170 | 1700 | 1700 | 1700 | 1700 | 1700 | s_17 | s_170 | s_1700 | 2023-11-14 10:13:20 | 17 | 170 | 1700 | 1700 | 1700 | 1700 | 1700 + s_18 | s_180 | s_1800 | 2027-01-15 08:00:00 | 18 | 180 | 1800 | 1800 | 1800 | 1800 | 1800 | s_18 | s_180 | s_1800 | 2027-01-15 08:00:00 | 18 | 180 | 1800 | 1800 | 1800 | 1800 | 1800 + s_19 | s_190 | s_1900 | 2030-03-17 05:46:40 | 19 | 190 | 1900 | 1900 | 1900 | 1900 | 1900 | s_19 | s_190 | s_1900 | 2030-03-17 05:46:40 | 19 | 190 | 1900 | 1900 | 1900 | 1900 | 1900 + s_20 | s_200 | s_2000 | 2033-05-18 03:33:20 | 20 | 200 | 2000 | 2000 | 2000 | 2000 | 2000 | s_20 | s_200 | s_2000 | 2033-05-18 03:33:20 | 20 | 200 | 2000 | 2000 | 2000 | 2000 | 2000 + s_21 | s_210 | s_2100 | 2036-07-18 01:20:00 | 21 | 210 | 2100 | 2100 | 2100 | 2100 | 2100 | s_21 | s_210 | s_2100 | 2036-07-18 01:20:00 | 21 | 210 | 2100 | 2100 | 2100 | 2100 | 2100 + s_22 | s_220 | s_2200 | 2039-09-18 11:06:40 | 22 | 220 | 2200 | 2200 | 2200 | 2200 | 2200 | s_22 | s_220 | s_2200 | 2039-09-18 11:06:40 | 22 | 220 | 2200 | 2200 | 2200 | 2200 | 2200 + s_23 | s_230 | s_2300 | 2042-11-19 08:53:20 | 23 | 230 | 2300 | 2300 | 2300 | 2300 | 2300 | s_23 | s_230 | s_2300 | 2042-11-19 08:53:20 | 23 | 230 | 2300 | 2300 | 2300 | 2300 | 2300 + s_24 | s_240 | s_2400 | 2046-01-19 06:40:00 | 24 | 240 | 2400 | 2400 | 2400 | 2400 | 2400 | s_24 | s_240 | s_2400 | 2046-01-19 06:40:00 | 24 | 240 | 2400 | 2400 | 2400 | 2400 | 2400 + s_25 | s_250 | s_2500 | 2049-03-22 04:26:40 | 25 | 250 | 2500 | 2500 | 2500 | 2500 | 2500 | s_25 | s_250 | s_2500 | 2049-03-22 04:26:40 | 25 | 250 | 2500 | 2500 | 2500 | 2500 | 2500 + s_26 | s_260 | s_2600 | 2052-05-22 02:13:20 | 26 | 260 | 2600 | 2600 | 2600 | 2600 | 2600 | s_26 | s_260 | s_2600 | 2052-05-22 02:13:20 | 26 | 260 | 2600 | 2600 | 2600 | 2600 | 2600 + s_27 | s_270 | s_2700 | 2055-07-24 12:00:00 | 27 | 270 | 2700 | 2700 | 2700 | 2700 | 2700 | s_27 | s_270 | s_2700 | 2055-07-24 12:00:00 | 27 | 270 | 2700 | 2700 | 2700 | 2700 | 2700 + s_28 | s_280 | s_2800 | 2058-09-23 09:46:40 | 28 | 280 | 2800 | 2800 | 2800 | 2800 | 2800 | s_28 | s_280 | s_2800 | 2058-09-23 09:46:40 | 28 | 280 | 2800 | 2800 | 2800 | 2800 | 2800 + s_29 | s_290 | s_2900 | 2061-11-23 07:33:20 | 29 | 290 | 2900 | 2900 | 2900 | 2900 | 2900 | s_29 | s_290 | s_2900 | 2061-11-23 07:33:20 | 29 | 290 | 2900 | 2900 | 2900 | 2900 | 2900 + s_30 | s_300 | s_3000 | 2065-01-24 05:20:00 | 30 | 300 | 3000 | 3000 | 3000 | 3000 | 3000 | s_30 | s_300 | s_3000 | 2065-01-24 05:20:00 | 30 | 300 | 3000 | 3000 | 3000 | 3000 | 3000 + s_31 | s_310 | s_3100 | 2068-03-26 03:06:40 | 31 | 310 | 3100 | 3100 | 3100 | 3100 | 3100 | s_31 | s_310 | s_3100 | 2068-03-26 03:06:40 | 31 | 310 | 3100 | 3100 | 3100 | 3100 | 3100 + s_32 | s_320 | s_3200 | 2071-05-28 12:53:20 | 32 | 320 | 3200 | 3200 | 3200 | 3200 | 3200 | s_32 | s_320 | s_3200 | 2071-05-28 12:53:20 | 32 | 320 | 3200 | 3200 | 3200 | 3200 | 3200 + s_33 | s_330 | s_3300 | 2074-07-28 10:40:00 | 33 | 330 | 3300 | 3300 | 3300 | 3300 | 3300 | s_33 | s_330 | s_3300 | 2074-07-28 10:40:00 | 33 | 330 | 3300 | 3300 | 3300 | 3300 | 3300 + s_34 | s_340 | s_3400 | 2077-09-27 08:26:40 | 34 | 340 | 3400 | 3400 | 3400 | 3400 | 3400 | s_34 | s_340 | s_3400 | 2077-09-27 08:26:40 | 34 | 340 | 3400 | 3400 | 3400 | 3400 | 3400 + s_35 | s_350 | s_3500 | 2080-11-28 06:13:20 | 35 | 350 | 3500 | 3500 | 3500 | 3500 | 3500 | s_35 | s_350 | s_3500 | 2080-11-28 06:13:20 | 35 | 350 | 3500 | 3500 | 3500 | 3500 | 3500 + s_36 | s_360 | s_3600 | 2084-01-29 04:00:00 | 36 | 360 | 3600 | 3600 | 3600 | 3600 | 3600 | s_36 | s_360 | s_3600 | 2084-01-29 04:00:00 | 36 | 360 | 3600 | 3600 | 3600 | 3600 | 3600 + s_37 | s_370 | s_3700 | 2087-04-01 01:46:40 | 37 | 370 | 3700 | 3700 | 3700 | 3700 | 3700 | s_37 | s_370 | s_3700 | 2087-04-01 01:46:40 | 37 | 370 | 3700 | 3700 | 3700 | 3700 | 3700 + s_38 | s_380 | s_3800 | 2090-06-01 11:33:20 | 38 | 380 | 3800 | 3800 | 3800 | 3800 | 3800 | s_38 | s_380 | s_3800 | 2090-06-01 11:33:20 | 38 | 380 | 3800 | 3800 | 3800 | 3800 | 3800 + s_39 | s_390 | s_3900 | 2093-08-01 09:20:00 | 39 | 390 | 3900 | 3900 | 3900 | 3900 | 3900 | s_39 | s_390 | s_3900 | 2093-08-01 09:20:00 | 39 | 390 | 3900 | 3900 | 3900 | 3900 | 3900 + s_40 | s_400 | s_4000 | 2096-10-02 07:06:40 | 40 | 400 | 4000 | 4000 | 4000 | 4000 | 4000 | s_40 | s_400 | s_4000 | 2096-10-02 07:06:40 | 40 | 400 | 4000 | 4000 | 4000 | 4000 | 4000 + s_41 | s_410 | s_4100 | 2099-12-03 04:53:20 | 41 | 410 | 4100 | 4100 | 4100 | 4100 | 4100 | s_41 | s_410 | s_4100 | 2099-12-03 04:53:20 | 41 | 410 | 4100 | 4100 | 4100 | 4100 | 4100 + s_42 | s_420 | s_4200 | 2103-02-04 02:40:00 | 42 | 420 | 4200 | 4200 | 4200 | 4200 | 4200 | s_42 | s_420 | s_4200 | 2103-02-04 02:40:00 | 42 | 420 | 4200 | 4200 | 4200 | 4200 | 4200 + s_43 | s_430 | s_4300 | 2106-04-06 12:26:40 | 43 | 430 | 4300 | 4300 | 4300 | 4300 | 4300 | s_43 | s_430 | s_4300 | 2106-04-06 12:26:40 | 43 | 430 | 4300 | 4300 | 4300 | 4300 | 4300 + s_44 | s_440 | s_4400 | 2109-06-06 10:13:20 | 44 | 440 | 4400 | 4400 | 4400 | 4400 | 4400 | s_44 | s_440 | s_4400 | 2109-06-06 10:13:20 | 44 | 440 | 4400 | 4400 | 4400 | 4400 | 4400 + s_45 | s_450 | s_4500 | 2112-08-07 08:00:00 | 45 | 450 | 4500 | 4500 | 4500 | 4500 | 4500 | s_45 | s_450 | s_4500 | 2112-08-07 08:00:00 | 45 | 450 | 4500 | 4500 | 4500 | 4500 | 4500 + s_46 | s_460 | s_4600 | 2115-10-08 05:46:40 | 46 | 460 | 4600 | 4600 | 4600 | 4600 | 4600 | s_46 | s_460 | s_4600 | 2115-10-08 05:46:40 | 46 | 460 | 4600 | 4600 | 4600 | 4600 | 4600 + s_47 | s_470 | s_4700 | 2118-12-09 03:33:20 | 47 | 470 | 4700 | 4700 | 4700 | 4700 | 4700 | s_47 | s_470 | s_4700 | 2118-12-09 03:33:20 | 47 | 470 | 4700 | 4700 | 4700 | 4700 | 4700 + s_48 | s_480 | s_4800 | 2122-02-08 01:20:00 | 48 | 480 | 4800 | 4800 | 4800 | 4800 | 4800 | s_48 | s_480 | s_4800 | 2122-02-08 01:20:00 | 48 | 480 | 4800 | 4800 | 4800 | 4800 | 4800 + s_49 | s_490 | s_4900 | 2125-04-10 11:06:40 | 49 | 490 | 4900 | 4900 | 4900 | 4900 | 4900 | s_49 | s_490 | s_4900 | 2125-04-10 11:06:40 | 49 | 490 | 4900 | 4900 | 4900 | 4900 | 4900 + s_50 | s_500 | s_5000 | 2128-06-11 08:53:20 | 50 | 500 | 5000 | 5000 | 5000 | 5000 | 5000 | s_50 | s_500 | s_5000 | 2128-06-11 08:53:20 | 50 | 500 | 5000 | 5000 | 5000 | 5000 | 5000 + s_51 | s_510 | s_5100 | 2131-08-12 06:40:00 | 51 | 510 | 5100 | 5100 | 5100 | 5100 | 5100 | s_51 | s_510 | s_5100 | 2131-08-12 06:40:00 | 51 | 510 | 5100 | 5100 | 5100 | 5100 | 5100 + s_52 | s_520 | s_5200 | 2134-10-13 04:26:40 | 52 | 520 | 5200 | 5200 | 5200 | 5200 | 5200 | s_52 | s_520 | s_5200 | 2134-10-13 04:26:40 | 52 | 520 | 5200 | 5200 | 5200 | 5200 | 5200 + s_53 | s_530 | s_5300 | 2137-12-13 02:13:20 | 53 | 530 | 5300 | 5300 | 5300 | 5300 | 5300 | s_53 | s_530 | s_5300 | 2137-12-13 02:13:20 | 53 | 530 | 5300 | 5300 | 5300 | 5300 | 5300 + s_54 | s_540 | s_5400 | 2141-02-13 12:00:00 | 54 | 540 | 5400 | 5400 | 5400 | 5400 | 5400 | s_54 | s_540 | s_5400 | 2141-02-13 12:00:00 | 54 | 540 | 5400 | 5400 | 5400 | 5400 | 5400 + s_55 | s_550 | s_5500 | 2144-04-15 09:46:40 | 55 | 550 | 5500 | 5500 | 5500 | 5500 | 5500 | s_55 | s_550 | s_5500 | 2144-04-15 09:46:40 | 55 | 550 | 5500 | 5500 | 5500 | 5500 | 5500 + s_56 | s_560 | s_5600 | 2147-06-16 07:33:20 | 56 | 560 | 5600 | 5600 | 5600 | 5600 | 5600 | s_56 | s_560 | s_5600 | 2147-06-16 07:33:20 | 56 | 560 | 5600 | 5600 | 5600 | 5600 | 5600 + s_57 | s_570 | s_5700 | 2150-08-17 05:20:00 | 57 | 570 | 5700 | 5700 | 5700 | 5700 | 5700 | s_57 | s_570 | s_5700 | 2150-08-17 05:20:00 | 57 | 570 | 5700 | 5700 | 5700 | 5700 | 5700 + s_58 | s_580 | s_5800 | 2153-10-17 03:06:40 | 58 | 580 | 5800 | 5800 | 5800 | 5800 | 5800 | s_58 | s_580 | s_5800 | 2153-10-17 03:06:40 | 58 | 580 | 5800 | 5800 | 5800 | 5800 | 5800 + s_59 | s_590 | s_5900 | 2156-12-18 12:53:20 | 59 | 590 | 5900 | 5900 | 5900 | 5900 | 5900 | s_59 | s_590 | s_5900 | 2156-12-18 12:53:20 | 59 | 590 | 5900 | 5900 | 5900 | 5900 | 5900 + s_60 | s_600 | s_6000 | 2160-02-18 10:40:00 | 60 | 600 | 6000 | 6000 | 6000 | 6000 | 6000 | s_60 | s_600 | s_6000 | 2160-02-18 10:40:00 | 60 | 600 | 6000 | 6000 | 6000 | 6000 | 6000 + s_61 | s_610 | s_6100 | 2163-04-20 08:26:40 | 61 | 610 | 6100 | 6100 | 6100 | 6100 | 6100 | s_61 | s_610 | s_6100 | 2163-04-20 08:26:40 | 61 | 610 | 6100 | 6100 | 6100 | 6100 | 6100 + s_62 | s_620 | s_6200 | 2166-06-21 06:13:20 | 62 | 620 | 6200 | 6200 | 6200 | 6200 | 6200 | s_62 | s_620 | s_6200 | 2166-06-21 06:13:20 | 62 | 620 | 6200 | 6200 | 6200 | 6200 | 6200 + s_63 | s_630 | s_6300 | 2169-08-21 04:00:00 | 63 | 630 | 6300 | 6300 | 6300 | 6300 | 6300 | s_63 | s_630 | s_6300 | 2169-08-21 04:00:00 | 63 | 630 | 6300 | 6300 | 6300 | 6300 | 6300 + s_64 | s_640 | s_6400 | 2172-10-22 01:46:40 | 64 | 640 | 6400 | 6400 | 6400 | 6400 | 6400 | s_64 | s_640 | s_6400 | 2172-10-22 01:46:40 | 64 | 640 | 6400 | 6400 | 6400 | 6400 | 6400 + s_65 | s_650 | s_6500 | 2175-12-23 11:33:20 | 65 | 650 | 6500 | 6500 | 6500 | 6500 | 6500 | s_65 | s_650 | s_6500 | 2175-12-23 11:33:20 | 65 | 650 | 6500 | 6500 | 6500 | 6500 | 6500 + s_66 | s_660 | s_6600 | 2179-02-22 09:20:00 | 66 | 660 | 6600 | 6600 | 6600 | 6600 | 6600 | s_66 | s_660 | s_6600 | 2179-02-22 09:20:00 | 66 | 660 | 6600 | 6600 | 6600 | 6600 | 6600 + s_67 | s_670 | s_6700 | 2182-04-25 07:06:40 | 67 | 670 | 6700 | 6700 | 6700 | 6700 | 6700 | s_67 | s_670 | s_6700 | 2182-04-25 07:06:40 | 67 | 670 | 6700 | 6700 | 6700 | 6700 | 6700 + s_68 | s_680 | s_6800 | 2185-06-25 04:53:20 | 68 | 680 | 6800 | 6800 | 6800 | 6800 | 6800 | s_68 | s_680 | s_6800 | 2185-06-25 04:53:20 | 68 | 680 | 6800 | 6800 | 6800 | 6800 | 6800 + s_69 | s_690 | s_6900 | 2188-08-26 02:40:00 | 69 | 690 | 6900 | 6900 | 6900 | 6900 | 6900 | s_69 | s_690 | s_6900 | 2188-08-26 02:40:00 | 69 | 690 | 6900 | 6900 | 6900 | 6900 | 6900 + s_70 | s_700 | s_7000 | 2191-10-27 12:26:40 | 70 | 700 | 7000 | 7000 | 7000 | 7000 | 7000 | s_70 | s_700 | s_7000 | 2191-10-27 12:26:40 | 70 | 700 | 7000 | 7000 | 7000 | 7000 | 7000 + s_71 | s_710 | s_7100 | 2194-12-27 10:13:20 | 71 | 710 | 7100 | 7100 | 7100 | 7100 | 7100 | s_71 | s_710 | s_7100 | 2194-12-27 10:13:20 | 71 | 710 | 7100 | 7100 | 7100 | 7100 | 7100 + s_72 | s_720 | s_7200 | 2198-02-27 08:00:00 | 72 | 720 | 7200 | 7200 | 7200 | 7200 | 7200 | s_72 | s_720 | s_7200 | 2198-02-27 08:00:00 | 72 | 720 | 7200 | 7200 | 7200 | 7200 | 7200 + s_73 | s_730 | s_7300 | 2201-04-30 05:46:40 | 73 | 730 | 7300 | 7300 | 7300 | 7300 | 7300 | s_73 | s_730 | s_7300 | 2201-04-30 05:46:40 | 73 | 730 | 7300 | 7300 | 7300 | 7300 | 7300 + s_74 | s_740 | s_7400 | 2204-07-01 03:33:20 | 74 | 740 | 7400 | 7400 | 7400 | 7400 | 7400 | s_74 | s_740 | s_7400 | 2204-07-01 03:33:20 | 74 | 740 | 7400 | 7400 | 7400 | 7400 | 7400 + s_75 | s_750 | s_7500 | 2207-09-01 01:20:00 | 75 | 750 | 7500 | 7500 | 7500 | 7500 | 7500 | s_75 | s_750 | s_7500 | 2207-09-01 01:20:00 | 75 | 750 | 7500 | 7500 | 7500 | 7500 | 7500 + s_76 | s_760 | s_7600 | 2210-11-01 11:06:40 | 76 | 760 | 7600 | 7600 | 7600 | 7600 | 7600 | s_76 | s_760 | s_7600 | 2210-11-01 11:06:40 | 76 | 760 | 7600 | 7600 | 7600 | 7600 | 7600 + s_77 | s_770 | s_7700 | 2214-01-02 08:53:20 | 77 | 770 | 7700 | 7700 | 7700 | 7700 | 7700 | s_77 | s_770 | s_7700 | 2214-01-02 08:53:20 | 77 | 770 | 7700 | 7700 | 7700 | 7700 | 7700 + s_78 | s_780 | s_7800 | 2217-03-04 06:40:00 | 78 | 780 | 7800 | 7800 | 7800 | 7800 | 7800 | s_78 | s_780 | s_7800 | 2217-03-04 06:40:00 | 78 | 780 | 7800 | 7800 | 7800 | 7800 | 7800 + s_79 | s_790 | s_7900 | 2220-05-05 04:26:40 | 79 | 790 | 7900 | 7900 | 7900 | 7900 | 7900 | s_79 | s_790 | s_7900 | 2220-05-05 04:26:40 | 79 | 790 | 7900 | 7900 | 7900 | 7900 | 7900 + s_80 | s_800 | s_8000 | 2223-07-06 02:13:20 | 80 | 800 | 8000 | 8000 | 8000 | 8000 | 8000 | s_80 | s_800 | s_8000 | 2223-07-06 02:13:20 | 80 | 800 | 8000 | 8000 | 8000 | 8000 | 8000 + s_81 | s_810 | s_8100 | 2226-09-06 12:00:00 | 81 | 810 | 8100 | 8100 | 8100 | 8100 | 8100 | s_81 | s_810 | s_8100 | 2226-09-06 12:00:00 | 81 | 810 | 8100 | 8100 | 8100 | 8100 | 8100 + s_82 | s_820 | s_8200 | 2229-11-06 09:46:40 | 82 | 820 | 8200 | 8200 | 8200 | 8200 | 8200 | s_82 | s_820 | s_8200 | 2229-11-06 09:46:40 | 82 | 820 | 8200 | 8200 | 8200 | 8200 | 8200 + s_83 | s_830 | s_8300 | 2233-01-06 07:33:20 | 83 | 830 | 8300 | 8300 | 8300 | 8300 | 8300 | s_83 | s_830 | s_8300 | 2233-01-06 07:33:20 | 83 | 830 | 8300 | 8300 | 8300 | 8300 | 8300 + s_84 | s_840 | s_8400 | 2236-03-09 05:20:00 | 84 | 840 | 8400 | 8400 | 8400 | 8400 | 8400 | s_84 | s_840 | s_8400 | 2236-03-09 05:20:00 | 84 | 840 | 8400 | 8400 | 8400 | 8400 | 8400 + s_85 | s_850 | s_8500 | 2239-05-10 03:06:40 | 85 | 850 | 8500 | 8500 | 8500 | 8500 | 8500 | s_85 | s_850 | s_8500 | 2239-05-10 03:06:40 | 85 | 850 | 8500 | 8500 | 8500 | 8500 | 8500 + s_86 | s_860 | s_8600 | 2242-07-11 12:53:20 | 86 | 860 | 8600 | 8600 | 8600 | 8600 | 8600 | s_86 | s_860 | s_8600 | 2242-07-11 12:53:20 | 86 | 860 | 8600 | 8600 | 8600 | 8600 | 8600 + s_87 | s_870 | s_8700 | 2245-09-10 10:40:00 | 87 | 870 | 8700 | 8700 | 8700 | 8700 | 8700 | s_87 | s_870 | s_8700 | 2245-09-10 10:40:00 | 87 | 870 | 8700 | 8700 | 8700 | 8700 | 8700 + s_88 | s_880 | s_8800 | 2248-11-10 08:26:40 | 88 | 880 | 8800 | 8800 | 8800 | 8800 | 8800 | s_88 | s_880 | s_8800 | 2248-11-10 08:26:40 | 88 | 880 | 8800 | 8800 | 8800 | 8800 | 8800 + s_89 | s_890 | s_8900 | 2252-01-12 06:13:20 | 89 | 890 | 8900 | 8900 | 8900 | 8900 | 8900 | s_89 | s_890 | s_8900 | 2252-01-12 06:13:20 | 89 | 890 | 8900 | 8900 | 8900 | 8900 | 8900 + s_90 | s_900 | s_9000 | 2255-03-14 04:00:00 | 90 | 900 | 9000 | 9000 | 9000 | 9000 | 9000 | s_90 | s_900 | s_9000 | 2255-03-14 04:00:00 | 90 | 900 | 9000 | 9000 | 9000 | 9000 | 9000 + s_91 | s_910 | s_9100 | 2258-05-15 01:46:40 | 91 | 910 | 9100 | 9100 | 9100 | 9100 | 9100 | s_91 | s_910 | s_9100 | 2258-05-15 01:46:40 | 91 | 910 | 9100 | 9100 | 9100 | 9100 | 9100 + s_92 | s_920 | s_9200 | 2261-07-15 11:33:20 | 92 | 920 | 9200 | 9200 | 9200 | 9200 | 9200 | s_92 | s_920 | s_9200 | 2261-07-15 11:33:20 | 92 | 920 | 9200 | 9200 | 9200 | 9200 | 9200 + s_93 | s_930 | s_9300 | 2264-09-14 09:20:00 | 93 | 930 | 9300 | 9300 | 9300 | 9300 | 9300 | s_93 | s_930 | s_9300 | 2264-09-14 09:20:00 | 93 | 930 | 9300 | 9300 | 9300 | 9300 | 9300 + s_94 | s_940 | s_9400 | 2267-11-16 07:06:40 | 94 | 940 | 9400 | 9400 | 9400 | 9400 | 9400 | s_94 | s_940 | s_9400 | 2267-11-16 07:06:40 | 94 | 940 | 9400 | 9400 | 9400 | 9400 | 9400 + s_95 | s_950 | s_9500 | 2271-01-16 04:53:20 | 95 | 950 | 9500 | 9500 | 9500 | 9500 | 9500 | s_95 | s_950 | s_9500 | 2271-01-16 04:53:20 | 95 | 950 | 9500 | 9500 | 9500 | 9500 | 9500 + s_96 | s_960 | s_9600 | 2274-03-19 02:40:00 | 96 | 960 | 9600 | 9600 | 9600 | 9600 | 9600 | s_96 | s_960 | s_9600 | 2274-03-19 02:40:00 | 96 | 960 | 9600 | 9600 | 9600 | 9600 | 9600 + s_97 | s_970 | s_9700 | 2277-05-19 12:26:40 | 97 | 970 | 9700 | 9700 | 9700 | 9700 | 9700 | s_97 | s_970 | s_9700 | 2277-05-19 12:26:40 | 97 | 970 | 9700 | 9700 | 9700 | 9700 | 9700 + s_98 | s_980 | s_9800 | 2280-07-19 10:13:20 | 98 | 980 | 9800 | 9800 | 9800 | 9800 | 9800 | s_98 | s_980 | s_9800 | 2280-07-19 10:13:20 | 98 | 980 | 9800 | 9800 | 9800 | 9800 | 9800 + s_99 | s_990 | s_9900 | 2283-09-20 08:00:00 | 99 | 990 | 9900 | 9900 | 9900 | 9900 | 9900 | s_99 | s_990 | s_9900 | 2283-09-20 08:00:00 | 99 | 990 | 9900 | 9900 | 9900 | 9900 | 9900 + s_100 | s_1000 | s_10000 | 2286-11-20 05:46:40 | 100 | 1000 | 10000 | 10000 | 10000 | 10000 | 10000 | s_100 | s_1000 | s_10000 | 2286-11-20 05:46:40 | 100 | 1000 | 10000 | 10000 | 10000 | 10000 | 10000 +(100 rows) diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_quote/runTest.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_quote/runTest.py new file mode 100755 index 0000000000..40fff51f2c --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_quote/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfMultibyteTwoByteWithQuoteDelimiterData(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_quote/sql/query01.sql b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_quote/sql/query01.sql new file mode 100755 index 0000000000..0f90a49220 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_quote/sql/query01.sql @@ -0,0 +1,3 @@ +-- @description query01 for PXF Multibyte delimiter, 2-byte delim with quote case + +SELECT * from pxf_multibyte_twobyte_withquote_data ORDER BY n1; \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_quote_and_escape/__init__.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_quote_and_escape/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_quote_and_escape/expected/query01.ans b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_quote_and_escape/expected/query01.ans new file mode 100755 index 0000000000..2a589ee89f --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_quote_and_escape/expected/query01.ans @@ -0,0 +1,108 @@ +-- start_ignore +-- end_ignore +-- @description query01 for PXF Multibyte delimiter, 2-byte delim with quote and escape case +SELECT * from pxf_multibyte_twobyte_withquote_withescape_data ORDER BY n1; + s1 | s2 | s3 | d1 | n1 | n2 | n3 | n4 | n5 | n6 | n7 | s11 | s12 | s13 | d11 | n11 | n12 | n13 | n14 | n15 | n16 | n17 +-------+--------+---------+---------------------+-----+------+-------+-------+-------+-------+-------+------------------+--------+---------+---------------------+-----+------+-------+-------+-------+-------+------- + s_1 | s_10 | s_100 | 1973-03-03 09:46:40 | 1 | 10 | 100 | 100 | 100 | 100 | 100 | s_1 | s_10 | s_100 | 1973-03-03 09:46:40 | 1 | 10 | 100 | 100 | 100 | 100 | 100 + s_2 | s_20 | s_200 | 1976-05-03 07:33:20 | 2 | 20 | 200 | 200 | 200 | 200 | 200 | s_2 | s_20 | s_200 | 1976-05-03 07:33:20 | 2 | 20 | 200 | 200 | 200 | 200 | 200 + s_3 | s_30 | s_300 | 1979-07-05 05:20:00 | 3 | 30 | 300 | 300 | 300 | 300 | 300 | s_3 | s_30 | s_300 | 1979-07-05 05:20:00 | 3 | 30 | 300 | 300 | 300 | 300 | 300 + s_4 | s_40 | s_400 | 1982-09-04 03:06:40 | 4 | 40 | 400 | 400 | 400 | 400 | 400 | s_4 | s_40 | s_400 | 1982-09-04 03:06:40 | 4 | 40 | 400 | 400 | 400 | 400 | 400 + s_5 | s_50 | s_500 | 1985-11-05 12:53:20 | 5 | 50 | 500 | 500 | 500 | 500 | 500 | s_5 | s_50 | s_500 | 1985-11-05 12:53:20 | 5 | 50 | 500 | 500 | 500 | 500 | 500 + s_6 | s_60 | s_600 | 1989-01-05 10:40:00 | 6 | 60 | 600 | 600 | 600 | 600 | 600 | s_6 | s_60 | s_600 | 1989-01-05 10:40:00 | 6 | 60 | 600 | 600 | 600 | 600 | 600 + s_7 | s_70 | s_700 | 1992-03-07 08:26:40 | 7 | 70 | 700 | 700 | 700 | 700 | 700 | s_7 | s_70 | s_700 | 1992-03-07 08:26:40 | 7 | 70 | 700 | 700 | 700 | 700 | 700 + s_8 | s_80 | s_800 | 1995-05-09 06:13:20 | 8 | 80 | 800 | 800 | 800 | 800 | 800 | s_8 | s_80 | s_800 | 1995-05-09 06:13:20 | 8 | 80 | 800 | 800 | 800 | 800 | 800 + s_9 | s_90 | s_900 | 1998-07-09 04:00:00 | 9 | 90 | 900 | 900 | 900 | 900 | 900 | s_9 | s_90 | s_900 | 1998-07-09 04:00:00 | 9 | 90 | 900 | 900 | 900 | 900 | 900 + s_10 | s_100 | s_1000 | 2001-09-09 01:46:40 | 10 | 100 | 1000 | 1000 | 1000 | 1000 | 1000 | s_10 | s_100 | s_1000 | 2001-09-09 01:46:40 | 10 | 100 | 1000 | 1000 | 1000 | 1000 | 1000 + s_11 | s_110 | s_1100 | 2004-11-09 11:33:20 | 11 | 110 | 1100 | 1100 | 1100 | 1100 | 1100 | s_11 | s_110 | s_1100 | 2004-11-09 11:33:20 | 11 | 110 | 1100 | 1100 | 1100 | 1100 | 1100 + s_12 | s_120 | s_1200 | 2008-01-10 09:20:00 | 12 | 120 | 1200 | 1200 | 1200 | 1200 | 1200 | s_12 | s_120 | s_1200 | 2008-01-10 09:20:00 | 12 | 120 | 1200 | 1200 | 1200 | 1200 | 1200 + s_13 | s_130 | s_1300 | 2011-03-13 07:06:40 | 13 | 130 | 1300 | 1300 | 1300 | 1300 | 1300 | s_13 | s_130 | s_1300 | 2011-03-13 07:06:40 | 13 | 130 | 1300 | 1300 | 1300 | 1300 | 1300 + s_14 | s_140 | s_1400 | 2014-05-13 04:53:20 | 14 | 140 | 1400 | 1400 | 1400 | 1400 | 1400 | s_14 | s_140 | s_1400 | 2014-05-13 04:53:20 | 14 | 140 | 1400 | 1400 | 1400 | 1400 | 1400 + s_15 | s_150 | s_1500 | 2017-07-14 02:40:00 | 15 | 150 | 1500 | 1500 | 1500 | 1500 | 1500 | s_15 | s_150 | s_1500 | 2017-07-14 02:40:00 | 15 | 150 | 1500 | 1500 | 1500 | 1500 | 1500 + s_16 | s_160 | s_1600 | 2020-09-13 12:26:40 | 16 | 160 | 1600 | 1600 | 1600 | 1600 | 1600 | s_16 | s_160 | s_1600 | 2020-09-13 12:26:40 | 16 | 160 | 1600 | 1600 | 1600 | 1600 | 1600 + s_17 | s_170 | s_1700 | 2023-11-14 10:13:20 | 17 | 170 | 1700 | 1700 | 1700 | 1700 | 1700 | s_17 | s_170 | s_1700 | 2023-11-14 10:13:20 | 17 | 170 | 1700 | 1700 | 1700 | 1700 | 1700 + s_18 | s_180 | s_1800 | 2027-01-15 08:00:00 | 18 | 180 | 1800 | 1800 | 1800 | 1800 | 1800 | s_18 | s_180 | s_1800 | 2027-01-15 08:00:00 | 18 | 180 | 1800 | 1800 | 1800 | 1800 | 1800 + s_19 | s_190 | s_1900 | 2030-03-17 05:46:40 | 19 | 190 | 1900 | 1900 | 1900 | 1900 | 1900 | s_19 | s_190 | s_1900 | 2030-03-17 05:46:40 | 19 | 190 | 1900 | 1900 | 1900 | 1900 | 1900 + s_20 | s_200 | s_2000 | 2033-05-18 03:33:20 | 20 | 200 | 2000 | 2000 | 2000 | 2000 | 2000 | s_20 | s_200 | s_2000 | 2033-05-18 03:33:20 | 20 | 200 | 2000 | 2000 | 2000 | 2000 | 2000 + s_21 | s_210 | s_2100 | 2036-07-18 01:20:00 | 21 | 210 | 2100 | 2100 | 2100 | 2100 | 2100 | s_21 | s_210 | s_2100 | 2036-07-18 01:20:00 | 21 | 210 | 2100 | 2100 | 2100 | 2100 | 2100 + s_22 | s_220 | s_2200 | 2039-09-18 11:06:40 | 22 | 220 | 2200 | 2200 | 2200 | 2200 | 2200 | s_22 | s_220 | s_2200 | 2039-09-18 11:06:40 | 22 | 220 | 2200 | 2200 | 2200 | 2200 | 2200 + s_23 | s_230 | s_2300 | 2042-11-19 08:53:20 | 23 | 230 | 2300 | 2300 | 2300 | 2300 | 2300 | s_23 | s_230 | s_2300 | 2042-11-19 08:53:20 | 23 | 230 | 2300 | 2300 | 2300 | 2300 | 2300 + s_24 | s_240 | s_2400 | 2046-01-19 06:40:00 | 24 | 240 | 2400 | 2400 | 2400 | 2400 | 2400 | s_24 | s_240 | s_2400 | 2046-01-19 06:40:00 | 24 | 240 | 2400 | 2400 | 2400 | 2400 | 2400 + s_25 | s_250 | s_2500 | 2049-03-22 04:26:40 | 25 | 250 | 2500 | 2500 | 2500 | 2500 | 2500 | s_25 | s_250 | s_2500 | 2049-03-22 04:26:40 | 25 | 250 | 2500 | 2500 | 2500 | 2500 | 2500 + s_26 | s_260 | s_2600 | 2052-05-22 02:13:20 | 26 | 260 | 2600 | 2600 | 2600 | 2600 | 2600 | s_26 | s_260 | s_2600 | 2052-05-22 02:13:20 | 26 | 260 | 2600 | 2600 | 2600 | 2600 | 2600 + s_27 | s_270 | s_2700 | 2055-07-24 12:00:00 | 27 | 270 | 2700 | 2700 | 2700 | 2700 | 2700 | s_27 | s_270 | s_2700 | 2055-07-24 12:00:00 | 27 | 270 | 2700 | 2700 | 2700 | 2700 | 2700 + s_28 | s_280 | s_2800 | 2058-09-23 09:46:40 | 28 | 280 | 2800 | 2800 | 2800 | 2800 | 2800 | s_28 | s_280 | s_2800 | 2058-09-23 09:46:40 | 28 | 280 | 2800 | 2800 | 2800 | 2800 | 2800 + s_29 | s_290 | s_2900 | 2061-11-23 07:33:20 | 29 | 290 | 2900 | 2900 | 2900 | 2900 | 2900 | s_29 | s_290 | s_2900 | 2061-11-23 07:33:20 | 29 | 290 | 2900 | 2900 | 2900 | 2900 | 2900 + s_30 | s_300 | s_3000 | 2065-01-24 05:20:00 | 30 | 300 | 3000 | 3000 | 3000 | 3000 | 3000 | s_30 | s_300 | s_3000 | 2065-01-24 05:20:00 | 30 | 300 | 3000 | 3000 | 3000 | 3000 | 3000 + s_31 | s_310 | s_3100 | 2068-03-26 03:06:40 | 31 | 310 | 3100 | 3100 | 3100 | 3100 | 3100 | s_31 | s_310 | s_3100 | 2068-03-26 03:06:40 | 31 | 310 | 3100 | 3100 | 3100 | 3100 | 3100 + s_32 | s_320 | s_3200 | 2071-05-28 12:53:20 | 32 | 320 | 3200 | 3200 | 3200 | 3200 | 3200 | s_32 | s_320 | s_3200 | 2071-05-28 12:53:20 | 32 | 320 | 3200 | 3200 | 3200 | 3200 | 3200 + s_33 | s_330 | s_3300 | 2074-07-28 10:40:00 | 33 | 330 | 3300 | 3300 | 3300 | 3300 | 3300 | s_33 | s_330 | s_3300 | 2074-07-28 10:40:00 | 33 | 330 | 3300 | 3300 | 3300 | 3300 | 3300 + s_34 | s_340 | s_3400 | 2077-09-27 08:26:40 | 34 | 340 | 3400 | 3400 | 3400 | 3400 | 3400 | s_34 | s_340 | s_3400 | 2077-09-27 08:26:40 | 34 | 340 | 3400 | 3400 | 3400 | 3400 | 3400 + s_35 | s_350 | s_3500 | 2080-11-28 06:13:20 | 35 | 350 | 3500 | 3500 | 3500 | 3500 | 3500 | s_35 | s_350 | s_3500 | 2080-11-28 06:13:20 | 35 | 350 | 3500 | 3500 | 3500 | 3500 | 3500 + s_36 | s_360 | s_3600 | 2084-01-29 04:00:00 | 36 | 360 | 3600 | 3600 | 3600 | 3600 | 3600 | s_36 | s_360 | s_3600 | 2084-01-29 04:00:00 | 36 | 360 | 3600 | 3600 | 3600 | 3600 | 3600 + s_37 | s_370 | s_3700 | 2087-04-01 01:46:40 | 37 | 370 | 3700 | 3700 | 3700 | 3700 | 3700 | s_37 | s_370 | s_3700 | 2087-04-01 01:46:40 | 37 | 370 | 3700 | 3700 | 3700 | 3700 | 3700 + s_38 | s_380 | s_3800 | 2090-06-01 11:33:20 | 38 | 380 | 3800 | 3800 | 3800 | 3800 | 3800 | s_38 | s_380 | s_3800 | 2090-06-01 11:33:20 | 38 | 380 | 3800 | 3800 | 3800 | 3800 | 3800 + s_39 | s_390 | s_3900 | 2093-08-01 09:20:00 | 39 | 390 | 3900 | 3900 | 3900 | 3900 | 3900 | s_39 | s_390 | s_3900 | 2093-08-01 09:20:00 | 39 | 390 | 3900 | 3900 | 3900 | 3900 | 3900 + s_40 | s_400 | s_4000 | 2096-10-02 07:06:40 | 40 | 400 | 4000 | 4000 | 4000 | 4000 | 4000 | s_40 | s_400 | s_4000 | 2096-10-02 07:06:40 | 40 | 400 | 4000 | 4000 | 4000 | 4000 | 4000 + s_41 | s_410 | s_4100 | 2099-12-03 04:53:20 | 41 | 410 | 4100 | 4100 | 4100 | 4100 | 4100 | s_41 | s_410 | s_4100 | 2099-12-03 04:53:20 | 41 | 410 | 4100 | 4100 | 4100 | 4100 | 4100 + s_42 | s_420 | s_4200 | 2103-02-04 02:40:00 | 42 | 420 | 4200 | 4200 | 4200 | 4200 | 4200 | s_42 | s_420 | s_4200 | 2103-02-04 02:40:00 | 42 | 420 | 4200 | 4200 | 4200 | 4200 | 4200 + s_43 | s_430 | s_4300 | 2106-04-06 12:26:40 | 43 | 430 | 4300 | 4300 | 4300 | 4300 | 4300 | s_43 | s_430 | s_4300 | 2106-04-06 12:26:40 | 43 | 430 | 4300 | 4300 | 4300 | 4300 | 4300 + s_44 | s_440 | s_4400 | 2109-06-06 10:13:20 | 44 | 440 | 4400 | 4400 | 4400 | 4400 | 4400 | s_44 | s_440 | s_4400 | 2109-06-06 10:13:20 | 44 | 440 | 4400 | 4400 | 4400 | 4400 | 4400 + s_45 | s_450 | s_4500 | 2112-08-07 08:00:00 | 45 | 450 | 4500 | 4500 | 4500 | 4500 | 4500 | s_45 | s_450 | s_4500 | 2112-08-07 08:00:00 | 45 | 450 | 4500 | 4500 | 4500 | 4500 | 4500 + s_46 | s_460 | s_4600 | 2115-10-08 05:46:40 | 46 | 460 | 4600 | 4600 | 4600 | 4600 | 4600 | s_46 | s_460 | s_4600 | 2115-10-08 05:46:40 | 46 | 460 | 4600 | 4600 | 4600 | 4600 | 4600 + s_47 | s_470 | s_4700 | 2118-12-09 03:33:20 | 47 | 470 | 4700 | 4700 | 4700 | 4700 | 4700 | s_47 | s_470 | s_4700 | 2118-12-09 03:33:20 | 47 | 470 | 4700 | 4700 | 4700 | 4700 | 4700 + s_48 | s_480 | s_4800 | 2122-02-08 01:20:00 | 48 | 480 | 4800 | 4800 | 4800 | 4800 | 4800 | s_48 | s_480 | s_4800 | 2122-02-08 01:20:00 | 48 | 480 | 4800 | 4800 | 4800 | 4800 | 4800 + s_49 | s_490 | s_4900 | 2125-04-10 11:06:40 | 49 | 490 | 4900 | 4900 | 4900 | 4900 | 4900 | s_49 | s_490 | s_4900 | 2125-04-10 11:06:40 | 49 | 490 | 4900 | 4900 | 4900 | 4900 | 4900 + s_50 | s_500 | s_5000 | 2128-06-11 08:53:20 | 50 | 500 | 5000 | 5000 | 5000 | 5000 | 5000 | s_50 | s_500 | s_5000 | 2128-06-11 08:53:20 | 50 | 500 | 5000 | 5000 | 5000 | 5000 | 5000 + s_51 | s_510 | s_5100 | 2131-08-12 06:40:00 | 51 | 510 | 5100 | 5100 | 5100 | 5100 | 5100 | s_51 | s_510 | s_5100 | 2131-08-12 06:40:00 | 51 | 510 | 5100 | 5100 | 5100 | 5100 | 5100 + s_52 | s_520 | s_5200 | 2134-10-13 04:26:40 | 52 | 520 | 5200 | 5200 | 5200 | 5200 | 5200 | s_52 | s_520 | s_5200 | 2134-10-13 04:26:40 | 52 | 520 | 5200 | 5200 | 5200 | 5200 | 5200 + s_53 | s_530 | s_5300 | 2137-12-13 02:13:20 | 53 | 530 | 5300 | 5300 | 5300 | 5300 | 5300 | s_53 | s_530 | s_5300 | 2137-12-13 02:13:20 | 53 | 530 | 5300 | 5300 | 5300 | 5300 | 5300 + s_54 | s_540 | s_5400 | 2141-02-13 12:00:00 | 54 | 540 | 5400 | 5400 | 5400 | 5400 | 5400 | s_54 | s_540 | s_5400 | 2141-02-13 12:00:00 | 54 | 540 | 5400 | 5400 | 5400 | 5400 | 5400 + s_55 | s_550 | s_5500 | 2144-04-15 09:46:40 | 55 | 550 | 5500 | 5500 | 5500 | 5500 | 5500 | s_55 | s_550 | s_5500 | 2144-04-15 09:46:40 | 55 | 550 | 5500 | 5500 | 5500 | 5500 | 5500 + s_56 | s_560 | s_5600 | 2147-06-16 07:33:20 | 56 | 560 | 5600 | 5600 | 5600 | 5600 | 5600 | s_56 | s_560 | s_5600 | 2147-06-16 07:33:20 | 56 | 560 | 5600 | 5600 | 5600 | 5600 | 5600 + s_57 | s_570 | s_5700 | 2150-08-17 05:20:00 | 57 | 570 | 5700 | 5700 | 5700 | 5700 | 5700 | s_57 | s_570 | s_5700 | 2150-08-17 05:20:00 | 57 | 570 | 5700 | 5700 | 5700 | 5700 | 5700 + s_58 | s_580 | s_5800 | 2153-10-17 03:06:40 | 58 | 580 | 5800 | 5800 | 5800 | 5800 | 5800 | s_58 | s_580 | s_5800 | 2153-10-17 03:06:40 | 58 | 580 | 5800 | 5800 | 5800 | 5800 | 5800 + s_59 | s_590 | s_5900 | 2156-12-18 12:53:20 | 59 | 590 | 5900 | 5900 | 5900 | 5900 | 5900 | s_59 | s_590 | s_5900 | 2156-12-18 12:53:20 | 59 | 590 | 5900 | 5900 | 5900 | 5900 | 5900 + s_60 | s_600 | s_6000 | 2160-02-18 10:40:00 | 60 | 600 | 6000 | 6000 | 6000 | 6000 | 6000 | s_60 | s_600 | s_6000 | 2160-02-18 10:40:00 | 60 | 600 | 6000 | 6000 | 6000 | 6000 | 6000 + s_61 | s_610 | s_6100 | 2163-04-20 08:26:40 | 61 | 610 | 6100 | 6100 | 6100 | 6100 | 6100 | s_61 | s_610 | s_6100 | 2163-04-20 08:26:40 | 61 | 610 | 6100 | 6100 | 6100 | 6100 | 6100 + s_62 | s_620 | s_6200 | 2166-06-21 06:13:20 | 62 | 620 | 6200 | 6200 | 6200 | 6200 | 6200 | s_62 | s_620 | s_6200 | 2166-06-21 06:13:20 | 62 | 620 | 6200 | 6200 | 6200 | 6200 | 6200 + s_63 | s_630 | s_6300 | 2169-08-21 04:00:00 | 63 | 630 | 6300 | 6300 | 6300 | 6300 | 6300 | s_63 | s_630 | s_6300 | 2169-08-21 04:00:00 | 63 | 630 | 6300 | 6300 | 6300 | 6300 | 6300 + s_64 | s_640 | s_6400 | 2172-10-22 01:46:40 | 64 | 640 | 6400 | 6400 | 6400 | 6400 | 6400 | s_64 | s_640 | s_6400 | 2172-10-22 01:46:40 | 64 | 640 | 6400 | 6400 | 6400 | 6400 | 6400 + s_65 | s_650 | s_6500 | 2175-12-23 11:33:20 | 65 | 650 | 6500 | 6500 | 6500 | 6500 | 6500 | s_65 | s_650 | s_6500 | 2175-12-23 11:33:20 | 65 | 650 | 6500 | 6500 | 6500 | 6500 | 6500 + s_66 | s_660 | s_6600 | 2179-02-22 09:20:00 | 66 | 660 | 6600 | 6600 | 6600 | 6600 | 6600 | s_66 | s_660 | s_6600 | 2179-02-22 09:20:00 | 66 | 660 | 6600 | 6600 | 6600 | 6600 | 6600 + s_67 | s_670 | s_6700 | 2182-04-25 07:06:40 | 67 | 670 | 6700 | 6700 | 6700 | 6700 | 6700 | s_67 | s_670 | s_6700 | 2182-04-25 07:06:40 | 67 | 670 | 6700 | 6700 | 6700 | 6700 | 6700 + s_68 | s_680 | s_6800 | 2185-06-25 04:53:20 | 68 | 680 | 6800 | 6800 | 6800 | 6800 | 6800 | s_68 | s_680 | s_6800 | 2185-06-25 04:53:20 | 68 | 680 | 6800 | 6800 | 6800 | 6800 | 6800 + s_69 | s_690 | s_6900 | 2188-08-26 02:40:00 | 69 | 690 | 6900 | 6900 | 6900 | 6900 | 6900 | s_69 | s_690 | s_6900 | 2188-08-26 02:40:00 | 69 | 690 | 6900 | 6900 | 6900 | 6900 | 6900 + s_70 | s_700 | s_7000 | 2191-10-27 12:26:40 | 70 | 700 | 7000 | 7000 | 7000 | 7000 | 7000 | s_70 | s_700 | s_7000 | 2191-10-27 12:26:40 | 70 | 700 | 7000 | 7000 | 7000 | 7000 | 7000 + s_71 | s_710 | s_7100 | 2194-12-27 10:13:20 | 71 | 710 | 7100 | 7100 | 7100 | 7100 | 7100 | s_71 | s_710 | s_7100 | 2194-12-27 10:13:20 | 71 | 710 | 7100 | 7100 | 7100 | 7100 | 7100 + s_72 | s_720 | s_7200 | 2198-02-27 08:00:00 | 72 | 720 | 7200 | 7200 | 7200 | 7200 | 7200 | s_72 | s_720 | s_7200 | 2198-02-27 08:00:00 | 72 | 720 | 7200 | 7200 | 7200 | 7200 | 7200 + s_73 | s_730 | s_7300 | 2201-04-30 05:46:40 | 73 | 730 | 7300 | 7300 | 7300 | 7300 | 7300 | s_73 | s_730 | s_7300 | 2201-04-30 05:46:40 | 73 | 730 | 7300 | 7300 | 7300 | 7300 | 7300 + s_74 | s_740 | s_7400 | 2204-07-01 03:33:20 | 74 | 740 | 7400 | 7400 | 7400 | 7400 | 7400 | s_74 | s_740 | s_7400 | 2204-07-01 03:33:20 | 74 | 740 | 7400 | 7400 | 7400 | 7400 | 7400 + s_75 | s_750 | s_7500 | 2207-09-01 01:20:00 | 75 | 750 | 7500 | 7500 | 7500 | 7500 | 7500 | s_75 | s_750 | s_7500 | 2207-09-01 01:20:00 | 75 | 750 | 7500 | 7500 | 7500 | 7500 | 7500 + s_76 | s_760 | s_7600 | 2210-11-01 11:06:40 | 76 | 760 | 7600 | 7600 | 7600 | 7600 | 7600 | s_76 | s_760 | s_7600 | 2210-11-01 11:06:40 | 76 | 760 | 7600 | 7600 | 7600 | 7600 | 7600 + s_77 | s_770 | s_7700 | 2214-01-02 08:53:20 | 77 | 770 | 7700 | 7700 | 7700 | 7700 | 7700 | s_77 | s_770 | s_7700 | 2214-01-02 08:53:20 | 77 | 770 | 7700 | 7700 | 7700 | 7700 | 7700 + s_78 | s_780 | s_7800 | 2217-03-04 06:40:00 | 78 | 780 | 7800 | 7800 | 7800 | 7800 | 7800 | s_78 | s_780 | s_7800 | 2217-03-04 06:40:00 | 78 | 780 | 7800 | 7800 | 7800 | 7800 | 7800 + s_79 | s_790 | s_7900 | 2220-05-05 04:26:40 | 79 | 790 | 7900 | 7900 | 7900 | 7900 | 7900 | s_79 | s_790 | s_7900 | 2220-05-05 04:26:40 | 79 | 790 | 7900 | 7900 | 7900 | 7900 | 7900 + s_80 | s_800 | s_8000 | 2223-07-06 02:13:20 | 80 | 800 | 8000 | 8000 | 8000 | 8000 | 8000 | s_80 | s_800 | s_8000 | 2223-07-06 02:13:20 | 80 | 800 | 8000 | 8000 | 8000 | 8000 | 8000 + s_81 | s_810 | s_8100 | 2226-09-06 12:00:00 | 81 | 810 | 8100 | 8100 | 8100 | 8100 | 8100 | s_81 | s_810 | s_8100 | 2226-09-06 12:00:00 | 81 | 810 | 8100 | 8100 | 8100 | 8100 | 8100 + s_82 | s_820 | s_8200 | 2229-11-06 09:46:40 | 82 | 820 | 8200 | 8200 | 8200 | 8200 | 8200 | s_82 | s_820 | s_8200 | 2229-11-06 09:46:40 | 82 | 820 | 8200 | 8200 | 8200 | 8200 | 8200 + s_83 | s_830 | s_8300 | 2233-01-06 07:33:20 | 83 | 830 | 8300 | 8300 | 8300 | 8300 | 8300 | s_83 | s_830 | s_8300 | 2233-01-06 07:33:20 | 83 | 830 | 8300 | 8300 | 8300 | 8300 | 8300 + s_84 | s_840 | s_8400 | 2236-03-09 05:20:00 | 84 | 840 | 8400 | 8400 | 8400 | 8400 | 8400 | s_84 | s_840 | s_8400 | 2236-03-09 05:20:00 | 84 | 840 | 8400 | 8400 | 8400 | 8400 | 8400 + s_85 | s_850 | s_8500 | 2239-05-10 03:06:40 | 85 | 850 | 8500 | 8500 | 8500 | 8500 | 8500 | s_85 | s_850 | s_8500 | 2239-05-10 03:06:40 | 85 | 850 | 8500 | 8500 | 8500 | 8500 | 8500 + s_86 | s_860 | s_8600 | 2242-07-11 12:53:20 | 86 | 860 | 8600 | 8600 | 8600 | 8600 | 8600 | s_86 | s_860 | s_8600 | 2242-07-11 12:53:20 | 86 | 860 | 8600 | 8600 | 8600 | 8600 | 8600 + s_87 | s_870 | s_8700 | 2245-09-10 10:40:00 | 87 | 870 | 8700 | 8700 | 8700 | 8700 | 8700 | s_87 | s_870 | s_8700 | 2245-09-10 10:40:00 | 87 | 870 | 8700 | 8700 | 8700 | 8700 | 8700 + s_88 | s_880 | s_8800 | 2248-11-10 08:26:40 | 88 | 880 | 8800 | 8800 | 8800 | 8800 | 8800 | s_88 | s_880 | s_8800 | 2248-11-10 08:26:40 | 88 | 880 | 8800 | 8800 | 8800 | 8800 | 8800 + s_89 | s_890 | s_8900 | 2252-01-12 06:13:20 | 89 | 890 | 8900 | 8900 | 8900 | 8900 | 8900 | s_89 | s_890 | s_8900 | 2252-01-12 06:13:20 | 89 | 890 | 8900 | 8900 | 8900 | 8900 | 8900 + s_90 | s_900 | s_9000 | 2255-03-14 04:00:00 | 90 | 900 | 9000 | 9000 | 9000 | 9000 | 9000 | s_90 | s_900 | s_9000 | 2255-03-14 04:00:00 | 90 | 900 | 9000 | 9000 | 9000 | 9000 | 9000 + s_91 | s_910 | s_9100 | 2258-05-15 01:46:40 | 91 | 910 | 9100 | 9100 | 9100 | 9100 | 9100 | s_91 | s_910 | s_9100 | 2258-05-15 01:46:40 | 91 | 910 | 9100 | 9100 | 9100 | 9100 | 9100 + s_92 | s_920 | s_9200 | 2261-07-15 11:33:20 | 92 | 920 | 9200 | 9200 | 9200 | 9200 | 9200 | s_92 | s_920 | s_9200 | 2261-07-15 11:33:20 | 92 | 920 | 9200 | 9200 | 9200 | 9200 | 9200 + s_93 | s_930 | s_9300 | 2264-09-14 09:20:00 | 93 | 930 | 9300 | 9300 | 9300 | 9300 | 9300 | s_93 | s_930 | s_9300 | 2264-09-14 09:20:00 | 93 | 930 | 9300 | 9300 | 9300 | 9300 | 9300 + s_94 | s_940 | s_9400 | 2267-11-16 07:06:40 | 94 | 940 | 9400 | 9400 | 9400 | 9400 | 9400 | s_94 | s_940 | s_9400 | 2267-11-16 07:06:40 | 94 | 940 | 9400 | 9400 | 9400 | 9400 | 9400 + s_95 | s_950 | s_9500 | 2271-01-16 04:53:20 | 95 | 950 | 9500 | 9500 | 9500 | 9500 | 9500 | s_95 | s_950 | s_9500 | 2271-01-16 04:53:20 | 95 | 950 | 9500 | 9500 | 9500 | 9500 | 9500 + s_96 | s_960 | s_9600 | 2274-03-19 02:40:00 | 96 | 960 | 9600 | 9600 | 9600 | 9600 | 9600 | s_96 | s_960 | s_9600 | 2274-03-19 02:40:00 | 96 | 960 | 9600 | 9600 | 9600 | 9600 | 9600 + s_97 | s_970 | s_9700 | 2277-05-19 12:26:40 | 97 | 970 | 9700 | 9700 | 9700 | 9700 | 9700 | s_97 | s_970 | s_9700 | 2277-05-19 12:26:40 | 97 | 970 | 9700 | 9700 | 9700 | 9700 | 9700 + s_98 | s_980 | s_9800 | 2280-07-19 10:13:20 | 98 | 980 | 9800 | 9800 | 9800 | 9800 | 9800 | s_98 | s_980 | s_9800 | 2280-07-19 10:13:20 | 98 | 980 | 9800 | 9800 | 9800 | 9800 | 9800 + s_99 | s_990 | s_9900 | 2283-09-20 08:00:00 | 99 | 990 | 9900 | 9900 | 9900 | 9900 | 9900 | s_99 | s_990 | s_9900 | 2283-09-20 08:00:00 | 99 | 990 | 9900 | 9900 | 9900 | 9900 | 9900 + s_100 | s_1000 | s_10000 | 2286-11-20 05:46:40 | 100 | 1000 | 10000 | 10000 | 10000 | 10000 | 10000 | s_100 | s_1000 | s_10000 | 2286-11-20 05:46:40 | 100 | 1000 | 10000 | 10000 | 10000 | 10000 | 10000 + s_101 | s_1001 | s_10001 | 2299-11-28 05:46:40 | 101 | 1001 | 10001 | 10001 | 10001 | 10001 | 10001 | s_101 | escaped! | s_1001 | s_10001 | 2299-11-28 05:46:40 | 101 | 1001 | 10001 | 10001 | 10001 | 10001 | 10001 +(101 rows) diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_quote_and_escape/runTest.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_quote_and_escape/runTest.py new file mode 100755 index 0000000000..64b52831d7 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_quote_and_escape/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfMultibyteTwoByteWithQuoteAndEscapeDelimiterData(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_quote_and_escape/sql/query01.sql b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_quote_and_escape/sql/query01.sql new file mode 100755 index 0000000000..a406d3d083 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_quote_and_escape/sql/query01.sql @@ -0,0 +1,3 @@ +-- @description query01 for PXF Multibyte delimiter, 2-byte delim with quote and escape case + +SELECT * from pxf_multibyte_twobyte_withquote_withescape_data ORDER BY n1; \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_delim/__init__.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_delim/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_delim/expected/query01.ans b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_delim/expected/query01.ans new file mode 100755 index 0000000000..6125f595e3 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_delim/expected/query01.ans @@ -0,0 +1,7 @@ +-- start_ignore +-- end_ignore +-- @description query01 for PXF Multibyte delimiter, 2-byte delim with wrong delim case +SELECT * from pxf_multibyte_twobyte_wrong_delim_data ORDER BY n1; +ERROR: Expected 22 columns in row but found 1 +HINT: Is the `delimiter` value in the format options set correctly? +CONTEXT: External table pxf_multibyte_twobyte_wrong_delim_data diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_delim/runTest.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_delim/runTest.py new file mode 100755 index 0000000000..529dfbfd00 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_delim/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfMultibyteTwoByteWithWrongDelimiterData(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_delim/sql/query01.sql b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_delim/sql/query01.sql new file mode 100755 index 0000000000..aec229cbde --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_delim/sql/query01.sql @@ -0,0 +1,3 @@ +-- @description query01 for PXF Multibyte delimiter, 2-byte delim with wrong delim case + +SELECT * from pxf_multibyte_twobyte_wrong_delim_data ORDER BY n1; \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_eol/__init__.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_eol/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_eol/expected/query01.ans b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_eol/expected/query01.ans new file mode 100755 index 0000000000..641fef219b --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_eol/expected/query01.ans @@ -0,0 +1,6 @@ +-- @description query01 for PXF Multibyte delimiter, 2-byte delim with wrong eol cases + +SELECT * from pxf_multibyte_twobyte_wrong_eol_data ORDER BY n1; +ERROR: Did not find expected `newline` character when `quote` value was provided +HINT: Check the format options in the table definition. Additionally, make sure there are no extraneous characters between the `quote` and `newline` values in the data. +CONTEXT: External table pxf_multibyte_twobyte_wrong_eol_data \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_eol/runTest.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_eol/runTest.py new file mode 100755 index 0000000000..8c380576b2 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_eol/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfMultibyteTwoByteWithWrongEol(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_eol/sql/query01.sql b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_eol/sql/query01.sql new file mode 100755 index 0000000000..34c0506510 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_eol/sql/query01.sql @@ -0,0 +1,3 @@ +-- @description query01 for PXF Multibyte delimiter, 2-byte delim with wrong eol case + +SELECT * from pxf_multibyte_twobyte_wrong_eol_data ORDER BY n1; \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_eol_5X/__init__.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_eol_5X/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_eol_5X/expected/query01.ans b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_eol_5X/expected/query01.ans new file mode 100755 index 0000000000..94ffe7f2ec --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_eol_5X/expected/query01.ans @@ -0,0 +1,18 @@ +-- start_ignore +-- end_ignore +-- @description query01 for PXF Multibyte delimiter, 2-byte delim with wrong eol case + +-- start_matchsubs +-- +-- # create a match/subs + +-- m/WARNING/ +-- s/WARNING/GP_IGNORE: WARNING/ +-- +-- end_matchsubs +SELECT * from pxf_multibyte_twobyte_wrong_eol_data ORDER BY n1; +WARNING: unexpected end of file +CONTEXT: External table pxf_multibyte_twobyte_wrong_eol_data + s1 | s2 | s3 | d1 | n1 | n2 | n3 | n4 | n5 | n6 | n7 | s11 | s12 | s13 | d11 | n11 | n12 | n13 | n14 | n15 | n16 | n17 +----+----+----+----+----+----+----+----+----+----+----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+----- +(0 rows) \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_eol_5X/runTest.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_eol_5X/runTest.py new file mode 100755 index 0000000000..40f63aaa90 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_eol_5X/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfMultibyteTwoByteWithWrongEol_5X(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_eol_5X/sql/query01.sql b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_eol_5X/sql/query01.sql new file mode 100755 index 0000000000..9206a6bcfc --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_eol_5X/sql/query01.sql @@ -0,0 +1,11 @@ +-- @description query01 for PXF Multibyte delimiter, 2-byte delim with wrong eol case + +-- start_matchsubs +-- +-- # create a match/subs + +-- m/WARNING/ +-- s/WARNING/GP_IGNORE: WARNING/ +-- +-- end_matchsubs +SELECT * from pxf_multibyte_twobyte_wrong_eol_data ORDER BY n1; \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_escape/__init__.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_escape/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_escape/expected/query01.ans b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_escape/expected/query01.ans new file mode 100755 index 0000000000..16f972d74b --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_escape/expected/query01.ans @@ -0,0 +1,6 @@ +-- @description query01 for PXF Multibyte delimiter, 2-byte delim with wrong escape cases + +SELECT * from pxf_multibyte_twobyte_wrong_escape_data ORDER BY n1; +ERROR: Found an unescaped quote character +HINT: Is the `escape` value in the format options set correctly? +CONTEXT: External table pxf_multibyte_twobyte_wrong_escape_data \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_escape/runTest.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_escape/runTest.py new file mode 100755 index 0000000000..f119283690 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_escape/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfMultibyteTwoByteWithWrongEscapeDelimiterData(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_escape/sql/query01.sql b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_escape/sql/query01.sql new file mode 100755 index 0000000000..cc3a0fa1f8 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_escape/sql/query01.sql @@ -0,0 +1,3 @@ +-- @description query01 for PXF Multibyte delimiter, 2-byte delim with wrong escape case + +SELECT * from pxf_multibyte_twobyte_wrong_escape_data ORDER BY n1; \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_quote/__init__.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_quote/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_quote/expected/query01.ans b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_quote/expected/query01.ans new file mode 100755 index 0000000000..81301f5bba --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_quote/expected/query01.ans @@ -0,0 +1,6 @@ +-- @description query01 for PXF Multibyte delimiter, 2-byte delim with wrong quote cases + +SELECT * from pxf_multibyte_twobyte_wrong_quote_data ORDER BY n1; +ERROR: Did not find expected `newline` character when `quote` value was provided +HINT: Check the format options in the table definition. Additionally, make sure there are no extraneous characters between the `quote` and `newline` values in the data. +CONTEXT: External table pxf_multibyte_twobyte_wrong_quote_data \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_quote/runTest.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_quote/runTest.py new file mode 100755 index 0000000000..9144471151 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_quote/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfMultibyteTwoByteWithWrongQuote(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_quote/sql/query01.sql b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_quote/sql/query01.sql new file mode 100755 index 0000000000..b7e85fda95 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_quote/sql/query01.sql @@ -0,0 +1,3 @@ +-- @description query01 for PXF Multibyte delimiter, 2-byte delim with wrong quote case + +SELECT * from pxf_multibyte_twobyte_wrong_quote_data ORDER BY n1; \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_quote_5X/__init__.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_quote_5X/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_quote_5X/expected/query01.ans b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_quote_5X/expected/query01.ans new file mode 100755 index 0000000000..542264cf99 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_quote_5X/expected/query01.ans @@ -0,0 +1,18 @@ +-- start_ignore +-- end_ignore +-- @description query01 for PXF Multibyte delimiter, 2-byte delim with wrong quote case + +-- start_matchsubs +-- +-- # create a match/subs + +-- m/WARNING/ +-- s/WARNING/GP_IGNORE: WARNING/ +-- +-- end_matchsubs +SELECT * from pxf_multibyte_twobyte_wrong_quote_data ORDER BY n1; +WARNING: unexpected end of file +CONTEXT: External table pxf_multibyte_twobyte_wrong_quote_data + s1 | s2 | s3 | d1 | n1 | n2 | n3 | n4 | n5 | n6 | n7 | s11 | s12 | s13 | d11 | n11 | n12 | n13 | n14 | n15 | n16 | n17 +----+----+----+----+----+----+----+----+----+----+----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+----- +(0 rows) \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_quote_5X/runTest.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_quote_5X/runTest.py new file mode 100755 index 0000000000..77c4ddd878 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_quote_5X/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfMultibyteTwoByteWithWrongQuote_5X(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_quote_5X/sql/query01.sql b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_quote_5X/sql/query01.sql new file mode 100755 index 0000000000..32d7c2a1aa --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_with_wrong_quote_5X/sql/query01.sql @@ -0,0 +1,11 @@ +-- @description query01 for PXF Multibyte delimiter, 2-byte delim with wrong quote case + +-- start_matchsubs +-- +-- # create a match/subs + +-- m/WARNING/ +-- s/WARNING/GP_IGNORE: WARNING/ +-- +-- end_matchsubs +SELECT * from pxf_multibyte_twobyte_wrong_quote_data ORDER BY n1; \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_wrong_formatter/__init__.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_wrong_formatter/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_wrong_formatter/expected/query01.ans b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_wrong_formatter/expected/query01.ans new file mode 100755 index 0000000000..39514fa0d7 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_wrong_formatter/expected/query01.ans @@ -0,0 +1,8 @@ +-- start_ignore +-- m/DETAIL/ +-- s/DETAIL/CONTEXT/ +-- end_ignore +-- @description query01 for PXF Multibyte delimiter, 2-byte delim cases with wrong formatter provided +SELECT * from pxf_multibyte_twobyte_wrongformatter_data ORDER BY n1; +ERROR: Record has 1 fields but the schema size is 22 (seg1 slice1 127.0.0.1:6003 pid=51272) +CONTEXT: External table pxf_multibyte_twobyte_wrongformatter_data diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_wrong_formatter/runTest.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_wrong_formatter/runTest.py new file mode 100755 index 0000000000..f4808fcd6a --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_wrong_formatter/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfMultibyteTwoByteWrongFormatter(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_wrong_formatter/sql/query01.sql b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_wrong_formatter/sql/query01.sql new file mode 100755 index 0000000000..427c002b80 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/two_byte_wrong_formatter/sql/query01.sql @@ -0,0 +1,3 @@ +-- @description query01 for PXF Multibyte delimiter, 2-byte delim cases with wrong formatter provided + +SELECT * from pxf_multibyte_twobyte_wrongformatter_data ORDER BY n1; \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/wrong_profile/__init__.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/wrong_profile/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/wrong_profile/expected/query01.ans b/automation/tincrepo/main/pxf/features/multibyte_delimiter/wrong_profile/expected/query01.ans new file mode 100755 index 0000000000..fce6d68acc --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/wrong_profile/expected/query01.ans @@ -0,0 +1,8 @@ +-- start_ignore +-- end_ignore +-- @description query01 for PXF Multibyte delimiter, wrong profile case + +SELECT * from pxf_multibyte_wrong_profile ORDER BY age; +ERROR: The "pxfdelimited_import" formatter only works with *:text or *:csv profiles. +HINT: Please double check the profile option in the external table definition. +CONTEXT: External table pxf_multibyte_wrong_profile \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/wrong_profile/runTest.py b/automation/tincrepo/main/pxf/features/multibyte_delimiter/wrong_profile/runTest.py new file mode 100755 index 0000000000..bcb30405f1 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/wrong_profile/runTest.py @@ -0,0 +1,12 @@ +from mpp.models import SQLTestCase +from mpp.models import SQLConcurrencyTestCase + +class PxfMultibyteWrongProfile(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/multibyte_delimiter/wrong_profile/sql/query01.sql b/automation/tincrepo/main/pxf/features/multibyte_delimiter/wrong_profile/sql/query01.sql new file mode 100755 index 0000000000..db42ad5fac --- /dev/null +++ b/automation/tincrepo/main/pxf/features/multibyte_delimiter/wrong_profile/sql/query01.sql @@ -0,0 +1,3 @@ +-- @description query01 for PXF Multibyte delimiter, wrong profile case + +SELECT * from pxf_multibyte_wrong_profile ORDER BY age; \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_integer_digit/__init__.py b/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_integer_digit/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_integer_digit/expected/query01.ans b/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_integer_digit/expected/query01.ans new file mode 100755 index 0000000000..268ed0e0d5 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_integer_digit/expected/query01.ans @@ -0,0 +1,6 @@ +-- start_ignore +-- end_ignore +-- @description query01 for writing ORC decimals with large integer digit count +INSERT INTO orc_decimals_with_large_integer_digit_count_writable VALUES (0,false,'\x0001'::bytea,123456789000000000,10,100,'row-00',0.0,3.141592653589793,'0','var00','var-no-length-00','2010-01-01','10:11:00','2013-07-13 21:00:05.000456','1234567890123456789012345678901234567890.12345','476f35e4-da1a-43cf-8f7c-950a00000000'); +ERROR: PXF server error : The value 1234567890123456789012345678901234567890.12345 for the NUMERIC column c_numeric exceeds the maximum supported precision 38. + diff --git a/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_integer_digit/runTest.py b/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_integer_digit/runTest.py new file mode 100755 index 0000000000..aa9750d9f0 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_integer_digit/runTest.py @@ -0,0 +1,11 @@ +from mpp.models import SQLConcurrencyTestCase + +class OrcWriteDecimalWithLargeIntegerDigitCount(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_integer_digit/sql/query01.sql b/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_integer_digit/sql/query01.sql new file mode 100755 index 0000000000..9f345f8944 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_integer_digit/sql/query01.sql @@ -0,0 +1,2 @@ +-- @description query01 for writing ORC decimals with large integer digit count +INSERT INTO orc_decimals_with_large_integer_digit_count_writable VALUES (0,false,'\x0001'::bytea,123456789000000000,10,100,'row-00',0.0,3.141592653589793,'0','var00','var-no-length-00','2010-01-01','10:11:00','2013-07-13 21:00:05.000456','1234567890123456789012345678901234567890.12345','476f35e4-da1a-43cf-8f7c-950a00000000'); diff --git a/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_precision_defined/__init__.py b/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_precision_defined/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_precision_defined/expected/query01.ans b/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_precision_defined/expected/query01.ans new file mode 100755 index 0000000000..5b81997088 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_precision_defined/expected/query01.ans @@ -0,0 +1,5 @@ +-- start_ignore +-- end_ignore +-- @description query01 for writing ORC decimals with large precision defined in the table DDL +INSERT INTO orc_decimals_with_large_precision_writable VALUES (1, 1234567890123456789012345678901234567890.12345); +ERROR: PXF server error : Column c_numeric is defined as NUMERIC with precision 90 which exceeds the maximum supported precision 38. diff --git a/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_precision_defined/runTest.py b/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_precision_defined/runTest.py new file mode 100755 index 0000000000..b7e5651a2a --- /dev/null +++ b/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_precision_defined/runTest.py @@ -0,0 +1,11 @@ +from mpp.models import SQLConcurrencyTestCase + +class OrcWriteDecimalWithLargePrecisionDefined(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_precision_defined/sql/query01.sql b/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_precision_defined/sql/query01.sql new file mode 100755 index 0000000000..e6ad0fb710 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_precision_defined/sql/query01.sql @@ -0,0 +1,2 @@ +-- @description query01 for writing ORC decimals with large precision defined in the table DDL +INSERT INTO orc_decimals_with_large_precision_writable VALUES (1, 1234567890123456789012345678901234567890.12345); diff --git a/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_precision_not_defined/__init__.py b/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_precision_not_defined/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_precision_not_defined/expected/query01.ans b/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_precision_not_defined/expected/query01.ans new file mode 100755 index 0000000000..5392e3118f --- /dev/null +++ b/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_precision_not_defined/expected/query01.ans @@ -0,0 +1,6 @@ +-- start_ignore +-- end_ignore +-- @description query01 for writing ORC decimals with large precision not defined in the table DDL +INSERT INTO orc_decimals_with_large_precision_not_defined_writable VALUES (0,false,'\x0001'::bytea,123456789000000000,10,100,'row-00',0.0,3.141592653589793,'0','var00','var-no-length-00','2010-01-01','10:11:00','2013-07-13 21:00:05.000456','123456789012345678901234567890.12345','476f35e4-da1a-43cf-8f7c-950a00000000'); +ERROR: PXF server error : The value 123456789012345678901234567890.12345 for the NUMERIC column c_numeric exceeds the maximum supported precision and scale (38,10). + diff --git a/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_precision_not_defined/runTest.py b/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_precision_not_defined/runTest.py new file mode 100755 index 0000000000..8c5471c4a2 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_precision_not_defined/runTest.py @@ -0,0 +1,11 @@ +from mpp.models import SQLConcurrencyTestCase + +class OrcWriteDecimalWithLargePrecisionNotDefined(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_precision_not_defined/sql/query01.sql b/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_precision_not_defined/sql/query01.sql new file mode 100755 index 0000000000..0517b4c3cc --- /dev/null +++ b/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_precision_not_defined/sql/query01.sql @@ -0,0 +1,2 @@ +-- @description query01 for writing ORC decimals with large precision not defined in the table DDL +INSERT INTO orc_decimals_with_large_precision_not_defined_writable VALUES (0,false,'\x0001'::bytea,123456789000000000,10,100,'row-00',0.0,3.141592653589793,'0','var00','var-no-length-00','2010-01-01','10:11:00','2013-07-13 21:00:05.000456','123456789012345678901234567890.12345','476f35e4-da1a-43cf-8f7c-950a00000000'); diff --git a/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_scale/__init__.py b/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_scale/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_scale/expected/query01.ans b/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_scale/expected/query01.ans new file mode 100755 index 0000000000..7e971c45c8 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_scale/expected/query01.ans @@ -0,0 +1,11 @@ +-- start_ignore +-- end_ignore +-- @description query01 for writing ORC decimals with scale overflow +INSERT INTO orc_decimals_with_large_scale_writable VALUES (0,false,'\x0001'::bytea,123456789000000000,10,100,'row-00',0.0,3.141592653589793,'0','var00','var-no-length-00','2010-01-01','10:11:00','2013-07-13 21:00:05.000456','12345678901234567890.1234567899123456789012345','476f35e4-da1a-43cf-8f7c-950a00000000'); +INSERT 0 1 +SELECT * FROM orc_decimals_with_large_scale_readable; + id | c_bool | c_bytea | c_bigint | c_small | c_int | c_text | c_real | c_float | c_char | c_varchar | c_varchar_nolimit | c_date | c_time | c_timestamp | c_numeric | c_uuid +----+--------+----------+--------------------+---------+-------+--------+--------+------------------+--------+-----------+-------------------+------------+----------+----------------------------+---------------------------------+-------------------------------------- + 0 | f | \000\001 | 123456789000000000 | 10 | 100 | row-00 | 0 | 3.14159265358979 | 0 | var00 | var-no-length-00 | 2010-01-01 | 10:11:00 | 2013-07-13 21:00:05.000456 | 12345678901234567890.1234567899 | 476f35e4-da1a-43cf-8f7c-950a00000000 +(1 row) + diff --git a/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_scale/runTest.py b/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_scale/runTest.py new file mode 100755 index 0000000000..c99c40bac3 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_scale/runTest.py @@ -0,0 +1,11 @@ +from mpp.models import SQLConcurrencyTestCase + +class OrcWriteDecimalWithLargeScale(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_scale/sql/query01.sql b/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_scale/sql/query01.sql new file mode 100755 index 0000000000..0ede1c665b --- /dev/null +++ b/automation/tincrepo/main/pxf/features/orc/write/decimal_with_large_scale/sql/query01.sql @@ -0,0 +1,3 @@ +-- @description query01 for writing ORC decimals with scale overflow +INSERT INTO orc_decimals_with_large_scale_writable VALUES (0,false,'\x0001'::bytea,123456789000000000,10,100,'row-00',0.0,3.141592653589793,'0','var00','var-no-length-00','2010-01-01','10:11:00','2013-07-13 21:00:05.000456','12345678901234567890.1234567899123456789012345','476f35e4-da1a-43cf-8f7c-950a00000000'); +SELECT * FROM orc_decimals_with_large_scale_readable; \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/parquet/decimal/numeric_undefined_precision_large_data_length/expected/query01.ans b/automation/tincrepo/main/pxf/features/parquet/decimal/numeric_undefined_precision_large_data_length/expected/query01.ans index 12dfeee1ec..08dc7fd2c7 100644 --- a/automation/tincrepo/main/pxf/features/parquet/decimal/numeric_undefined_precision_large_data_length/expected/query01.ans +++ b/automation/tincrepo/main/pxf/features/parquet/decimal/numeric_undefined_precision_large_data_length/expected/query01.ans @@ -2,4 +2,4 @@ -- end_ignore -- @description query01 for writing undefined precision numeric with pxf.parquet.write.decimal.overflow = round. When try to write a numeric with data size > 38, an error will be thrown. INSERT INTO pxf_parquet_write_undefined_precision_numeric_large_data_length SELECT * FROM numeric_undefined_precision; -ERROR: PXF server error : The value 1234567890123456789012345678901234567890.12345 for the NUMERIC column value exceeds maximum precision 38. +ERROR: PXF server error : The value 1234567890123456789012345678901234567890.12345 for the NUMERIC column value exceeds the maximum supported precision 38. diff --git a/automation/tincrepo/main/pxf/features/parquet/decimal/numeric_undefined_precision_large_integer_digit/expected/query01.ans b/automation/tincrepo/main/pxf/features/parquet/decimal/numeric_undefined_precision_large_integer_digit/expected/query01.ans index d7049876c6..eeee8bbddf 100644 --- a/automation/tincrepo/main/pxf/features/parquet/decimal/numeric_undefined_precision_large_integer_digit/expected/query01.ans +++ b/automation/tincrepo/main/pxf/features/parquet/decimal/numeric_undefined_precision_large_integer_digit/expected/query01.ans @@ -2,4 +2,4 @@ -- end_ignore -- @description query01 for writing undefined precision numeric with pxf.parquet.write.decimal.overflow = round. When try to write a numeric with integerDigitCount > 20, an error will be thrown. INSERT INTO parquet_write_undefined_precision_numeric_large_integer_digit SELECT * FROM numeric_undefined_precision; -ERROR: PXF server error : The value 1234567890123456789012345.1234567890 for the NUMERIC column value exceeds maximum precision and scale (38,18). +ERROR: PXF server error : The value 1234567890123456789012345.1234567890 for the NUMERIC column value exceeds the maximum supported precision and scale (38,18). diff --git a/automation/tincrepo/main/pxf/features/parquet/decimal/numeric_with_large_precision/expected/query01.ans b/automation/tincrepo/main/pxf/features/parquet/decimal/numeric_with_large_precision/expected/query01.ans index e008718ee9..afa989896d 100644 --- a/automation/tincrepo/main/pxf/features/parquet/decimal/numeric_with_large_precision/expected/query01.ans +++ b/automation/tincrepo/main/pxf/features/parquet/decimal/numeric_with_large_precision/expected/query01.ans @@ -2,4 +2,4 @@ -- end_ignore -- @description query01 for writing defined precision numeric with pxf.parquet.write.decimal.overflow = round. When try to write a numeric with precision > 38, an error will be thrown. INSERT INTO parquet_write_defined_large_precision_numeric SELECT * FROM numeric_precision; -ERROR: PXF server error : Column a is defined as NUMERIC with precision 90 which exceeds maximum supported precision 38. \ No newline at end of file +ERROR: PXF server error : Column a is defined as NUMERIC with precision 90 which exceeds the maximum supported precision 38. \ No newline at end of file diff --git a/concourse/Makefile b/concourse/Makefile index 5b4ab8c71c..5fda6dd540 100644 --- a/concourse/Makefile +++ b/concourse/Makefile @@ -18,6 +18,7 @@ NUM_GPDB5_VERSIONS ?= 10 NUM_GPDB6_VERSIONS ?= 9 REDHAT_MAJOR_VERSION ?= 7 FLY_CMD ?= fly +CHECK_CREDS ?= true TEMPLATE_CMD = ./template_tool FLY_OPTION_NON-INTERACTIVE = SLACK ?= true @@ -37,6 +38,11 @@ MINIO ?= false OEL7 ?= false FILE ?= false +SET_PIPELINE := set-pipeline +ifeq ($(CHECK_CREDS), true) +SET_PIPELINE += --check-creds +endif + .PHONY: build certification dev pr cloudbuild longevity build: set-build-pipeline certification: set-certification-pipeline @@ -48,6 +54,7 @@ longevity: set-longevity-pipeline # ============================= BUILD PIPELINE TARGETS ============================= .PHONY: set-build-pipeline +set-build-pipeline: SLACK=true set-build-pipeline: @PIPELINE_FILE=$$(mktemp) && \ $(TEMPLATE_CMD) --template build_pipeline-tpl.yml --vars \ @@ -56,8 +63,7 @@ set-build-pipeline: num_gpdb5_versions=$(NUM_GPDB5_VERSIONS) \ num_gpdb6_versions=$(NUM_GPDB6_VERSIONS) >"$${PIPELINE_FILE}" && \ $(FLY_CMD) --target=$(CONCOURSE) \ - set-pipeline \ - --check-creds \ + $(SET_PIPELINE) \ --pipeline=$(BUILD_PIPELINE_NAME) \ --config "$${PIPELINE_FILE}" \ --load-vars-from=$(HOME)/workspace/pxf/concourse/settings/pxf-multinode-params.yml \ @@ -78,8 +84,7 @@ set-dev-release-pipeline: num_gpdb5_versions=$(NUM_GPDB5_VERSIONS) \ num_gpdb6_versions=$(NUM_GPDB6_VERSIONS) >"$${PIPELINE_FILE}" && \ $(FLY_CMD) --target=$(CONCOURSE) \ - set-pipeline \ - --check-creds \ + $(SET_PIPELINE) \ \ --pipeline=$(DEV_BUILD_PIPELINE_NAME) \ --config "$${PIPELINE_FILE}" \ --load-vars-from=$(HOME)/workspace/pxf/concourse/settings/pxf-multinode-params.yml \ @@ -117,8 +122,7 @@ set-dev-build-pipeline: num_gpdb5_versions=1 \ num_gpdb6_versions=1 >"$${PIPELINE_FILE}" && \ $(FLY_CMD) --target=$(CONCOURSE) \ - set-pipeline \ - --check-creds \ + $(SET_PIPELINE) \ --pipeline=$(DEV_BUILD_PIPELINE_NAME) \ --config "$${PIPELINE_FILE}" \ --load-vars-from=$(HOME)/workspace/pxf/concourse/settings/pxf-multinode-params.yml \ @@ -131,8 +135,7 @@ set-dev-build-pipeline: .PHONY: set-hadoop-cluster-cleaner set-hadoop-cluster-cleaner: $(FLY_CMD) --target=$(CONCOURSE) \ - set-pipeline \ - --check-creds \ + $(SET_PIPELINE) \ --pipeline=hadoop-cluster-cleaner \ --config=pipelines/hadoop-cluster-cleaner.yml \ --var=pxf-git-branch=main \ @@ -150,8 +153,7 @@ set-pr-build-pipeline: num_gpdb5_versions=1 \ num_gpdb6_versions=1 >"$${PIPELINE_FILE}" && \ $(FLY_CMD) --target=$(CONCOURSE) \ - set-pipeline \ - --check-creds \ + $(SET_PIPELINE) \ --pipeline=$(PR_BUILD_PIPELINE_NAME) \ --config "$${PIPELINE_FILE}" \ ${FLY_OPTION_NON-INTERACTIVE} || echo "Generated yaml has errors: check $${PIPELINE_FILE}" @@ -165,8 +167,7 @@ set-pr-build-pipeline: .PHONY: set-certification-pipeline set-certification-pipeline: @$(FLY_CMD) --target=$(CONCOURSE) \ - set-pipeline \ - --check-creds \ + $(SET_PIPELINE) \ --pipeline=$(CERTIFICATION_PIPELINE_NAME) \ --config pipelines/certification_pipeline.yml \ --var=pxf-git-branch=${BRANCH} \ @@ -180,8 +181,7 @@ set-certification-pipeline: .PHONY: set-cloudbuild-pipeline set-cloudbuild-pipeline: $(FLY_CMD) --target=$(CONCOURSE) \ - set-pipeline \ - --check-creds \ + $(SET_PIPELINE) \ --config $(HOME)/workspace/pxf/concourse/pipelines/cloudbuild_pipeline.yml \ --var pxf-git-branch=$(BRANCH) \ --pipeline cloudbuild @@ -196,8 +196,7 @@ set-pivnet-pipeline: num_gpdb5_versions=$(NUM_GPDB5_VERSIONS) \ num_gpdb6_versions=$(NUM_GPDB6_VERSIONS) >"$${PIPELINE_FILE}" && \ $(FLY_CMD) --target=$(CONCOURSE) \ - set-pipeline \ - --check-creds \ + $(SET_PIPELINE) \ --pipeline=${PIVNET_PIPELINE_NAME} \ --config "$${PIPELINE_FILE}" \ --var pxf-git-branch=$(BRANCH) \ @@ -224,8 +223,7 @@ perf: $(TEMPLATE_CMD) --template perf_pipeline-tpl.yml --vars \ redhat_major_version=$(REDHAT_MAJOR_VERSION) >"$${PIPELINE_FILE}" && \ $(FLY_CMD) --target=$(CONCOURSE) \ - set-pipeline \ - --check-creds \ + $(SET_PIPELINE) \ --config "$${PIPELINE_FILE}" \ --load-vars-from=$(HOME)/workspace/pxf/concourse/settings/perf-settings-$(SCALE)g.yml \ --var pxf-git-branch=$(BRANCH) \ @@ -258,8 +256,7 @@ query-execution-parquet-1g: .PHONY: set-longevity-pipeline set-longevity-pipeline: $(FLY_CMD) --target=$(CONCOURSE) \ - set-pipeline \ - --check-creds \ + $(SET_PIPELINE) \ --pipeline=dev:longevity_$(YOUR_TAG)_6X_STABLE \ --config pipelines/longevity_pipeline.yml \ --load-vars-from=settings/pxf-multinode-params.yml \ diff --git a/concourse/docker/README.md b/concourse/docker/README.md index b91cb76e01..84893608ad 100644 --- a/concourse/docker/README.md +++ b/concourse/docker/README.md @@ -15,45 +15,13 @@ changes to `pxf-build-base` and is also in charge of tagging the images as ## Available docker images - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 Greenplum 5Greenplum 6Greenplum 7
CentOS7 gpdb5-centos7-test-pxf gpdb6-centos7-test-pxf gpdb7-centos7-test-pxf
OEL7 N/A gpdb6-oel7-test-pxf N/A
Ubuntu 18.04 N/A gpdb6-ubuntu18.04-test-pxf gpdb7-ubuntu18.04-test-pxf
Rocky8 N/A N/A gpdb7-rocky8-test-pxf
MapR on CentOS7 gpdb6-centos7-test-pxf-mapr N/A
- -* Note: GCR_PROJECT_ID is the name of the Google Cloud Project ID +| | Greenplum 5 | Greenplum 6 | Greenplum 7 | +|------------------|--------------------------|-------------------------------|------------------------------| +| CentOS 7 | `gpdb5-centos7-test-pxf` | `gpdb6-centos7-test-pxf` | N/A | +| OEL 7 | N/A | `gpdb6-oel7-test-pxf` | N/A | +| Ubuntu 18.04 | N/A | `gpdb6-ubuntu18.04-test-pxf` | N/A | +| Rocky Linux 8 | N/A | `gpdb6-rocky8-test-pxf` | `gpdb7-rocky8-test-pxf` | +| MapR on CentOS 7 | N/A | `gpdb6-centos7-test-pxf-mapr` | N/A | ## Development docker image @@ -62,3 +30,226 @@ A PXF development docker image can be pulled with the following command: ```shell script docker pull gcr.io/${GCR_PROJECT_ID}/gpdb-pxf-dev/gpdb6-centos7-test-pxf-hdp2:latest ``` + +## Diagram of Container Image Building + +This [Mermaid](https://mermaid.js.org/intro/) diagram details the docker images that are used and created by PXF pipelines and developers. + +```mermaid +%%{init: {'theme':'neutral'}}%% +flowchart TD + classDef subgraphStyle fill:none,stroke-dasharray:5,5,stroke:black + classDef dockerhubStyle fill:#268bd2,color:white,stroke:none + classDef gcrPublicStyle fill:#2aa198,stroke:none,color:white + classDef dockerfileStyle fill:#fdf6e3,stroke:none + classDef pipelineStyle fill:#d33682,color:white,stroke:none + classDef latestStyle fill:#6c71c4,color:white,stroke:none + classDef plainStyle fill:none,stroke:black + + subgraph dockerhub [Official DockerHub] + centos7[centos:7] + rocky8[rockylinux:8] + class centos7 dockerhubStyle + class rocky8 dockerhubStyle + + end + class dockerhub subgraphStyle + + subgraph gcr_images ["GP RelEng Images (gcr.io/data-gpdb-public-images)"] + gp5_centos7_latest[centos-gpdb-dev:7-gcc6.2-llvm3.7] + gp6_centos7_latest[gpdb6-centos7-test:latest] + gp6_ubuntu18_latest[gpdb6-ubuntu18.04-test:latest] + gp6_oel7_latest[gpdb6-oel7-test:latest] + gp6_rocky8_latest[gpdb6-rocky8-test:latest] + gp7_rocky8_latest[gpdb7-rocky8-test:latest] + + class gp5_centos7_latest gcrPublicStyle + class gp6_centos7_latest gcrPublicStyle + class gp6_ubuntu18_latest gcrPublicStyle + class gp6_oel7_latest gcrPublicStyle + class gp6_rocky8_latest gcrPublicStyle + class gp7_rocky8_latest gcrPublicStyle + end + class gcr_images subgraphStyle + + subgraph pxf_dev_base [pxf-dev-base/cloudbuild.yaml] + gp5_centos7_dockerfile[gpdb5/centos7] + gp6_centos7_dockerfile[gpdb6/centos7] + gp6_rocky8_dockerfile[gpdb6/rocky8] + gp6_ubuntu18_dockerfile[gpdb6/ubuntu18.04] + gp6_oel7_dockerfile[gpdb6/oel7] + gp7_rocky8_dockerfile[gpdb7/rocky8] + + class gp5_centos7_dockerfile dockerfileStyle + class gp6_centos7_dockerfile dockerfileStyle + class gp6_rocky8_dockerfile dockerfileStyle + class gp6_ubuntu18_dockerfile dockerfileStyle + class gp6_oel7_dockerfile dockerfileStyle + class gp7_rocky8_dockerfile dockerfileStyle + end + class pxf_dev_base subgraphStyle + + subgraph rpmrebuild [rpmrebuild/cloudbuild.yaml] + rpm_docker_centos7[centos/Dockerfile] + rpm_docker_rocky8[rocky/Dockerfile] + + class rpm_docker_centos7 dockerfileStyle + class rpm_docker_rocky8 dockerfileStyle + end + class rpmrebuild subgraphStyle + + subgraph gcr_data_gpdb_ud [gcr.io/data-gpdb-ud] + subgraph gpdb_pxf_dev [gpdb-pxf-dev] + gp5_centos7_pxf_sha[gpdb5-centos7-test-pxf:$COMMIT_SHA] + gp6_centos7_pxf_sha[gpdb6-centos7-test-pxf:$COMMIT_SHA] + gp6_rocky8_pxf_sha[gpdb6-rocky8-test-pxf:$COMMIT_SHA] + gp6_ubuntu18_pxf_sha[gpdb6-ubuntu18.04-test-pxf:$COMMIT_SHA] + gp6_oel7_pxf_sha[gpdb6-oel7-test-pxf:$COMMIT_SHA] + gp7_rocky8_pxf_sha[gpdb7-rocky8-test-pxf:$COMMIT_SHA] + + class gp5_centos7_pxf_sha plainStyle + class gp6_centos7_pxf_sha plainStyle + class gp6_rocky8_pxf_sha plainStyle + class gp6_ubuntu18_pxf_sha plainStyle + class gp6_oel7_pxf_sha plainStyle + class gp7_rocky8_pxf_sha plainStyle + + gp5_centos7_pxf_latest[gpdb5-centos7-test-pxf:latest] + gp6_centos7_pxf_latest[gpdb6-centos7-test-pxf:latest] + gp6_rocky8_pxf_latest[gpdb6-rocky8-test-pxf:latest] + gp6_ubuntu18_pxf_latest[gpdb6-ubuntu18.04-test-pxf:latest] + gp6_oel7_pxf_latest[gpdb6-oel7-test-pxf:latest] + gp7_rocky8_pxf_latest[gpdb7-rocky8-test-pxf:latest] + + class gp5_centos7_pxf_latest latestStyle + class gp6_centos7_pxf_latest latestStyle + class gp6_rocky8_pxf_latest latestStyle + class gp6_ubuntu18_pxf_latest latestStyle + class gp6_oel7_pxf_latest latestStyle + class gp7_rocky8_pxf_latest latestStyle + end + class gpdb_pxf_dev subgraphStyle + + rpm_centos7_latest[rpmrebuild-centos7:latest] + rpm_rocky8_latest[rpmrebuild-rocky8:latest] + + class rpm_centos7_latest latestStyle + class rpm_rocky8_latest latestStyle + end + class gcr_data_gpdb_ud subgraphStyle + + subgraph local_use_only [For local development use] + subgraph pxf_dev_server [pxf-dev-server/cloudbuild.yaml] + server_dockerfile[Dockerfile] + class server_dockerfile dockerfileStyle + end + class pxf_dev_server subgraphStyle + + subgraph mapr [mapr/cloudbuild.yaml] + mapr_dockerfile[Dockerfile] + class mapr_dockerfile dockerfileStyle + end + class mapr subgraphStyle + + subgraph gcr_data_gpdb_ud_mapr ["MapR Images (gcr.io/data-gpdb-ud)"] + gp6_centos7_pxf_mapr_sha[gpdb-pxf-dev/gpdb6-centos7-test-pxf-mapr:$COMMIT_SHA] + gp6_centos7_pxf_mapr_latest[gpdb-pxf-dev/gpdb6-centos7-test-pxf-mapr:latest] + + class gp6_centos7_pxf_mapr_sha plainStyle + class gp6_centos7_pxf_mapr_latest latestStyle + end + class gcr_data_gpdb_ud_mapr subgraphStyle + + subgraph gcr_data_gpdb_ud_hdp2 ["HDP2 (gcr.io/data-gpdb-ud)"] + gp6_centos7_pxf_hdp2_sha[gpdb-pxf-dev/gpdb6-centos7-test-pxf-hdp2:$COMMIT_SHA] + gp6_centos7_pxf_hdp2_latest[gpdb-pxf-dev/gpdb6-centos7-test-pxf-hdp2] + + class gp6_centos7_pxf_hdp2_sha plainStyle + style gp6_centos7_pxf_hdp2_latest fill:#b58900,color:white,stroke:none + end + class gcr_data_gpdb_ud_hdp2 subgraphStyle + end + class local_use_only subgraphStyle + + subgraph pipelines [Pipelines] + certification + perf + longevity + build[pxf-build] + pr[pxf_pr_pipeline] + + class certification pipelineStyle + class perf pipelineStyle + class longevity pipelineStyle + class build pipelineStyle + class pr pipelineStyle + end + class pipelines subgraphStyle + + gp5_centos7_latest --> gp5_centos7_dockerfile + gp5_centos7_dockerfile -- CloudBuild --> gp5_centos7_pxf_sha + gp5_centos7_pxf_sha -- "tag (concourse pipeline)" --> gp5_centos7_pxf_latest + + gp6_centos7_latest --> gp6_centos7_dockerfile + gp6_centos7_dockerfile -- CloudBuild --> gp6_centos7_pxf_sha + gp6_centos7_pxf_sha -- "tag (concourse pipeline)" --> gp6_centos7_pxf_latest + + gp6_rocky8_latest --> gp6_rocky8_dockerfile + gp6_rocky8_dockerfile -- CloudBuild --> gp6_rocky8_pxf_sha + gp6_rocky8_pxf_sha -- "tag (concourse pipeline)" --> gp6_rocky8_pxf_latest + + gp6_ubuntu18_latest --> gp6_ubuntu18_dockerfile + gp6_ubuntu18_dockerfile -- CloudBuild --> gp6_ubuntu18_pxf_sha + gp6_ubuntu18_pxf_sha -- "tag (concourse pipeline)" --> gp6_ubuntu18_pxf_latest + + gp6_oel7_latest --> gp6_oel7_dockerfile + gp6_oel7_dockerfile -- CloudBuild --> gp6_oel7_pxf_sha + gp6_oel7_pxf_sha -- "tag (concourse pipeline)" --> gp6_oel7_pxf_latest + + gp7_rocky8_latest --> gp7_rocky8_dockerfile + gp7_rocky8_dockerfile -- CloudBuild --> gp7_rocky8_pxf_sha + gp7_rocky8_pxf_sha -- "tag (concourse pipeline)" --> gp7_rocky8_pxf_latest + + centos7 --> rpm_docker_centos7 + rpm_docker_centos7 --> rpm_centos7_latest + rocky8 --> rpm_docker_rocky8 + rpm_docker_rocky8 --> rpm_rocky8_latest + + gp5_centos7_pxf_latest --> mapr_dockerfile + gp6_centos7_pxf_latest --> mapr_dockerfile + mapr_dockerfile -- "CloudBuild (install MapR)" --> gp6_centos7_pxf_mapr_sha + gp6_centos7_pxf_mapr_sha -- "tag (concourse pipeline)" --> gp6_centos7_pxf_mapr_latest + + gp6_centos7_pxf_latest --> server_dockerfile + server_dockerfile -- "CloudBuild (add singlecluster, build deps, & automation deps)" --> gp6_centos7_pxf_hdp2_sha + gp6_centos7_pxf_hdp2_sha --> gp6_centos7_pxf_hdp2_latest + + gp5_centos7_pxf_latest --> certification + gp5_centos7_pxf_latest --> build + gp5_centos7_pxf_latest --> pr + + gp6_centos7_pxf_latest --> certification + gp6_centos7_pxf_latest --> longevity + gp6_centos7_pxf_latest --> perf + gp6_centos7_pxf_latest --> build + gp6_centos7_pxf_latest --> pr + + gp6_rocky8_pxf_latest --> certification + gp6_rocky8_pxf_latest --> perf + gp6_rocky8_pxf_latest --> build + gp6_rocky8_pxf_latest --> pr + + gp6_ubuntu18_pxf_latest --> certification + gp6_ubuntu18_pxf_latest --> build + gp6_ubuntu18_pxf_latest --> pr + + gp6_oel7_pxf_latest --> build + gp6_oel7_pxf_latest --> pr + + gp7_rocky8_pxf_latest --> certification + gp7_rocky8_pxf_latest --> build + gp7_rocky8_pxf_latest --> pr + + rpm_centos7_latest --> build + rpm_rocky8_latest --> build +``` diff --git a/concourse/docker/diagram/Makefile b/concourse/docker/diagram/Makefile deleted file mode 100644 index fe1dd23055..0000000000 --- a/concourse/docker/diagram/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -ifeq (, $(shell type dot)) -$(error "No dot in $(PATH), install graphviz") -endif - -images.svg: images.dot - dot -Tsvg images.dot -o images.svg diff --git a/concourse/docker/diagram/README.md b/concourse/docker/diagram/README.md deleted file mode 100644 index b76aa50da6..0000000000 --- a/concourse/docker/diagram/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Diagram of Container Image Building - -![container image building](./images.svg) - -This diagram details the docker images that are used and created by PXF pipelines and developers. It was generated using graphviz. - -It can be recreated using the dot file by running `make` within this directory. diff --git a/concourse/docker/diagram/images.dot b/concourse/docker/diagram/images.dot deleted file mode 100644 index c804f6cf0a..0000000000 --- a/concourse/docker/diagram/images.dot +++ /dev/null @@ -1,242 +0,0 @@ -digraph pxf_container_image_flow { - subgraph cluster_dockerhub { - label = "Official DockerHub" - style=dashed - node [shape=box3d fillcolor="#268bd2" style=filled fontcolor=white] - - centos7[label="centos:7"] - } - - # gcr.io/data-gpdb-public-images - subgraph cluster_gcr_images { - label="GP RelEng Images (gcr.io/data-gpdb-public-images)" - style=dashed - node [shape=box3d fillcolor="#2aa198" style=filled fontcolor=white] - gp5_centos7_latest[label="centos-gpdb-dev:7-gcc6.2-llvm3.7"] - gp6_centos7_latest[label="gpdb6-centos7-test:latest"] - gp6_ubuntu18_latest[label="gpdb6-ubuntu18.04-test:latest"] - gp6_oel7_latest[label="gpdb6-oel7-test:latest"] - gp7_centos7_latest[label="gpdb7-centos7-test:latest"] - gp7_rocky8_latest[label="gpdb7-rocky8-test:latest"] - gp7_ubuntu18_latest[label="gpdb7-ubuntu18.04-test:latest"] - } - - subgraph cluster_gcr_images_private { - label="GP RelEng Images (gcr.io/data-gpdb-private-images)" - style=dashed - node [shape=box3d fillcolor="#2aa198" style=filled fontcolor=white] - gp6_rhel8_latest[label="gpdb6-rhel8-test:latest"] - gp7_rhel8_latest[label="gpdb7-rhel8-test:latest"] - } - - # PXF Cloudbuild & Dockerfiles - subgraph cluster_pxf_dev_base { - label = "pxf-dev-base/cloudbuild.yaml" - style=dashed - node [shape=note fillcolor="#fdf6e3" style=filled] - - gp5_centos7_dockerfile[label="gpdb5/centos7"] - gp6_centos7_dockerfile[label="gpdb6/centos7"] - gp6_rhel8_dockerfile[label="gpdb6/rhel8"] - gp6_ubuntu18_dockerfile[label="gpdb6/ubuntu18.04"] - gp6_oel7_dockerfile[label="gpdb6/oel7"] - gp7_centos7_dockerfile[label="gpdb7/centos7"] - gp7_rhel8_dockerfile[label="gpdb7/rhel8"] - gp7_rocky8_dockerfile[label="gpdb7/rocky8"] - gp7_ubuntu18_dockerfile[label="gpdb7/ubuntu18.04"] - - } - - subgraph cluster_rpmrebuild { - label = "rpmrebuild/cloudbuild.yaml" - style=dashed - node [shape=note fillcolor="#fdf6e3" style=filled] - - rpm_docker_centos7[label="centos/Dockerfile"] - rpm_docker_rhel8[label="rhel/Dockerfile"] - } - - # UD GCR images - subgraph cluster_gcr_data_gpdb_ud { - label = "gcr.io/data-gpdb-ud" - style=dashed - node [shape=box] - - subgraph cluster_gpdb_pxf_dev { - label = "gpdb-pxf-dev" - style=dashed - node [shape=box] - - gp5_centos7_pxf_sha[label="gpdb5-centos7-test-pxf:$COMMIT_SHA"] - gp6_centos7_pxf_sha[label="gpdb6-centos7-test-pxf:$COMMIT_SHA"] - gp6_rhel8_pxf_sha[label="gpdb6-rhel8-test-pxf:$COMMIT_SHA"] - gp6_ubuntu18_pxf_sha[label="gpdb6-ubuntu18.04-test-pxf:$COMMIT_SHA"] - gp6_oel7_pxf_sha[label="gpdb6-oel7-test-pxf:$COMMIT_SHA"] - gp7_centos7_pxf_sha[label="gpdb7-centos7-test-pxf:$COMMIT_SHA"] - gp7_rhel8_pxf_sha[label="gpdb7-rhel8-test-pxf:$COMMIT_SHA"] - gp7_rocky8_pxf_sha[label="gpdb7-rocky8-test-pxf:$COMMIT_SHA"] - gp7_ubuntu18_pxf_sha[label="gpdb7-ubuntu18.04-test-pxf:$COMMIT_SHA"] - - gp5_centos7_pxf_latest[label="gpdb5-centos7-test-pxf:latest" style=filled fillcolor="#6c71c4" fontcolor=white] - gp6_centos7_pxf_latest[label="gpdb6-centos7-test-pxf:latest" style=filled fillcolor="#6c71c4" fontcolor=white] - gp6_rhel8_pxf_latest[label="gpdb6-rhel8-test-pxf:latest" style=filled fillcolor="#6c71c4" fontcolor=white] - gp6_ubuntu18_pxf_latest[label="gpdb6-ubuntu18.04-test-pxf:latest" style=filled fillcolor="#6c71c4" fontcolor=white] - gp6_oel7_pxf_latest[label="gpdb6-oel7-test-pxf:latest" style=filled fillcolor="#6c71c4" fontcolor=white] - gp7_centos7_pxf_latest[label="gpdb7-centos7-test-pxf:latest" style=filled fillcolor="#6c71c4" fontcolor=white] - gp7_rhel8_pxf_latest[label="gpdb7-rhel8-test-pxf:latest" style=filled fillcolor="#6c71c4" fontcolor=white] - gp7_rocky8_pxf_latest[label="gpdb7-rocky8-test-pxf:latest" style=filled fillcolor="#6c71c4" fontcolor=white] - gp7_ubuntu18_pxf_latest[label="gpdb7-ubuntu18.04-test-pxf:latest" style=filled fillcolor="#6c71c4" fontcolor=white] - } - - rpm_centos7_latest[label="rpmrebuild-centos7:latest" style=filled fillcolor="#6c71c4" fontcolor=white] - rpm_rhel8_latest[label="rpmrebuild-rhel8:latest" style=filled fillcolor="#6c71c4" fontcolor=white] - rpm_rocky8_latest[label="rpmrebuild-rocky8:latest" style=filled fillcolor="#6c71c4" fontcolor=white] - } - - subgraph cluster_pipelines { - label = "Pipelines" - style=dashed - node [shape=box style=filled fillcolor="#d33682 " fontcolor=white] - - certification - perf - longevity - build[label="pxf-build"] - pr[label="pxf_pr_pipeline"] - } - - subgraph cluster_local_use_only { - label = "For local development use" - style=dashed - node [shape=box] - - subgraph cluster_pxf_dev_server { - label = "pxf-dev-server/cloudbuild.yaml" - style=dashed - node [shape=note fillcolor="#fdf6e3" style=filled] - - server_dockerfile[label="Dockerfile"] - } - - subgraph cluster_mapr { - label = "mapr/cloudbuild.yaml" - style=dashed - node [shape=note fillcolor="#fdf6e3" style=filled] - - mapr_dockerfile[label="Dockerfile"] - - } - - subgraph cluster_gcr_data_gpdb_ud_mapr { - label = "MapR Images (gcr.io/data-gpdb-ud)" - style=dashed - node [shape=box] - gp6_centos7_pxf_mapr_sha[label="gpdb-pxf-dev/gpdb6-centos7-test-pxf-mapr:$COMMIT_SHA"] - gp6_centos7_pxf_mapr_latest[label="gpdb-pxf-dev/gpdb6-centos7-test-pxf-mapr:latest" style=filled fillcolor="#6c71c4" fontcolor=white] - - } - - subgraph cluster_gcr_data_gpdb_ud_hdp2 { - label = "HDP2 (gcr.io/data-gpdb-ud)" - style=dashed - node [shape=box] - gp6_centos7_pxf_hdp2_sha[label="gpdb-pxf-dev/gpdb6-centos7-test-pxf-hdp2:$COMMIT_SHA"] - gp6_centos7_pxf_hdp2_latest[label="gpdb-pxf-dev/gpdb6-centos7-test-pxf-hdp2" style=filled fillcolor="#b58900" fontcolor=white] - } - } - gp5_centos7_latest -> gp5_centos7_dockerfile - gp5_centos7_dockerfile -> gp5_centos7_pxf_sha[label="CloudBuild"] - gp5_centos7_pxf_sha -> gp5_centos7_pxf_latest[label="tag (concourse pipeline)"] - - gp6_centos7_latest -> gp6_centos7_dockerfile - gp6_centos7_dockerfile -> gp6_centos7_pxf_sha[label="CloudBuild"] - gp6_centos7_pxf_sha -> gp6_centos7_pxf_latest[label="tag (concourse pipeline)"] - - gp6_rhel8_latest -> gp6_rhel8_dockerfile - gp6_rhel8_dockerfile -> gp6_rhel8_pxf_sha[label="CloudBuild"] - gp6_rhel8_pxf_sha -> gp6_rhel8_pxf_latest[label="tag (concourse pipeline)"] - - gp6_ubuntu18_latest -> gp6_ubuntu18_dockerfile - gp6_ubuntu18_dockerfile -> gp6_ubuntu18_pxf_sha[label="CloudBuild"] - gp6_ubuntu18_pxf_sha -> gp6_ubuntu18_pxf_latest[label="tag (concourse pipeline)"] - - gp6_oel7_latest -> gp6_oel7_dockerfile - gp6_oel7_dockerfile -> gp6_oel7_pxf_sha[label="CloudBuild"] - gp6_oel7_pxf_sha -> gp6_oel7_pxf_latest[label="tag (concourse pipeline)"] - - gp7_centos7_latest -> gp7_centos7_dockerfile - gp7_centos7_dockerfile -> gp7_centos7_pxf_sha[label="CloudBuild"] - gp7_centos7_pxf_sha -> gp7_centos7_pxf_latest[label="tag (concourse pipeline)"] - - gp7_rhel8_latest -> gp7_rhel8_dockerfile - gp7_rhel8_dockerfile -> gp7_rhel8_pxf_sha[label="CloudBuild"] - gp7_rhel8_pxf_sha -> gp7_rhel8_pxf_latest[label="tag (concourse pipeline)"] - - gp7_rocky8_latest -> gp7_rocky8_dockerfile - gp7_rocky8_dockerfile -> gp7_rocky8_pxf_sha[label="CloudBuild"] - gp7_rocky8_pxf_sha -> gp7_rocky8_pxf_latest[label="tag (concourse pipeline)"] - - gp7_ubuntu18_latest -> gp7_ubuntu18_dockerfile - gp7_ubuntu18_dockerfile -> gp7_ubuntu18_pxf_sha[label="CloudBuild"] - gp7_ubuntu18_pxf_sha -> gp7_ubuntu18_pxf_latest[label="tag (concourse pipeline)"] - - centos7 -> rpm_docker_centos7 - rpm_docker_centos7 -> rpm_centos7_latest - - gp6_rhel8_latest -> rpm_docker_rhel8 - gp7_rocky8_latest -> rpm_docker_rhel8 - rpm_docker_rhel8 -> rpm_rhel8_latest - rpm_docker_rhel8 -> rpm_rocky8_latest - - gp5_centos7_pxf_latest -> mapr_dockerfile - gp6_centos7_pxf_latest -> mapr_dockerfile - mapr_dockerfile -> gp6_centos7_pxf_mapr_sha[label="CloudBuild (install MapR)"] - gp6_centos7_pxf_mapr_sha -> gp6_centos7_pxf_mapr_latest[label="tag (concourse pipeline)"] - - gp6_centos7_pxf_latest -> server_dockerfile - server_dockerfile -> gp6_centos7_pxf_hdp2_sha[label="CloudBuild (add singlecluster, build deps, & automation deps)"] - gp6_centos7_pxf_hdp2_sha -> gp6_centos7_pxf_hdp2_latest - - gp5_centos7_pxf_latest -> certification - gp5_centos7_pxf_latest -> build - gp5_centos7_pxf_latest -> pr - - gp6_centos7_pxf_latest -> certification - gp6_centos7_pxf_latest -> longevity - gp6_centos7_pxf_latest -> perf - gp6_centos7_pxf_latest -> build - gp6_centos7_pxf_latest -> pr - - gp6_rhel8_pxf_latest -> certification - gp6_rhel8_pxf_latest -> perf - gp6_rhel8_pxf_latest -> build - gp6_rhel8_pxf_latest -> pr - - gp6_ubuntu18_pxf_latest -> certification - gp6_ubuntu18_pxf_latest -> build - gp6_ubuntu18_pxf_latest -> pr - - gp6_oel7_pxf_latest -> build - gp6_oel7_pxf_latest -> pr - - gp7_centos7_pxf_latest -> build - gp7_centos7_pxf_latest -> pr - - gp7_rhel8_pxf_latest -> certification - gp7_rhel8_pxf_latest -> build - gp7_rhel8_pxf_latest -> pr - - gp7_rocky8_pxf_latest -> certification - gp7_rocky8_pxf_latest -> build - gp7_rocky8_pxf_latest -> pr - - gp7_ubuntu18_pxf_latest -> build - gp7_ubuntu18_pxf_latest -> pr - - rpm_centos7_latest -> build - rpm_rhel8_latest -> build - rpm_rocky8_latest -> build - - gp6_centos7_pxf_mapr_latest -> build[label="Conditionally added based off mapr variable"] - -} diff --git a/concourse/docker/diagram/images.svg b/concourse/docker/diagram/images.svg deleted file mode 100644 index 02aa07a9d4..0000000000 --- a/concourse/docker/diagram/images.svg +++ /dev/null @@ -1,900 +0,0 @@ - - - - - - -pxf_container_image_flow - - -cluster_dockerhub - -Official DockerHub - - -cluster_gcr_images - -GP RelEng Images (gcr.io/data-gpdb-public-images) - - -cluster_pxf_dev_base - -pxf-dev-base/cloudbuild.yaml - - -cluster_rpmrebuild - -rpmrebuild/cloudbuild.yaml - - -cluster_gcr_data_gpdb_ud - -gcr.io/data-gpdb-ud - - -cluster_gpdb_pxf_dev - -gpdb-pxf-dev - - -cluster_pipelines - -Pipelines - - -cluster_local_use_only - -For local development use - - -cluster_pxf_dev_server - -pxf-dev-server/cloudbuild.yaml - - -cluster_mapr - -mapr/cloudbuild.yaml - - -cluster_gcr_data_gpdb_ud_mapr - -MapR Images (gcr.io/data-gpdb-ud) - - -cluster_gcr_data_gpdb_ud_hdp2 - -HDP2 (gcr.io/data-gpdb-ud) - - -cluster_gcr_images_private - -GP RelEng Images (gcr.io/data-gpdb-private-images) - - - -centos7 - - - - -centos:7 - - - -rpm_docker_centos7 - - - -centos/Dockerfile - - - -centos7->rpm_docker_centos7 - - - - - -gp5_centos7_latest - - - - -centos-gpdb-dev:7-gcc6.2-llvm3.7 - - - -gp5_centos7_dockerfile - - - -gpdb5/centos7 - - - -gp5_centos7_latest->gp5_centos7_dockerfile - - - - - -gp6_centos7_latest - - - - -gpdb6-centos7-test:latest - - - -gp6_centos7_dockerfile - - - -gpdb6/centos7 - - - -gp6_centos7_latest->gp6_centos7_dockerfile - - - - - -gp6_ubuntu18_latest - - - - -gpdb6-ubuntu18.04-test:latest - - - -gp6_ubuntu18_dockerfile - - - -gpdb6/ubuntu18.04 - - - -gp6_ubuntu18_latest->gp6_ubuntu18_dockerfile - - - - - -gp6_oel7_latest - - - - -gpdb6-oel7-test:latest - - - -gp6_oel7_dockerfile - - - -gpdb6/oel7 - - - -gp6_oel7_latest->gp6_oel7_dockerfile - - - - - -gp7_centos7_latest - - - - -gpdb7-centos7-test:latest - - - -gp7_centos7_dockerfile - - - -gpdb7/centos7 - - - -gp7_centos7_latest->gp7_centos7_dockerfile - - - - - -gp7_rocky8_latest - - - - -gpdb7-rocky8-test:latest - - - -gp7_rocky8_dockerfile - - - -gpdb7/rocky8 - - - -gp7_rocky8_latest->gp7_rocky8_dockerfile - - - - - -rpm_docker_rhel8 - - - -rhel/Dockerfile - - - -gp7_rocky8_latest->rpm_docker_rhel8 - - - - - -gp7_ubuntu18_latest - - - - -gpdb7-ubuntu18.04-test:latest - - - -gp7_ubuntu18_dockerfile - - - -gpdb7/ubuntu18.04 - - - -gp7_ubuntu18_latest->gp7_ubuntu18_dockerfile - - - - - -gp6_rhel8_latest - - - - -gpdb6-rhel8-test:latest - - - -gp6_rhel8_dockerfile - - - -gpdb6/rhel8 - - - -gp6_rhel8_latest->gp6_rhel8_dockerfile - - - - - -gp6_rhel8_latest->rpm_docker_rhel8 - - - - - -gp7_rhel8_latest - - - - -gpdb7-rhel8-test:latest - - - -gp7_rhel8_dockerfile - - - -gpdb7/rhel8 - - - -gp7_rhel8_latest->gp7_rhel8_dockerfile - - - - - -gp5_centos7_pxf_sha - -gpdb5-centos7-test-pxf:$COMMIT_SHA - - - -gp5_centos7_dockerfile->gp5_centos7_pxf_sha - - -CloudBuild - - - -gp6_centos7_pxf_sha - -gpdb6-centos7-test-pxf:$COMMIT_SHA - - - -gp6_centos7_dockerfile->gp6_centos7_pxf_sha - - -CloudBuild - - - -gp6_rhel8_pxf_sha - -gpdb6-rhel8-test-pxf:$COMMIT_SHA - - - -gp6_rhel8_dockerfile->gp6_rhel8_pxf_sha - - -CloudBuild - - - -gp6_ubuntu18_pxf_sha - -gpdb6-ubuntu18.04-test-pxf:$COMMIT_SHA - - - -gp6_ubuntu18_dockerfile->gp6_ubuntu18_pxf_sha - - -CloudBuild - - - -gp6_oel7_pxf_sha - -gpdb6-oel7-test-pxf:$COMMIT_SHA - - - -gp6_oel7_dockerfile->gp6_oel7_pxf_sha - - -CloudBuild - - - -gp7_centos7_pxf_sha - -gpdb7-centos7-test-pxf:$COMMIT_SHA - - - -gp7_centos7_dockerfile->gp7_centos7_pxf_sha - - -CloudBuild - - - -gp7_rhel8_pxf_sha - -gpdb7-rhel8-test-pxf:$COMMIT_SHA - - - -gp7_rhel8_dockerfile->gp7_rhel8_pxf_sha - - -CloudBuild - - - -gp7_rocky8_pxf_sha - -gpdb7-rocky8-test-pxf:$COMMIT_SHA - - - -gp7_rocky8_dockerfile->gp7_rocky8_pxf_sha - - -CloudBuild - - - -gp7_ubuntu18_pxf_sha - -gpdb7-ubuntu18.04-test-pxf:$COMMIT_SHA - - - -gp7_ubuntu18_dockerfile->gp7_ubuntu18_pxf_sha - - -CloudBuild - - - -rpm_centos7_latest - -rpmrebuild-centos7:latest - - - -rpm_docker_centos7->rpm_centos7_latest - - - - - -rpm_rhel8_latest - -rpmrebuild-rhel8:latest - - - -rpm_docker_rhel8->rpm_rhel8_latest - - - - - -rpm_rocky8_latest - -rpmrebuild-rocky8:latest - - - -rpm_docker_rhel8->rpm_rocky8_latest - - - - - -gp5_centos7_pxf_latest - -gpdb5-centos7-test-pxf:latest - - - -gp5_centos7_pxf_sha->gp5_centos7_pxf_latest - - -tag (concourse pipeline) - - - -gp6_centos7_pxf_latest - -gpdb6-centos7-test-pxf:latest - - - -gp6_centos7_pxf_sha->gp6_centos7_pxf_latest - - -tag (concourse pipeline) - - - -gp6_rhel8_pxf_latest - -gpdb6-rhel8-test-pxf:latest - - - -gp6_rhel8_pxf_sha->gp6_rhel8_pxf_latest - - -tag (concourse pipeline) - - - -gp6_ubuntu18_pxf_latest - -gpdb6-ubuntu18.04-test-pxf:latest - - - -gp6_ubuntu18_pxf_sha->gp6_ubuntu18_pxf_latest - - -tag (concourse pipeline) - - - -gp6_oel7_pxf_latest - -gpdb6-oel7-test-pxf:latest - - - -gp6_oel7_pxf_sha->gp6_oel7_pxf_latest - - -tag (concourse pipeline) - - - -gp7_centos7_pxf_latest - -gpdb7-centos7-test-pxf:latest - - - -gp7_centos7_pxf_sha->gp7_centos7_pxf_latest - - -tag (concourse pipeline) - - - -gp7_rhel8_pxf_latest - -gpdb7-rhel8-test-pxf:latest - - - -gp7_rhel8_pxf_sha->gp7_rhel8_pxf_latest - - -tag (concourse pipeline) - - - -gp7_rocky8_pxf_latest - -gpdb7-rocky8-test-pxf:latest - - - -gp7_rocky8_pxf_sha->gp7_rocky8_pxf_latest - - -tag (concourse pipeline) - - - -gp7_ubuntu18_pxf_latest - -gpdb7-ubuntu18.04-test-pxf:latest - - - -gp7_ubuntu18_pxf_sha->gp7_ubuntu18_pxf_latest - - -tag (concourse pipeline) - - - -certification - -certification - - - -gp5_centos7_pxf_latest->certification - - - - - -build - -pxf-build - - - -gp5_centos7_pxf_latest->build - - - - - -pr - -pxf_pr_pipeline - - - -gp5_centos7_pxf_latest->pr - - - - - -mapr_dockerfile - - - -Dockerfile - - - -gp5_centos7_pxf_latest->mapr_dockerfile - - - - - -gp6_centos7_pxf_latest->certification - - - - - -perf - -perf - - - -gp6_centos7_pxf_latest->perf - - - - - -longevity - -longevity - - - -gp6_centos7_pxf_latest->longevity - - - - - -gp6_centos7_pxf_latest->build - - - - - -gp6_centos7_pxf_latest->pr - - - - - -server_dockerfile - - - -Dockerfile - - - -gp6_centos7_pxf_latest->server_dockerfile - - - - - -gp6_centos7_pxf_latest->mapr_dockerfile - - - - - -gp6_rhel8_pxf_latest->certification - - - - - -gp6_rhel8_pxf_latest->perf - - - - - -gp6_rhel8_pxf_latest->build - - - - - -gp6_rhel8_pxf_latest->pr - - - - - -gp6_ubuntu18_pxf_latest->certification - - - - - -gp6_ubuntu18_pxf_latest->build - - - - - -gp6_ubuntu18_pxf_latest->pr - - - - - -gp6_oel7_pxf_latest->build - - - - - -gp6_oel7_pxf_latest->pr - - - - - -gp7_centos7_pxf_latest->build - - - - - -gp7_centos7_pxf_latest->pr - - - - - -gp7_rhel8_pxf_latest->certification - - - - - -gp7_rhel8_pxf_latest->build - - - - - -gp7_rhel8_pxf_latest->pr - - - - - -gp7_rocky8_pxf_latest->certification - - - - - -gp7_rocky8_pxf_latest->build - - - - - -gp7_rocky8_pxf_latest->pr - - - - - -gp7_ubuntu18_pxf_latest->build - - - - - -gp7_ubuntu18_pxf_latest->pr - - - - - -rpm_centos7_latest->build - - - - - -rpm_rhel8_latest->build - - - - - -rpm_rocky8_latest->build - - - - - -gp6_centos7_pxf_hdp2_sha - -gpdb-pxf-dev/gpdb6-centos7-test-pxf-hdp2:$COMMIT_SHA - - - -server_dockerfile->gp6_centos7_pxf_hdp2_sha - - -CloudBuild (add singlecluster, build deps, & automation deps) - - - -gp6_centos7_pxf_mapr_sha - -gpdb-pxf-dev/gpdb6-centos7-test-pxf-mapr:$COMMIT_SHA - - - -mapr_dockerfile->gp6_centos7_pxf_mapr_sha - - -CloudBuild (install MapR) - - - -gp6_centos7_pxf_mapr_latest - -gpdb-pxf-dev/gpdb6-centos7-test-pxf-mapr:latest - - - -gp6_centos7_pxf_mapr_sha->gp6_centos7_pxf_mapr_latest - - -tag (concourse pipeline) - - - -gp6_centos7_pxf_mapr_latest->build - - -Conditionally added based off mapr variable - - - -gp6_centos7_pxf_hdp2_latest - -gpdb-pxf-dev/gpdb6-centos7-test-pxf-hdp2 - - - -gp6_centos7_pxf_hdp2_sha->gp6_centos7_pxf_hdp2_latest - - - - - diff --git a/concourse/docker/pxf-dev-base/README.md b/concourse/docker/pxf-dev-base/README.md index c1a49d6112..c064f34a06 100644 --- a/concourse/docker/pxf-dev-base/README.md +++ b/concourse/docker/pxf-dev-base/README.md @@ -29,7 +29,8 @@ directory. The `cloudbuild.yaml` file produces the following docker images: You can build these images individually by first setting these local variables: ``` -export GO_VERSION=1.19 +export GO_VERSION=1.19.6 +export MAVEN_VERSION=3.9.2 ``` ## Greenplum 5 Images @@ -44,6 +45,7 @@ command to build the image: pushd ~/workspace/pxf/concourse/docker/pxf-dev-base/ docker build \ --build-arg=BASE_IMAGE=gcr.io/data-gpdb-public-images/gpdb5-centos7-build-test:latest \ + --build-arg=GO_VERSION=${GO_VERSION} \ --tag=gpdb5-centos7-test-pxf \ -f ~/workspace/pxf/concourse/docker/pxf-dev-base/gpdb5/centos7/Dockerfile \ . @@ -60,21 +62,24 @@ command to build the image: docker build \ --build-arg=BASE_IMAGE=gcr.io/data-gpdb-public-images/gpdb6-centos7-test:latest \ --build-arg=GO_VERSION=${GO_VERSION} \ + --build-arg=MAVEN_VERSION=${MAVEN_VERSION} \ --tag=gpdb6-centos7-test-pxf \ -f ~/workspace/pxf/concourse/docker/pxf-dev-base/gpdb6/centos7/Dockerfile \ . popd -### Docker gpdb6-rhel8-test-pxf-image image +### Docker gpdb6-rocky8-test-pxf-image image Build this image for Greenplum 6 running on Rhel 8. Run the following command to build the image: pushd ~/workspace/pxf/concourse/docker/pxf-dev-base/ docker build \ - --build-arg=BASE_IMAGE=gcr.io/data-gpdb-private-images/gpdb6-rhel8-test:latest \ - --tag=gpdb6-rhel8-test-pxf \ - -f ~/workspace/pxf/concourse/docker/pxf-dev-base/gpdb6/rhel8/Dockerfile \ + --build-arg=BASE_IMAGE=gcr.io/data-gpdb-public-images/gpdb6-rocky8-test:latest \ + --build-arg=GO_VERSION=${GO_VERSION} \ + --build-arg=MAVEN_VERSION=${MAVEN_VERSION} \ + --tag=gpdb6-rocky8-test-pxf \ + -f ~/workspace/pxf/concourse/docker/pxf-dev-base/gpdb6/rocky8/Dockerfile \ . popd @@ -87,6 +92,7 @@ command to build the image: docker build \ --build-arg=BASE_IMAGE=gcr.io/data-gpdb-public-images/gpdb6-ubuntu18.04-test:latest \ --build-arg=GO_VERSION=${GO_VERSION} \ + --build-arg=MAVEN_VERSION=${MAVEN_VERSION} \ --tag=gpdb6-ubuntu18.04-test-pxf \ -f ~/workspace/pxf/concourse/docker/pxf-dev-base/gpdb6/ubuntu18.04/Dockerfile \ . @@ -101,6 +107,7 @@ following command to build the image: docker build \ --build-arg=BASE_IMAGE=gcr.io/data-gpdb-public-images/gpdb6-oel7-test:latest \ --build-arg=GO_VERSION=${GO_VERSION} \ + --build-arg=MAVEN_VERSION=${MAVEN_VERSION} \ --tag=gpdb6-oel7-test-pxf \ -f ~/workspace/pxf/concourse/docker/pxf-dev-base/gpdb6/oel7/Dockerfile \ . @@ -108,36 +115,7 @@ following command to build the image: ## Greenplum 7 Images -### Docker gpdb7-centos7-test-pxf-image image - -TODO: ===> remove this if Greenplum 7 will not be supported on Centos7 <=== - -Build this image for Greenplum 7 running on CentOS 7. Run the following -command to build the image: - - pushd ~/workspace/pxf/concourse/docker/pxf-dev-base/ - docker build \ - --build-arg=BASE_IMAGE=gcr.io/data-gpdb-public-images/gpdb7-centos7-test:latest \ - --build-arg=GO_VERSION=${GO_VERSION} \ - --tag=gpdb7-centos7-test-pxf \ - -f ~/workspace/pxf/concourse/docker/pxf-dev-base/gpdb7/centos7/Dockerfile \ - . - popd - -### Docker gpdb7-rhel8-test-pxf-image image - -Build this image for Greenplum 7 running on Rhel 8. Run the following -command to build the image: - - pushd ~/workspace/pxf/concourse/docker/pxf-dev-base/ - docker build \ - --build-arg=BASE_IMAGE=gcr.io/data-gpdb-private-images/gpdb7-rhel8-test:latest \ - --tag=gpdb7-rhel8-test-pxf \ - -f ~/workspace/pxf/concourse/docker/pxf-dev-base/gpdb7/rhel8/Dockerfile \ - . - popd - -### Docker gpdb7-rhel8-test-pxf-image image +### Docker gpdb7-rocky8-test-pxf-image image Build this image for Greenplum 7 running on Rocky 8. Run the following command to build the image: @@ -145,21 +123,9 @@ command to build the image: pushd ~/workspace/pxf/concourse/docker/pxf-dev-base/ docker build \ --build-arg=BASE_IMAGE=gcr.io/data-gpdb-public-images/gpdb7-rocky8-test:latest \ + --build-arg=GO_VERSION=${GO_VERSION} \ + --build-arg=MAVEN_VERSION=${MAVEN_VERSION} \ --tag=gpdb7-rocky8-test-pxf \ -f ~/workspace/pxf/concourse/docker/pxf-dev-base/gpdb7/rocky8/Dockerfile \ . popd - -### Docker gpdb7-ubuntu18.04-test-pxf-image image - -Build this image for Greenplum 7 running on Ubuntu 18.04. Run the following -command to build the image: - - pushd ~/workspace/pxf/concourse/docker/pxf-dev-base/ - docker build \ - --build-arg=BASE_IMAGE=gcr.io/data-gpdb-public-images/gpdb7-ubuntu18.04-test:latest \ - --build-arg=GO_VERSION=${GO_VERSION} \ - --tag=gpdb7-ubuntu18.04-test-pxf \ - -f ~/workspace/pxf/concourse/docker/pxf-dev-base/gpdb7/ubuntu18.04/Dockerfile \ - . - popd diff --git a/concourse/docker/pxf-dev-base/cloudbuild.yaml b/concourse/docker/pxf-dev-base/cloudbuild.yaml index d817965fdd..a63b12fb80 100644 --- a/concourse/docker/pxf-dev-base/cloudbuild.yaml +++ b/concourse/docker/pxf-dev-base/cloudbuild.yaml @@ -44,7 +44,6 @@ steps: # GPDB 6 Images ############################################################################## -# Corresponds to the docker-gpdb-pxf-dev-centos7 job in the docker pipeline - name: 'gcr.io/cloud-builders/docker' id: gpdb6-centos7-test-pxf-image-cache entrypoint: 'bash' @@ -61,6 +60,7 @@ steps: - 'build' - '--build-arg=BASE_IMAGE=${_BASE_IMAGE_REPOSITORY}/gpdb6-centos7-test:latest' - '--build-arg=GO_VERSION=${_GO_VERSION}' + - '--build-arg=MAVEN_VERSION=${_MAVEN_VERSION}' - '--tag=gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb6-centos7-test-pxf:$COMMIT_SHA' - '--cache-from' - 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb6-centos7-test-pxf:latest' @@ -70,33 +70,32 @@ steps: waitFor: - gpdb6-centos7-test-pxf-image-cache -# Corresponds to the docker-gpdb-pxf-dev-rhel8 job in the docker pipeline - name: 'gcr.io/cloud-builders/docker' - id: gpdb6-rhel8-test-pxf-image-cache + id: gpdb6-rocky8-test-pxf-image-cache entrypoint: 'bash' args: - '-c' - | mkdir -p /workspace/build - docker pull gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb6-rhel8-test-pxf:latest || exit 0 + docker pull gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb6-rocky8-test-pxf:latest || exit 0 waitFor: ['-'] - name: 'gcr.io/cloud-builders/docker' - id: gpdb6-rhel8-test-pxf-image + id: gpdb6-rocky8-test-pxf-image args: - 'build' - - '--build-arg=BASE_IMAGE=${_PRIVATE_BASE_IMAGE_REPOSITORY}/gpdb6-rhel8-test:latest' + - '--build-arg=BASE_IMAGE=${_BASE_IMAGE_REPOSITORY}/gpdb6-rocky8-test:latest' - '--build-arg=GO_VERSION=${_GO_VERSION}' - - '--tag=gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb6-rhel8-test-pxf:$COMMIT_SHA' + - '--build-arg=MAVEN_VERSION=${_MAVEN_VERSION}' + - '--tag=gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb6-rocky8-test-pxf:$COMMIT_SHA' - '--cache-from' - - 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb6-rhel8-test-pxf:latest' + - 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb6-rocky8-test-pxf:latest' - '-f' - - 'concourse/docker/pxf-dev-base/gpdb6/rhel8/Dockerfile' + - 'concourse/docker/pxf-dev-base/gpdb6/rocky8/Dockerfile' - '/workspace/build/' waitFor: - - gpdb6-rhel8-test-pxf-image-cache + - gpdb6-rocky8-test-pxf-image-cache -# Corresponds to the docker-gpdb-pxf-dev-ubuntu18 job in the docker pipeline - name: 'gcr.io/cloud-builders/docker' id: gpdb6-ubuntu18.04-test-pxf-image-cache entrypoint: 'bash' @@ -113,6 +112,7 @@ steps: - 'build' - '--build-arg=BASE_IMAGE=${_BASE_IMAGE_REPOSITORY}/gpdb6-ubuntu18.04-test:latest' - '--build-arg=GO_VERSION=${_GO_VERSION}' + - '--build-arg=MAVEN_VERSION=${_MAVEN_VERSION}' - '--tag=gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb6-ubuntu18.04-test-pxf:$COMMIT_SHA' - '--cache-from' - 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb6-ubuntu18.04-test-pxf:latest' @@ -139,6 +139,7 @@ steps: - 'build' - '--build-arg=BASE_IMAGE=${_BASE_IMAGE_REPOSITORY}/gpdb6-oel7-test:latest' - '--build-arg=GO_VERSION=${_GO_VERSION}' + - '--build-arg=MAVEN_VERSION=${_MAVEN_VERSION}' - '--tag=gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb6-oel7-test-pxf:$COMMIT_SHA' - '--cache-from' - 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb6-oel7-test-pxf:latest' @@ -152,59 +153,7 @@ steps: # GPDB 7 Images ############################################################################## -# Greenplum 7 Centos 7 Image -- name: 'gcr.io/cloud-builders/docker' - id: gpdb7-centos7-test-pxf-image-cache - entrypoint: 'bash' - args: - - '-c' - - | - mkdir -p /workspace/build - docker pull gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-centos7-test-pxf:latest || exit 0 - waitFor: ['-'] - -- name: 'gcr.io/cloud-builders/docker' - id: gpdb7-centos7-test-pxf-image - args: - - 'build' - - '--build-arg=BASE_IMAGE=${_BASE_IMAGE_REPOSITORY}/gpdb7-centos7-test:latest' - - '--build-arg=GO_VERSION=${_GO_VERSION}' - - '--tag=gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-centos7-test-pxf:$COMMIT_SHA' - - '--cache-from' - - 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-centos7-test-pxf:latest' - - '-f' - - 'concourse/docker/pxf-dev-base/gpdb7/centos7/Dockerfile' - - '/workspace/build/' - waitFor: - - gpdb7-centos7-test-pxf-image-cache - -# Greenplum 7 Rhel 8 Image -- name: 'gcr.io/cloud-builders/docker' - id: gpdb7-rhel8-test-pxf-image-cache - entrypoint: 'bash' - args: - - '-c' - - | - mkdir -p /workspace/build - docker pull gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-rhel8-test-pxf:latest || exit 0 - waitFor: ['-'] - -- name: 'gcr.io/cloud-builders/docker' - id: gpdb7-rhel8-test-pxf-image - args: - - 'build' - - '--build-arg=BASE_IMAGE=${_PRIVATE_BASE_IMAGE_REPOSITORY}/gpdb7-rhel8-test:latest' - - '--build-arg=GO_VERSION=${_GO_VERSION}' - - '--tag=gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-rhel8-test-pxf:$COMMIT_SHA' - - '--cache-from' - - 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-rhel8-test-pxf:latest' - - '-f' - - 'concourse/docker/pxf-dev-base/gpdb7/rhel8/Dockerfile' - - '/workspace/build/' - waitFor: - - gpdb7-rhel8-test-pxf-image-cache - - # Greenplum 7 Rocky 8 Image +# Greenplum 7 Rocky 8 Image - name: 'gcr.io/cloud-builders/docker' id: gpdb7-rocky8-test-pxf-image-cache entrypoint: 'bash' @@ -221,6 +170,7 @@ steps: - 'build' - '--build-arg=BASE_IMAGE=${_BASE_IMAGE_REPOSITORY}/gpdb7-rocky8-test:latest' - '--build-arg=GO_VERSION=${_GO_VERSION}' + - '--build-arg=MAVEN_VERSION=${_MAVEN_VERSION}' - '--tag=gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-rocky8-test-pxf:$COMMIT_SHA' - '--cache-from' - 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-rocky8-test-pxf:latest' @@ -230,43 +180,15 @@ steps: waitFor: - gpdb7-rocky8-test-pxf-image-cache -# Greenplum 7 Ubuntu 18.04 Image -- name: 'gcr.io/cloud-builders/docker' - id: gpdb7-ubuntu18.04-test-pxf-image-cache - entrypoint: 'bash' - args: - - '-c' - - | - mkdir -p /workspace/build - docker pull gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-ubuntu18.04-test-pxf:latest || exit 0 - waitFor: ['-'] - -- name: 'gcr.io/cloud-builders/docker' - id: gpdb7-ubuntu18.04-test-pxf-image - args: - - 'build' - - '--build-arg=BASE_IMAGE=${_BASE_IMAGE_REPOSITORY}/gpdb7-ubuntu18.04-test:latest' - - '--build-arg=GO_VERSION=${_GO_VERSION}' - - '--tag=gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-ubuntu18.04-test-pxf:$COMMIT_SHA' - - '--cache-from' - - 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-ubuntu18.04-test-pxf:latest' - - '-f' - - 'concourse/docker/pxf-dev-base/gpdb7/ubuntu18.04/Dockerfile' - - '/workspace/build/' - waitFor: - - gpdb7-ubuntu18.04-test-pxf-image-cache - substitutions: _GO_VERSION: '1.19.6' # default values + _MAVEN_VERSION: '3.9.2' # default values # Push images to Cloud Build to Container Registry images: - 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb5-centos7-test-pxf:$COMMIT_SHA' - 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb6-centos7-test-pxf:$COMMIT_SHA' -- 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb6-rhel8-test-pxf:$COMMIT_SHA' +- 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb6-rocky8-test-pxf:$COMMIT_SHA' - 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb6-ubuntu18.04-test-pxf:$COMMIT_SHA' - 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb6-oel7-test-pxf:$COMMIT_SHA' -- 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-centos7-test-pxf:$COMMIT_SHA' -- 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-rhel8-test-pxf:$COMMIT_SHA' - 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-rocky8-test-pxf:$COMMIT_SHA' -- 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-ubuntu18.04-test-pxf:$COMMIT_SHA' diff --git a/concourse/docker/pxf-dev-base/gpdb6/centos7/Dockerfile b/concourse/docker/pxf-dev-base/gpdb6/centos7/Dockerfile index a197b2b6d0..a485b47f5d 100644 --- a/concourse/docker/pxf-dev-base/gpdb6/centos7/Dockerfile +++ b/concourse/docker/pxf-dev-base/gpdb6/centos7/Dockerfile @@ -24,7 +24,7 @@ RUN useradd -s /sbin/nologin -d /opt/minio minio \ && chmod +x /opt/minio/bin/minio \ && chown -R minio:minio /opt/minio -ARG MAVEN_VERSION=3.6.3 +ARG MAVEN_VERSION ARG USER_HOME_DIR="/root" ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries diff --git a/concourse/docker/pxf-dev-base/gpdb6/oel7/Dockerfile b/concourse/docker/pxf-dev-base/gpdb6/oel7/Dockerfile index cfdca92d00..a27a00e8c3 100644 --- a/concourse/docker/pxf-dev-base/gpdb6/oel7/Dockerfile +++ b/concourse/docker/pxf-dev-base/gpdb6/oel7/Dockerfile @@ -9,7 +9,7 @@ RUN mkdir -p /tmp/pxf_src/ && cd /tmp \ && wget -O go.tgz -q https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz \ && rm -rf /usr/local/go && tar -C /usr/local -xzf go.tgz && rm go.tgz -ARG MAVEN_VERSION=3.6.3 +ARG MAVEN_VERSION ARG USER_HOME_DIR="/root" ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries diff --git a/concourse/docker/pxf-dev-base/gpdb6/rhel8/Dockerfile b/concourse/docker/pxf-dev-base/gpdb6/rocky8/Dockerfile similarity index 97% rename from concourse/docker/pxf-dev-base/gpdb6/rhel8/Dockerfile rename to concourse/docker/pxf-dev-base/gpdb6/rocky8/Dockerfile index 9467e788e9..aa42adcf48 100644 --- a/concourse/docker/pxf-dev-base/gpdb6/rhel8/Dockerfile +++ b/concourse/docker/pxf-dev-base/gpdb6/rocky8/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=gcr.io/data-gpdb-private-images/gpdb6-rhel8-test:latest +ARG BASE_IMAGE=gcr.io/data-gpdb-public-images/gpdb6-rocky8-test:latest FROM ${BASE_IMAGE} @@ -24,7 +24,7 @@ RUN useradd -s /sbin/nologin -d /opt/minio minio \ && chmod +x /opt/minio/bin/minio \ && chown -R minio:minio /opt/minio -ARG MAVEN_VERSION=3.6.3 +ARG MAVEN_VERSION ARG USER_HOME_DIR="/root" ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries diff --git a/concourse/docker/pxf-dev-base/gpdb6/ubuntu18.04/Dockerfile b/concourse/docker/pxf-dev-base/gpdb6/ubuntu18.04/Dockerfile index 01062bf2aa..289d6dec58 100644 --- a/concourse/docker/pxf-dev-base/gpdb6/ubuntu18.04/Dockerfile +++ b/concourse/docker/pxf-dev-base/gpdb6/ubuntu18.04/Dockerfile @@ -12,7 +12,7 @@ RUN mkdir -p /tmp/pxf_src/ && cd /tmp \ # install dependencies that are missing on the base images # install a specific version of perl for tinc -ARG MAVEN_VERSION=3.6.3 +ARG MAVEN_VERSION ARG USER_HOME_DIR="/root" ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries diff --git a/concourse/docker/pxf-dev-base/gpdb7/centos7/Dockerfile b/concourse/docker/pxf-dev-base/gpdb7/centos7/Dockerfile index 8bf26b0250..ca869cdd58 100644 --- a/concourse/docker/pxf-dev-base/gpdb7/centos7/Dockerfile +++ b/concourse/docker/pxf-dev-base/gpdb7/centos7/Dockerfile @@ -18,7 +18,7 @@ RUN useradd -s /sbin/nologin -d /opt/minio minio \ && chmod +x /opt/minio/bin/minio \ && chown -R minio:minio /opt/minio -ARG MAVEN_VERSION=3.6.3 +ARG MAVEN_VERSION ARG USER_HOME_DIR="/root" ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries diff --git a/concourse/docker/pxf-dev-base/gpdb7/rhel8/Dockerfile b/concourse/docker/pxf-dev-base/gpdb7/rhel8/Dockerfile index 2fee0c3208..619d69a180 100644 --- a/concourse/docker/pxf-dev-base/gpdb7/rhel8/Dockerfile +++ b/concourse/docker/pxf-dev-base/gpdb7/rhel8/Dockerfile @@ -18,7 +18,7 @@ RUN useradd -s /sbin/nologin -d /opt/minio minio \ && chmod +x /opt/minio/bin/minio \ && chown -R minio:minio /opt/minio -ARG MAVEN_VERSION=3.6.3 +ARG MAVEN_VERSION ARG USER_HOME_DIR="/root" ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries diff --git a/concourse/docker/pxf-dev-base/gpdb7/rocky8/Dockerfile b/concourse/docker/pxf-dev-base/gpdb7/rocky8/Dockerfile index c20ae43667..6f732bba43 100644 --- a/concourse/docker/pxf-dev-base/gpdb7/rocky8/Dockerfile +++ b/concourse/docker/pxf-dev-base/gpdb7/rocky8/Dockerfile @@ -18,7 +18,7 @@ RUN useradd -s /sbin/nologin -d /opt/minio minio \ && chmod +x /opt/minio/bin/minio \ && chown -R minio:minio /opt/minio -ARG MAVEN_VERSION=3.6.3 +ARG MAVEN_VERSION ARG USER_HOME_DIR="/root" ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries diff --git a/concourse/docker/pxf-dev-base/gpdb7/ubuntu18.04/Dockerfile b/concourse/docker/pxf-dev-base/gpdb7/ubuntu18.04/Dockerfile index 3e2e465978..2ef6b480a6 100644 --- a/concourse/docker/pxf-dev-base/gpdb7/ubuntu18.04/Dockerfile +++ b/concourse/docker/pxf-dev-base/gpdb7/ubuntu18.04/Dockerfile @@ -12,7 +12,7 @@ RUN mkdir -p /tmp/pxf_src/ && cd /tmp \ # install dependencies that are missing on the base images # install a specific version of perl for tinc -ARG MAVEN_VERSION=3.6.3 +ARG MAVEN_VERSION ARG USER_HOME_DIR="/root" ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries diff --git a/concourse/docker/rpmrebuild/cloudbuild.yaml b/concourse/docker/rpmrebuild/cloudbuild.yaml index 041304443c..23e0f671ef 100644 --- a/concourse/docker/rpmrebuild/cloudbuild.yaml +++ b/concourse/docker/rpmrebuild/cloudbuild.yaml @@ -15,32 +15,19 @@ steps: - 'concourse/docker/rpmrebuild/centos' waitFor: ['-'] -# Builds the rpmrebuild-rhel8 image -- name: 'gcr.io/cloud-builders/docker' - id: rpmrebuild-rhel8-image - args: - - 'build' - - '--build-arg=BASE_IMAGE=${_PRIVATE_BASE_IMAGE_REPOSITORY}/gpdb6-rhel8-test:latest' - - '--tag=gcr.io/$PROJECT_ID/rpmrebuild-rhel8:latest' - - '-f' - - 'concourse/docker/rpmrebuild/rhel/Dockerfile' - - 'concourse/docker/rpmrebuild/rhel' - waitFor: ['-'] - - # Builds the rpmrebuild-rocky8 image +# Builds the rpmrebuild-rocky8 image - name: 'gcr.io/cloud-builders/docker' id: rpmrebuild-rocky8-image args: - 'build' - - '--build-arg=BASE_IMAGE=${_BASE_IMAGE_REPOSITORY}/gpdb7-rocky8-test:latest' + - '--build-arg=BASE_IMAGE=rockylinux:8' - '--tag=gcr.io/$PROJECT_ID/rpmrebuild-rocky8:latest' - '-f' - - 'concourse/docker/rpmrebuild/rhel/Dockerfile' - - 'concourse/docker/rpmrebuild/rhel' + - 'concourse/docker/rpmrebuild/rocky/Dockerfile' + - 'concourse/docker/rpmrebuild/rocky' waitFor: ['-'] # Push images from Cloud Build to Container Registry images: - 'gcr.io/$PROJECT_ID/rpmrebuild-centos7:latest' - - 'gcr.io/$PROJECT_ID/rpmrebuild-rhel8:latest' - 'gcr.io/$PROJECT_ID/rpmrebuild-rocky8:latest' diff --git a/concourse/docker/rpmrebuild/rhel/Dockerfile b/concourse/docker/rpmrebuild/rhel/Dockerfile deleted file mode 100644 index 54b3392d8b..0000000000 --- a/concourse/docker/rpmrebuild/rhel/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -ARG BASE_IMAGE=gcr.io/data-gpdb-private-images/gpdb6-rhel8-test:latest - -FROM centos:7 as downloader - -# download source RPM for rpmrebuild from EPEL repository for EL7 -RUN yum install -y epel-release && yumdownloader --source rpmrebuild - -FROM ${BASE_IMAGE} - -COPY --from=downloader /rpmrebuild-*.src.rpm / - -# build EL8 RPM for rpmrebuild from the downloaded source RPM -RUN rpmbuild --rebuild rpmrebuild-*.src.rpm \ - && rpm -i /root/rpmbuild/RPMS/noarch/rpmrebuild-*.el8.noarch.rpm \ - && rm rpmrebuild-*.src.rpm \ - && rm -rf /root/rpmbuild diff --git a/concourse/docker/rpmrebuild/rocky/Dockerfile b/concourse/docker/rpmrebuild/rocky/Dockerfile new file mode 100644 index 0000000000..ced3b26c2c --- /dev/null +++ b/concourse/docker/rpmrebuild/rocky/Dockerfile @@ -0,0 +1,6 @@ +ARG BASE_IMAGE=rockylinux:8 + +FROM ${BASE_IMAGE} + +RUN dnf install -y epel-release +RUN dnf install -y rpmrebuild diff --git a/concourse/pipelines/certification_pipeline.yml b/concourse/pipelines/certification_pipeline.yml index 2e3be8ddbb..bb6571a37d 100644 --- a/concourse/pipelines/certification_pipeline.yml +++ b/concourse/pipelines/certification_pipeline.yml @@ -89,11 +89,11 @@ resources: username: _json_key password: ((ud/pxf/secrets/pxf-cloudbuild-service-account-key)) -- name: gpdb6-pxf-dev-rhel8-image +- name: gpdb6-pxf-dev-rocky8-image type: registry-image icon: docker source: - repository: gcr.io/data-gpdb-ud/gpdb-pxf-dev/gpdb6-rhel8-test-pxf + repository: gcr.io/data-gpdb-ud/gpdb-pxf-dev/gpdb6-rocky8-test-pxf tag: latest username: _json_key password: ((ud/pxf/secrets/pxf-cloudbuild-service-account-key)) @@ -266,14 +266,14 @@ jobs: - get: pxf_package resource: pxf_gp6_rpm_rhel8 trigger: true - - get: gpdb6-pxf-dev-rhel8-image + - get: gpdb6-pxf-dev-rocky8-image - get: ccp-7-image - get: pxf-automation-dependencies - get: singlecluster resource: singlecluster-hdp2 - task: Test GPDB-6 with PXF-GP6-HDP2 on RHEL8 file: pxf_src/concourse/tasks/test_certification.yml - image: gpdb6-pxf-dev-rhel8-image + image: gpdb6-pxf-dev-rocky8-image params: ACCESS_KEY_ID: ((tf-machine-access-key-id)) GP_VER: 6 diff --git a/concourse/pipelines/cloudbuild_pipeline.yml b/concourse/pipelines/cloudbuild_pipeline.yml index 22748ad5d6..846433bce9 100644 --- a/concourse/pipelines/cloudbuild_pipeline.yml +++ b/concourse/pipelines/cloudbuild_pipeline.yml @@ -109,7 +109,7 @@ jobs: GOOGLE_CREDENTIALS: ((ud/pxf/secrets/pxf-cloudbuild-service-account-key)) GOOGLE_PROJECT_ID: ((ud/pxf/common/google-project-id)) GOOGLE_ZONE: ((ud/pxf/common/google-zone)) - IMAGE_LIST: "gpdb5-centos7-test-pxf gpdb6-centos7-test-pxf gpdb6-rhel8-test-pxf gpdb6-ubuntu18.04-test-pxf gpdb6-oel7-test-pxf gpdb7-centos7-test-pxf gpdb7-rhel8-test-pxf gpdb7-rocky8-test-pxf gpdb7-ubuntu18.04-test-pxf" + IMAGE_LIST: "gpdb5-centos7-test-pxf gpdb6-centos7-test-pxf gpdb6-rocky8-test-pxf gpdb6-ubuntu18.04-test-pxf gpdb6-oel7-test-pxf gpdb7-rocky8-test-pxf" config: platform: linux inputs: diff --git a/concourse/pipelines/templates/build_pipeline-tpl.yml b/concourse/pipelines/templates/build_pipeline-tpl.yml index 1182e10c72..5a576b4c7b 100644 --- a/concourse/pipelines/templates/build_pipeline-tpl.yml +++ b/concourse/pipelines/templates/build_pipeline-tpl.yml @@ -43,6 +43,7 @@ groups: - Test PXF-GP[[gp_ver]]-MINIO-NO-IMPERS on RHEL7 - Test PXF-GP[[gp_ver]]-HDP2-SECURE-MULTI-IMPERS on RHEL7 - Test PXF-GP[[gp_ver]]-HDP2-SECURE-MULTI-NO-IMPERS on RHEL7 + - Test PXF-GP[[gp_ver]]-HDP2-Upgrade-Extension on RHEL7 {% set gp_ver = None %} - Compatibility Gate for PXF-GP - Promote PXF-GP5 and PXF-GP6 Artifacts @@ -89,6 +90,7 @@ groups: - Test PXF-GP[[gp_ver]]-MINIO-NO-IMPERS on RHEL7 - Test PXF-GP[[gp_ver]]-HDP2-SECURE-MULTI-IMPERS on RHEL7 - Test PXF-GP[[gp_ver]]-HDP2-SECURE-MULTI-NO-IMPERS on RHEL7 + - Test PXF-GP[[gp_ver]]-HDP2-Upgrade-Extension on RHEL7 {% set gp_ver = None %} - name: Backwards Compatibility jobs: @@ -374,15 +376,11 @@ resources: {% endfor %} {# gp5, gp6 #} {% for gp_ver in range(6, 8) %} -- name: gpdb[[gp_ver]]-pxf-dev-rhel8-image +- name: gpdb[[gp_ver]]-pxf-dev-rocky8-image type: registry-image icon: docker source: -{% if gp_ver == 6 %} - repository: gcr.io/data-gpdb-ud/gpdb-pxf-dev/gpdb[[gp_ver]]-rhel8-test-pxf -{% else %} repository: gcr.io/data-gpdb-ud/gpdb-pxf-dev/gpdb[[gp_ver]]-rocky8-test-pxf -{% endif %} tag: latest username: _json_key password: ((ud/pxf/secrets/pxf-cloudbuild-service-account-key)) @@ -427,15 +425,6 @@ resources: username: _json_key password: ((ud/pxf/secrets/pxf-cloudbuild-service-account-key)) -- name: rpmrebuild-rhel8-image - type: registry-image - icon: docker - source: - repository: gcr.io/data-gpdb-ud/rpmrebuild-rhel8 - tag: latest - username: _json_key - password: ((ud/pxf/secrets/pxf-cloudbuild-service-account-key)) - - name: rpmrebuild-rocky8-image type: registry-image icon: docker @@ -471,7 +460,7 @@ resources: url: ((ud/pxf/secrets/ud-pipeline-bot-gp-releng-webhook)) {% endif %} -## ---------- Product Packages ---------- +## ---------- Greenplum Packages ---------- {% set gp_ver = 5 %} {% for i in range(num_gpdb5_versions) %} - name: gpdb[[gp_ver]]_rhel7_rpm_latest-[[i]] @@ -480,7 +469,7 @@ resources: source: bucket: ((ud/pxf/common/pivnet-artifacts-bucket-name)) json_key: ((ud/pxf/secrets/pxf-storage-service-account-key)) - regexp: latest-[[i]]_gpdb[[gp_ver]]/greenplum-db-(.*)-rhel7-x86_64.rpm + regexp: latest-[[i]]_gpdb[[gp_ver]]/greenplum-db-([[gp_ver]].*)-rhel7-x86_64.rpm {% endfor %} {# range(num_gpdb5_versions) #} {% set gp_ver = None %} @@ -492,7 +481,7 @@ resources: source: bucket: ((ud/pxf/common/pivnet-artifacts-bucket-name)) json_key: ((ud/pxf/secrets/pxf-storage-service-account-key)) - regexp: latest-[[i]]_gpdb[[gp_ver]]/greenplum-db-(.*)-rhel7-x86_64.rpm + regexp: latest-[[i]]_gpdb[[gp_ver]]/greenplum-db-([[gp_ver]].*)-rhel7-x86_64.rpm - name: gpdb[[gp_ver]]_ubuntu18_deb_latest-[[i]] type: gcs @@ -500,7 +489,7 @@ resources: source: bucket: ((ud/pxf/common/pivnet-artifacts-bucket-name)) json_key: ((ud/pxf/secrets/pxf-storage-service-account-key)) - regexp: latest-[[i]]_gpdb[[gp_ver]]/greenplum-db-(.*)-ubuntu18.04-amd64.deb + regexp: latest-[[i]]_gpdb[[gp_ver]]/greenplum-db-([[gp_ver]].*)-ubuntu18.04-amd64.deb {% endfor %} {# range(num_gpdb6_versions) #} {% set gp_ver = None %} @@ -512,7 +501,7 @@ resources: source: bucket: ((ud/pxf/common/pivnet-artifacts-bucket-name)) json_key: ((ud/pxf/secrets/pxf-storage-service-account-key)) - regexp: latest-[[i]]_gpdb[[gp_ver]]/greenplum-db-(.*)-rhel8-x86_64.rpm + regexp: latest-[[i]]_gpdb[[gp_ver]]/greenplum-db-([[gp_ver]].*)-rhel8-x86_64.rpm {% endfor %} {# range(num_gpdb6_versions) #} {% set gp_ver = None %} @@ -523,16 +512,25 @@ resources: source: bucket: ((ud/pxf/common/gpdb-concourse-resources-prod-bucket-name)) json_key: ((concourse-gcs-resources-service-account-key)) - regexp: server/published/main/greenplum-db-(7.*)-el8-x86_64.rpm + regexp: server/released/gpdb7/greenplum-db-(7.*)-el8-x86_64.rpm ## ---------- PXF 5 (for GPDB 6) Artifact --------------- - name: pxf5_gp6_rhel7_released type: gcs source: - bucket: ((ud/pxf/[[ environment ]]/releng-drop-bucket-name)) + bucket: ((ud/pxf/prod/releng-drop-bucket-name)) json_key: ((concourse-gcs-resources-service-account-key)) regexp: ((ud/pxf/common/releng-drop-path))/gpdb6/pxf-gp6-5.(.*)-2.el7.x86_64.rpm +## ---------- PXF 6.6 (for GPDB 6) Artifact --------------- +# This is for extension upgrade testing. PXF 6.6.X contains extension version 2.0. PXF 6.7.+ contains version 2.1 +- name: pxf6_6_gp6_rhel7_released + type: gcs + source: + bucket: ((ud/pxf/prod/releng-drop-bucket-name)) + json_key: ((concourse-gcs-resources-service-account-key)) + regexp: ((ud/pxf/common/releng-drop-path))/gpdb6/pxf-gp6-(6.6.*)-2.el7.x86_64.rpm + ## ---------- PXF Build Artifacts ---------- {% set gp_ver = None %} @@ -731,10 +729,10 @@ jobs: trigger: true - get: gpdb_package resource: gpdb[[gp_ver]]_rhel8_rpm_latest-0 - - get: gpdb[[gp_ver]]-pxf-dev-rhel8-image + - get: gpdb[[gp_ver]]-pxf-dev-rocky8-image - get: pxf-build-dependencies - task: Build PXF-GP[[gp_ver]] on RHEL8 - image: gpdb[[gp_ver]]-pxf-dev-rhel8-image + image: gpdb[[gp_ver]]-pxf-dev-rocky8-image file: pxf_src/concourse/tasks/build.yml params: LICENSE: ((ud/pxf/common/rpm-license)) @@ -755,7 +753,7 @@ jobs: - get: gpdb_package resource: gpdb[[gp_ver]]_rhel8_rpm_latest-0 passed: [Build PXF-GP[[gp_ver]] on RHEL8] - - get: gpdb[[gp_ver]]-pxf-dev-rhel8-image + - get: gpdb[[gp_ver]]-pxf-dev-rocky8-image passed: [Build PXF-GP[[gp_ver]] on RHEL8] - get: pxf-automation-dependencies {% if gp_ver == 7 %} @@ -765,7 +763,7 @@ jobs: resource: singlecluster-hdp2 - task: Test PXF-GP[[gp_ver]]-HDP2 on RHEL8 file: pxf_src/concourse/tasks/test.yml - image: gpdb[[gp_ver]]-pxf-dev-rhel8-image + image: gpdb[[gp_ver]]-pxf-dev-rocky8-image params: ACCESS_KEY_ID: ((tf-machine-access-key-id)) GP_VER: [[gp_ver]] @@ -978,6 +976,34 @@ jobs: HADOOP_CLIENT: CDH SECRET_ACCESS_KEY: ((tf-machine-secret-access-key)) +- name: Test PXF-GP[[gp_ver]]-HDP2-Upgrade-Extension on RHEL7 + plan: + - in_parallel: + - get: pxf_src + passed: [Testing Gate for PXF-GP] + trigger: true + - get: pxf_package + resource: pxf6_6_gp6_rhel7_released # for upgrade test + - get: pxf_tarball + resource: pxf_gp[[gp_ver]]_tarball_rhel7 + passed: [Testing Gate for PXF-GP] + - get: gpdb_package + resource: gpdb[[gp_ver]]_rhel7_rpm_latest-0 + passed: [Testing Gate for PXF-GP] + - get: gpdb[[gp_ver]]-pxf-dev-centos7-image + - get: pxf-automation-dependencies + - get: singlecluster + resource: singlecluster-hdp2 + - task: Test PXF-GP[[gp_ver]]-HDP2-New-Extension on RHEL7 + file: pxf_src/concourse/tasks/upgrade_extension.yml + image: gpdb[[gp_ver]]-pxf-dev-centos7-image + params: + ACCESS_KEY_ID: ((tf-machine-access-key-id)) + GP_VER: [[gp_ver]] + GROUP: pxfExtensionVersion2 + SECOND_GROUP: pxfExtensionVersion2_1 + SECRET_ACCESS_KEY: ((tf-machine-secret-access-key)) + ## ---------- FILE tests ----------------- - name: Test PXF-GP[[gp_ver]]-FILE-NO-IMPERS on RHEL7 @@ -1512,6 +1538,7 @@ jobs: - Test PXF-GP[[gp_ver]]-MINIO-NO-IMPERS on RHEL7 - Test PXF-GP[[gp_ver]]-HDP2-SECURE-MULTI-IMPERS on RHEL7 - Test PXF-GP[[gp_ver]]-HDP2-SECURE-MULTI-NO-IMPERS on RHEL7 + - Test PXF-GP[[gp_ver]]-HDP2-Upgrade-Extension on RHEL7 trigger: true {% set gp_ver = None %} {% set gp_ver = 5 %} @@ -1651,7 +1678,7 @@ jobs: - get: pxf_tarball passed: [Compatibility Gate for PXF-GP] resource: pxf_gp[[gp_ver]]_tarball_rhel8 - - get: gpdb[[gp_ver]]-pxf-dev-rhel8-image + - get: gpdb[[gp_ver]]-pxf-dev-rocky8-image - get: singlecluster resource: singlecluster-hdp2 {% for i in range(1, num_gpdb6_versions) %} @@ -1664,7 +1691,7 @@ jobs: steps: {% for i in range(1, num_gpdb6_versions) %} - task: Test Against Greenplum Latest - [[i]] RHEL8 - image: gpdb[[gp_ver]]-pxf-dev-rhel8-image + image: gpdb[[gp_ver]]-pxf-dev-rocky8-image config: platform: linux inputs: @@ -1752,7 +1779,6 @@ jobs: trigger: true - get: google-cloud-sdk-slim-image - get: rpmrebuild-centos7-image - - get: rpmrebuild-rhel8-image - get: rpmrebuild-rocky8-image - get: gpdb6-pxf-dev-ubuntu18-image - task: Get PXF-GP5 and PXF-GP6 Artifacts from Releases Directory @@ -1775,7 +1801,7 @@ jobs: {% endfor %} {% set gp_ver = 6 %} - task: Add OSL file to PXF-GP[[gp_ver]] RPM on RHEL8 - image: rpmrebuild-rhel8-image + image: rpmrebuild-rocky8-image file: pxf_src/concourse/tasks/add_osl_rpm.yml params: GP_VER: [[gp_ver]] diff --git a/concourse/pipelines/templates/dev_build_pipeline-tpl.yml b/concourse/pipelines/templates/dev_build_pipeline-tpl.yml index e8c69b0015..21863fb011 100644 --- a/concourse/pipelines/templates/dev_build_pipeline-tpl.yml +++ b/concourse/pipelines/templates/dev_build_pipeline-tpl.yml @@ -193,15 +193,11 @@ resources: {% endfor %} {# gp5, gp6, and gp7 #} {% for gp_ver in range(6, 8) %} -- name: gpdb[[gp_ver]]-pxf-dev-rhel8-image +- name: gpdb[[gp_ver]]-pxf-dev-rocky8-image type: registry-image icon: docker source: -{% if gp_ver == 6 %} - repository: gcr.io/data-gpdb-ud/gpdb-pxf-dev/gpdb[[gp_ver]]-rhel8-test-pxf -{% else %} repository: gcr.io/data-gpdb-ud/gpdb-pxf-dev/gpdb[[gp_ver]]-rocky8-test-pxf -{% endif %} tag: latest username: _json_key password: ((ud/pxf/secrets/pxf-cloudbuild-service-account-key)) @@ -257,7 +253,7 @@ resources: url: ((ud-pipeline-bot-[[user]]-webhook)) {% endif %} -## ---------- Product Packages ---------- +## ---------- Greenplum Packages ---------- {% set gp_ver = 5 %} - name: gpdb[[gp_ver]]_rhel7_rpm_latest-0 type: gcs @@ -265,7 +261,7 @@ resources: source: bucket: ((ud/pxf/common/pivnet-artifacts-bucket-name)) json_key: ((ud/pxf/secrets/pxf-storage-service-account-key)) - regexp: latest-0_gpdb[[gp_ver]]/greenplum-db-(.*)-rhel7-x86_64.rpm + regexp: latest-0_gpdb[[gp_ver]]/greenplum-db-([[gp_ver]].*)-rhel7-x86_64.rpm {% set gp_ver = None %} {% set gp_ver = 6 %} @@ -275,7 +271,7 @@ resources: source: bucket: ((ud/pxf/common/pivnet-artifacts-bucket-name)) json_key: ((ud/pxf/secrets/pxf-storage-service-account-key)) - regexp: latest-0_gpdb[[gp_ver]]/greenplum-db-(.*)-rhel7-x86_64.rpm + regexp: latest-0_gpdb[[gp_ver]]/greenplum-db-([[gp_ver]].*)-rhel7-x86_64.rpm - name: gpdb[[gp_ver]]_ubuntu18_deb_latest-0 type: gcs @@ -283,15 +279,15 @@ resources: source: bucket: ((ud/pxf/common/pivnet-artifacts-bucket-name)) json_key: ((ud/pxf/secrets/pxf-storage-service-account-key)) - regexp: latest-0_gpdb[[gp_ver]]/greenplum-db-(.*)-ubuntu18.04-amd64.deb + regexp: latest-0_gpdb[[gp_ver]]/greenplum-db-([[gp_ver]].*)-ubuntu18.04-amd64.deb - name: gpdb[[gp_ver]]_rhel8_rpm_latest-0 type: gcs icon: google-drive source: - bucket: ((ud/pxf/common/gpdb-concourse-resources-prod-bucket-name)) - json_key: ((concourse-gcs-resources-service-account-key)) - regexp: server/published/gpdb6/greenplum-db-(6.*)-rhel8-x86_64.rpm + bucket: ((ud/pxf/common/pivnet-artifacts-bucket-name)) + json_key: ((ud/pxf/secrets/pxf-storage-service-account-key)) + regexp: latest-0_gpdb[[gp_ver]]/greenplum-db-([[gp_ver]].*)-rhel8-x86_64.rpm {% set gp_ver = None %} ## ---------- Greenplum 7 Beta Builds ---------- @@ -301,7 +297,7 @@ resources: source: bucket: ((ud/pxf/common/gpdb-concourse-resources-prod-bucket-name)) json_key: ((concourse-gcs-resources-service-account-key)) - regexp: server/published/main/greenplum-db-(7.*)-el8-x86_64.rpm + regexp: server/released/gpdb7/greenplum-db-(7.*)-el8-x86_64.rpm ## ---------- PXF 5 Artifact --------------- {% if multinode %} @@ -535,10 +531,10 @@ jobs: trigger: true - get: gpdb_package resource: gpdb[[gp_ver]]_rhel8_rpm_latest-0 - - get: gpdb[[gp_ver]]-pxf-dev-rhel8-image + - get: gpdb[[gp_ver]]-pxf-dev-rocky8-image - get: pxf-build-dependencies - task: Build PXF-GP[[gp_ver]] on RHEL8 - image: gpdb[[gp_ver]]-pxf-dev-rhel8-image + image: gpdb[[gp_ver]]-pxf-dev-rocky8-image file: pxf_src/concourse/tasks/build.yml params: LICENSE: ((ud/pxf/common/rpm-license)) @@ -564,7 +560,7 @@ jobs: - get: gpdb_package resource: gpdb[[gp_ver]]_rhel8_rpm_latest-0 passed: [Build PXF-GP[[gp_ver]] on RHEL8] - - get: gpdb[[gp_ver]]-pxf-dev-rhel8-image + - get: gpdb[[gp_ver]]-pxf-dev-rocky8-image passed: [Build PXF-GP[[gp_ver]] on RHEL8] - get: pxf-automation-dependencies {% if gp_ver == 7 %} @@ -574,7 +570,7 @@ jobs: resource: singlecluster-hdp2 - task: Test PXF-GP[[gp_ver]]-HDP2 on RHEL8 file: pxf_src/concourse/tasks/test.yml - image: gpdb[[gp_ver]]-pxf-dev-rhel8-image + image: gpdb[[gp_ver]]-pxf-dev-rocky8-image params: ACCESS_KEY_ID: ((tf-machine-access-key-id)) GP_VER: [[gp_ver]] @@ -605,14 +601,14 @@ jobs: {% if gp_ver == 7 %} - get: gp6-python-libs {% endif %} - - get: gpdb[[gp_ver]]-pxf-dev-rhel8-image + - get: gpdb[[gp_ver]]-pxf-dev-rocky8-image passed: [Build PXF-GP[[gp_ver]] on RHEL8] - get: pxf-automation-dependencies - get: singlecluster resource: singlecluster-hdp2 - task: Test PXF-FDW-GP[[gp_ver]]-HDP2 on RHEL8 file: pxf_src/concourse/tasks/test.yml - image: gpdb[[gp_ver]]-pxf-dev-rhel8-image + image: gpdb[[gp_ver]]-pxf-dev-rocky8-image params: ACCESS_KEY_ID: ((tf-machine-access-key-id)) GP_VER: [[gp_ver]] diff --git a/concourse/pipelines/templates/perf_pipeline-tpl.yml b/concourse/pipelines/templates/perf_pipeline-tpl.yml index e3045ff531..c26ced08de 100644 --- a/concourse/pipelines/templates/perf_pipeline-tpl.yml +++ b/concourse/pipelines/templates/perf_pipeline-tpl.yml @@ -127,7 +127,7 @@ resources: type: registry-image icon: docker source: - repository: gcr.io/data-gpdb-ud/gpdb-pxf-dev/gpdb6-rhel8-test-pxf + repository: gcr.io/data-gpdb-ud/gpdb-pxf-dev/gpdb6-rocky8-test-pxf tag: latest username: _json_key password: ((ud/pxf/secrets/pxf-cloudbuild-service-account-key)) diff --git a/concourse/pipelines/templates/pr_pipeline-tpl.yml b/concourse/pipelines/templates/pr_pipeline-tpl.yml index ce0e69389f..2319d2fd36 100644 --- a/concourse/pipelines/templates/pr_pipeline-tpl.yml +++ b/concourse/pipelines/templates/pr_pipeline-tpl.yml @@ -58,15 +58,11 @@ resources: {% endfor %} {# gp5, gp6 #} {% for gp_ver in range(6, 8) %} -- name: gpdb[[gp_ver]]-pxf-dev-rhel8-image +- name: gpdb[[gp_ver]]-pxf-dev-rocky8-image type: registry-image icon: docker source: -{% if gp_ver == 6 %} - repository: gcr.io/data-gpdb-ud/gpdb-pxf-dev/gpdb[[gp_ver]]-rhel8-test-pxf -{% else %} repository: gcr.io/data-gpdb-ud/gpdb-pxf-dev/gpdb[[gp_ver]]-rocky8-test-pxf -{% endif %} tag: latest username: _json_key password: ((ud/pxf/secrets/pxf-cloudbuild-service-account-key)) @@ -83,7 +79,7 @@ resources: password: ((ud/pxf/secrets/pxf-cloudbuild-service-account-key)) {% set gp_ver = None %} -## ---------- Product Packages ---------- +## ---------- Greenplum Packages ---------- {% set gp_ver = 5 %} {% for i in range(num_gpdb5_versions) %} - name: gpdb[[gp_ver]]_rhel7_rpm_latest-[[i]] @@ -92,7 +88,7 @@ resources: source: bucket: ((ud/pxf/common/pivnet-artifacts-bucket-name)) json_key: ((ud/pxf/secrets/pxf-storage-service-account-key)) - regexp: latest-[[i]]_gpdb[[gp_ver]]/greenplum-db-(.*)-rhel7-x86_64.rpm + regexp: latest-[[i]]_gpdb[[gp_ver]]/greenplum-db-([[gp_ver]].*)-rhel7-x86_64.rpm {% endfor %} {# range(num_gpdb5_versions) #} {% set gp_ver = None %} @@ -104,7 +100,7 @@ resources: source: bucket: ((ud/pxf/common/pivnet-artifacts-bucket-name)) json_key: ((ud/pxf/secrets/pxf-storage-service-account-key)) - regexp: latest-[[i]]_gpdb[[gp_ver]]/greenplum-db-(.*)-rhel7-x86_64.rpm + regexp: latest-[[i]]_gpdb[[gp_ver]]/greenplum-db-([[gp_ver]].*)-rhel7-x86_64.rpm - name: gpdb[[gp_ver]]_rhel8_rpm_latest-[[i]] type: gcs @@ -112,7 +108,7 @@ resources: source: bucket: ((ud/pxf/common/pivnet-artifacts-bucket-name)) json_key: ((ud/pxf/secrets/pxf-storage-service-account-key)) - regexp: latest-[[i]]_gpdb[[gp_ver]]/greenplum-db-(.*)-rhel8-x86_64.rpm + regexp: latest-[[i]]_gpdb[[gp_ver]]/greenplum-db-([[gp_ver]].*)-rhel8-x86_64.rpm - name: gpdb[[gp_ver]]_ubuntu18_deb_latest-[[i]] type: gcs @@ -120,7 +116,7 @@ resources: source: bucket: ((ud/pxf/common/pivnet-artifacts-bucket-name)) json_key: ((ud/pxf/secrets/pxf-storage-service-account-key)) - regexp: latest-[[i]]_gpdb[[gp_ver]]/greenplum-db-(.*)-ubuntu18.04-amd64.deb + regexp: latest-[[i]]_gpdb[[gp_ver]]/greenplum-db-([[gp_ver]].*)-ubuntu18.04-amd64.deb {% endfor %} {# range(num_gpdb6_versions) #} {% set gp_ver = None %} @@ -131,7 +127,7 @@ resources: source: bucket: ((ud/pxf/common/gpdb-concourse-resources-prod-bucket-name)) json_key: ((concourse-gcs-resources-service-account-key)) - regexp: server/published/main/greenplum-db-(7.*)-el8-x86_64.rpm + regexp: server/released/gpdb7/greenplum-db-(7.*)-el8-x86_64.rpm ## ====================================================================== ## JOBS @@ -215,18 +211,18 @@ jobs: trigger: true - get: gpdb_package resource: gpdb[[gp_ver]]_rhel8_rpm_latest-0 - - get: gpdb[[gp_ver]]-pxf-dev-rhel8-image + - get: gpdb[[gp_ver]]-pxf-dev-rocky8-image - get: pxf-build-dependencies - get: singlecluster resource: singlecluster-hdp2 - task: Build PXF-GP[[gp_ver]] on RHEL8 - image: gpdb[[gp_ver]]-pxf-dev-rhel8-image + image: gpdb[[gp_ver]]-pxf-dev-rocky8-image file: pxf_src/concourse/tasks/build.yml params: LICENSE: ((ud/pxf/common/rpm-license)) VENDOR: ((ud/pxf/common/rpm-vendor)) - task: Test Against Greenplum Latest RHEL8 - image: gpdb[[gp_ver]]-pxf-dev-rhel8-image + image: gpdb[[gp_ver]]-pxf-dev-rocky8-image config: platform: linux inputs: diff --git a/concourse/scripts/cli/test_reset_init.sh b/concourse/scripts/cli/test_reset_init.sh index dd9dfcd8f6..1ba0531999 100755 --- a/concourse/scripts/cli/test_reset_init.sh +++ b/concourse/scripts/cli/test_reset_init.sh @@ -94,7 +94,7 @@ PXF initialized successfully on ${num_hosts} out of ${num_hosts} hosts" control_file_content=\ "directory = '/usr/local/pxf-gp6/gpextable/' -default_version = '2.0' +default_version = '2.1' comment = 'Extension which allows to access unmanaged data' module_pathname = '/usr/local/pxf-gp6/gpextable/pxf' superuser = true diff --git a/concourse/scripts/pxf_common.bash b/concourse/scripts/pxf_common.bash index 7764bf5f4d..1db48a4624 100755 --- a/concourse/scripts/pxf_common.bash +++ b/concourse/scripts/pxf_common.bash @@ -84,6 +84,66 @@ function set_env() { export TIMEFORMAT=$'\e[4;33mIt took %R seconds to complete this step\e[0m'; } +function run_pxf_automation() { + # Let's make sure that automation/singlecluster directories are writeable + chmod a+w pxf_src/automation /singlecluster || true + find pxf_src/automation/tinc* -type d -exec chmod a+w {} \; + + local extension_name="pxf" + if [[ ${USE_FDW} == "true" ]]; then + extension_name="pxf_fdw" + fi + + #TODO: remove once exttable tests with GP7 are set + if [[ ${GROUP} == fdw_gpdb_schedule ]]; then + extension_name="pxf_fdw" + fi + + su gpadmin -c " + source '${GPHOME}/greenplum_path.sh' && + psql -p ${PGPORT} -d template1 -c 'CREATE EXTENSION IF NOT EXISTS ${extension_name}' + " + # prepare certification output directory + mkdir -p certification + chmod a+w certification + + cat > ~gpadmin/run_pxf_automation_test.sh <<-EOF + #!/usr/bin/env bash + set -exo pipefail + + source ~gpadmin/.pxfrc + + export PATH=\$PATH:${GPHD_ROOT}/bin + export GPHD_ROOT=${GPHD_ROOT} + export PXF_HOME=${PXF_HOME} + export PGPORT=${PGPORT} + export USE_FDW=${USE_FDW} + + cd pxf_src/automation + time make GROUP=${GROUP} test + + # if the test is successful, create certification file + gpdb_build_from_sql=\$(psql -c 'select version()' | grep Greenplum | cut -d ' ' -f 6,8) + gpdb_build_clean=\${gpdb_build_from_sql%)} + pxf_version=\$(< ${PXF_HOME}/version) + echo "GPDB-\${gpdb_build_clean/ commit:/-}-PXF-\${pxf_version}" > "${PWD}/certification/certification.txt" + echo + echo '****************************************************************************************************' + echo "Wrote certification : \$(< ${PWD}/certification/certification.txt)" + echo '****************************************************************************************************' + EOF + + chown gpadmin:gpadmin ~gpadmin/run_pxf_automation_test.sh + chmod a+x ~gpadmin/run_pxf_automation_test.sh + + if [[ ${ACCEPTANCE} == true ]]; then + echo 'Acceptance test pipeline' + exit 1 + fi + + su gpadmin -c ~gpadmin/run_pxf_automation_test.sh +} + function run_regression_test() { ln -s "${PWD}/gpdb_src" ~gpadmin/gpdb_src cat > ~gpadmin/run_regression_test.sh <<-EOF @@ -392,6 +452,19 @@ function setup_impersonation() { fi } +function setup_hadoop() { + local hdfsrepo=$1 + + [[ -z ${GROUP} ]] && return 0 + + export SLAVES=1 + setup_impersonation "${hdfsrepo}" + if grep 'hadoop-3' "${hdfsrepo}/versions.txt"; then + adjust_for_hadoop3 "${hdfsrepo}" + fi + start_hadoop_services "${hdfsrepo}" +} + function adjust_for_hadoop3() { local GPHD_ROOT=${1} diff --git a/concourse/scripts/test.bash b/concourse/scripts/test.bash index 15856aadce..227a893298 100755 --- a/concourse/scripts/test.bash +++ b/concourse/scripts/test.bash @@ -70,65 +70,6 @@ function run_pg_regress() { su gpadmin -c ~gpadmin/run_pxf_automation_test.sh } -function run_pxf_automation() { - # Let's make sure that automation/singlecluster directories are writeable - chmod a+w pxf_src/automation /singlecluster || true - find pxf_src/automation/tinc* -type d -exec chmod a+w {} \; - - local extension_name="pxf" - if [[ ${USE_FDW} == "true" ]]; then - extension_name="pxf_fdw" - fi - - #TODO: remove once exttable tests with GP7 are set - if [[ ${GROUP} == fdw_gpdb_schedule ]]; then - extension_name="pxf_fdw" - fi - - su gpadmin -c " - source '${GPHOME}/greenplum_path.sh' && - psql -p ${PGPORT} -d template1 -c 'CREATE EXTENSION ${extension_name}' - " - # prepare certification output directory - mkdir -p certification - chmod a+w certification - - cat > ~gpadmin/run_pxf_automation_test.sh <<-EOF - #!/usr/bin/env bash - set -exo pipefail - - source ~gpadmin/.pxfrc - - export PATH=\$PATH:${GPHD_ROOT}/bin - export GPHD_ROOT=${GPHD_ROOT} - export PXF_HOME=${PXF_HOME} - export PGPORT=${PGPORT} - export USE_FDW=${USE_FDW} - - cd pxf_src/automation - time make GROUP=${GROUP} test - - # if the test is successful, create certification file - gpdb_build_from_sql=\$(psql -c 'select version()' | grep Greenplum | cut -d ' ' -f 6,8) - gpdb_build_clean=\${gpdb_build_from_sql%)} - pxf_version=\$(< ${PXF_HOME}/version) - echo "GPDB-\${gpdb_build_clean/ commit:/-}-PXF-\${pxf_version}" > "${PWD}/certification/certification.txt" - echo - echo '****************************************************************************************************' - echo "Wrote certification : \$(< ${PWD}/certification/certification.txt)" - echo '****************************************************************************************************' - EOF - - chown gpadmin:gpadmin ~gpadmin/run_pxf_automation_test.sh - chmod a+x ~gpadmin/run_pxf_automation_test.sh - - if [[ ${ACCEPTANCE} == true ]]; then - echo 'Acceptance test pipeline' - exit 1 - fi - - su gpadmin -c ~gpadmin/run_pxf_automation_test.sh -} function generate_extras_fat_jar() { mkdir -p /tmp/fatjar @@ -248,7 +189,7 @@ function _main() { fi # Certification jobs might install non-latest PXF, make sure automation code corresponds to what is installed - if [[ -f ${PXF_HOME}/commit.sha ]]; then + if [[ -f ${PXF_HOME}/commit.sha ]] && [[ ${ADJUST_AUTOMATION} != false ]]; then adjust_automation_code else echo "WARNING: no commit.sha file is found in PXF_HOME=${PXF_HOME}" diff --git a/concourse/scripts/test_pxf.bash b/concourse/scripts/test_pxf.bash index 518792a60c..1ed2edbe3f 100755 --- a/concourse/scripts/test_pxf.bash +++ b/concourse/scripts/test_pxf.bash @@ -125,19 +125,6 @@ function configure_mapr_dependencies() { sed -i 's|8020|7222|' pxf_src/automation/src/test/resources/sut/default.xml } -function setup_hadoop() { - local hdfsrepo=$1 - - [[ -z ${GROUP} ]] && return 0 - - export SLAVES=1 - setup_impersonation "${hdfsrepo}" - if grep 'hadoop-3' "${hdfsrepo}/versions.txt"; then - adjust_for_hadoop3 "${hdfsrepo}" - fi - start_hadoop_services "${hdfsrepo}" -} - function configure_sut() { AMBARI_DIR=$(find /tmp/build/ -name ambari_env_files) if [[ -n $AMBARI_DIR ]]; then diff --git a/concourse/scripts/test_upgrade_extension.bash b/concourse/scripts/test_upgrade_extension.bash new file mode 100755 index 0000000000..f94b733651 --- /dev/null +++ b/concourse/scripts/test_upgrade_extension.bash @@ -0,0 +1,96 @@ +#!/bin/bash + +set -exo pipefail + +CWDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# make sure GP_VER is set so that we know what PXF_HOME will be +: "${GP_VER:?GP_VER must be set}" +# We run 2 sets of automation tests, once before upgrading and once after +# make sure that SECOND_GROUP is set so that we actually have a set of tests +# to run against after we upgrade PXF. +: "${SECOND_GROUP:?SECOND_GROUP must be set}" + +# set our own GPHOME for RPM-based installs before sourcing common script +export GPHOME=/usr/local/greenplum-db +export PXF_HOME=/usr/local/pxf-gp${GP_VER} +export PXF_BASE_DIR=${PXF_BASE_DIR:-$PXF_HOME} + +source "${CWDIR}/pxf_common.bash" + +export GOOGLE_PROJECT_ID=${GOOGLE_PROJECT_ID:-data-gpdb-ud} +export JAVA_TOOL_OPTIONS=-Dfile.encoding=UTF8 +export HADOOP_HEAPSIZE=512 +export YARN_HEAPSIZE=512 +export GPHD_ROOT=/singlecluster +export PGPORT=${PGPORT:-5432} + +function upgrade_pxf() { + existing_pxf_version=$(cat "${PXF_HOME}"/version) + echo "Stopping PXF ${existing_pxf_version}" + su gpadmin -c "${PXF_HOME}/bin/pxf version && ${PXF_HOME}/bin/pxf cluster stop" + + echo "Installing Newer Version of PXF 6" + install_pxf_tarball + + echo "Check the PXF 6 version" + su gpadmin -c "${PXF_HOME}/bin/pxf version" + + echo "Register the PXF extension into Greenplum" + su gpadmin -c "GPHOME=${GPHOME} ${PXF_HOME}/bin/pxf cluster register" + + if [[ "${PXF_BASE_DIR}" != "${PXF_HOME}" ]]; then + echo "Prepare PXF in ${PXF_BASE_DIR}" + PXF_BASE="${PXF_BASE_DIR}" "${PXF_HOME}"/bin/pxf cluster prepare + echo "export PXF_BASE=${PXF_BASE_DIR}" >> ~gpadmin/.bashrc + fi + updated_pxf_version=$(cat "${PXF_HOME}"/version) + + echo "Starting PXF ${updated_pxf_version}" + + if [[ "${existing_pxf_version}" > "${updated_pxf_version}" ]]; then + echo "Existing version of PXF (${existing_pxf_version}) is greater than or equal to the new version (${updated_pxf_version})" + fi + + su gpadmin -c "PXF_BASE=${PXF_BASE_DIR} ${PXF_HOME}/bin/pxf cluster start" + + # the new version of PXF brought in a new version of the extension. For databases that already had PXF installed, + # we need to explicitly upgrade the PXF extension to the new version + echo "ALTER EXTENSION pxf UPDATE - for multibyte delimiter tests" + + su gpadmin <<'EOSU' + source ${GPHOME}/greenplum_path.sh && + psql --no-align --tuples-only --command "SELECT datname FROM pg_catalog.pg_database WHERE datname != 'template0';" | while read -r dbname; do + echo -n "checking if database '${dbname}' has PXF extension installed... " + if ! psql --dbname="${dbname}" --no-align --tuples-only --command "SELECT extname FROM pg_catalog.pg_extension WHERE extname = 'pxf'" | grep . &>/dev/null; then + echo "skipping database '${dbname}'" + continue + fi + echo "updating PXF extension in database '${dbname}'" + psql --dbname="${dbname}" --set ON_ERROR_STOP=on --command "ALTER EXTENSION pxf UPDATE;" + done +EOSU + +} + +function _main() { + + # Upgrade to latest PXF + echo + echo + echo '****************************************************************************************************' + echo "* Upgrading PXF *" + echo '****************************************************************************************************' + echo + echo + + # Upgrade from older version of PXF to newer version of PXF present in the tarball + upgrade_pxf + + # Run tests after upgrading PXF + # second time running automation so we should be running the second group + GROUP=${SECOND_GROUP} + run_pxf_automation +} + +_main diff --git a/concourse/tasks/test.yml b/concourse/tasks/test.yml index af443573a1..b9d5c8ba5b 100644 --- a/concourse/tasks/test.yml +++ b/concourse/tasks/test.yml @@ -34,6 +34,7 @@ params: PROTOCOL: PG_REGRESS: USE_FDW: false + ADJUST_AUTOMATION: true run: path: pxf_src/concourse/scripts/test.bash diff --git a/concourse/tasks/test_certification.yml b/concourse/tasks/test_certification.yml index a6320d97fd..41c285ca35 100644 --- a/concourse/tasks/test_certification.yml +++ b/concourse/tasks/test_certification.yml @@ -26,6 +26,7 @@ params: RUN_JDK_VERSION: 8 PROTOCOL: PG_REGRESS: + ADJUST_AUTOMATION: true run: path: pxf_src/concourse/scripts/test.bash diff --git a/concourse/tasks/upgrade_extension.yml b/concourse/tasks/upgrade_extension.yml new file mode 100644 index 0000000000..97ede79fe8 --- /dev/null +++ b/concourse/tasks/upgrade_extension.yml @@ -0,0 +1,34 @@ +platform: linux + +image_resource: + type: registry-image + +inputs: + - name: pxf_package + - name: pxf_tarball + - name: pxf_src + - name: gpdb_package + - name: singlecluster + optional: true + - name: pxf-automation-dependencies + optional: true + +params: + GP_VER: + GROUP: pxfExtensionVersion2 + SECOND_GROUP: pxfExtensionVersion2_1 + HADOOP_CLIENT: HDP + IMPERSONATION: true + ADJUST_AUTOMATION: false + PGPORT: 5432 + PXF_BASE_DIR: + PROTOCOL: + USE_FDW: false + +run: + path: sh + args: + - -exc + - | + pxf_src/concourse/scripts/test.bash || exit 1 + pxf_src/concourse/scripts/test_upgrade_extension.bash diff --git a/docs/content/about_pxf_dir.html.md.erb b/docs/content/about_pxf_dir.html.md.erb index d7c5b2f237..b1f5f0df49 100644 --- a/docs/content/about_pxf_dir.html.md.erb +++ b/docs/content/about_pxf_dir.html.md.erb @@ -22,7 +22,7 @@ The following PXF files and directories are installed to `` whe | application/ | The PXF Server application JAR file. | | bin/ | The PXF command line executable directory. | | commit.sha | The commit identifier for this PXF release. | -| gpextable/ | The PXF extension files. PXF copies the `pxf.control` file from this directory to the Greenplum installation (`$GPHOME`) on a single host when you run the `pxf register` command, or on all hosts in the cluster when you run the `pxf [cluster] register` command from the Greenplum master host. | +| gpextable/ | The PXF extension files. PXF copies the `pxf.control` file from this directory to the Greenplum installation (`$GPHOME`) on a single host when you run the `pxf register` command, or on all hosts in the cluster when you run the `pxf [cluster] register` command from the Greenplum coordinator host. | | share/ | The directory for shared PXF files that you may require depending on the external data stores that you access. `share/` initially includes only the PXF HBase JAR file. | | templates/ | The PXF directory for server configuration file templates. | | version | The PXF version. | @@ -49,13 +49,13 @@ PXF provides the [pxf [cluster] prepare](ref/pxf-cluster.html) command to prepar For example, to relocate `$PXF_BASE` to the `/path/to/dir` directory on all Greenplum hosts, run the command as follows: ``` shell -gpadmin@gpmaster$ PXF_BASE=/path/to/dir pxf cluster prepare +gpadmin@coordinator$ PXF_BASE=/path/to/dir pxf cluster prepare ``` When your `$PXF_BASE` is different than ``, inform PXF by setting the `PXF_BASE` environment variable when you run a `pxf` command: ``` pre -gpadmin@gpmaster$ PXF_BASE=/path/to/dir pxf cluster start +gpadmin@coordinator$ PXF_BASE=/path/to/dir pxf cluster start ``` Set the environment variable in the `.bashrc` shell initialization script for the PXF installation owner (typically the `gpadmin` user) as follows: diff --git a/docs/content/access_hdfs.html.md.erb b/docs/content/access_hdfs.html.md.erb index eeeeaf13df..d4199a4f6d 100644 --- a/docs/content/access_hdfs.html.md.erb +++ b/docs/content/access_hdfs.html.md.erb @@ -25,7 +25,7 @@ PXF is compatible with Cloudera, Hortonworks Data Platform, MapR, and generic Ap ## Architecture -HDFS is the primary distributed storage mechanism used by Apache Hadoop. When a user or application performs a query on a PXF external table that references an HDFS file, the Greenplum Database master host dispatches the query to all segment instances. Each segment instance contacts the PXF Service running on its host. When it receives the request from a segment instance, the PXF Service: +HDFS is the primary distributed storage mechanism used by Apache Hadoop. When a user or application performs a query on a PXF external table that references an HDFS file, the Greenplum Database coordinator host dispatches the query to all segment instances. Each segment instance contacts the PXF Service running on its host. When it receives the request from a segment instance, the PXF Service: 1. Allocates a worker thread to serve the request from the segment instance. 2. Invokes the HDFS Java API to request metadata information for the HDFS file from the HDFS NameNode. @@ -36,7 +36,7 @@ HDFS is the primary distributed storage mechanism used by Apache Hadoop. When a A PXF worker thread works on behalf of a segment instance. A worker thread uses its Greenplum Database `gp_segment_id` and the file block information described in the metadata to assign itself a specific portion of the query data. This data may reside on one or more HDFS DataNodes. -The PXF worker thread invokes the HDFS Java API to read the data and delivers it to the segment instance. The segment instance delivers its portion of the data to the Greenplum Database master host. This communication occurs across segment hosts and segment instances in parallel. +The PXF worker thread invokes the HDFS Java API to read the data and delivers it to the segment instance. The segment instance delivers its portion of the data to the Greenplum Database coordinator host. This communication occurs across segment hosts and segment instances in parallel. ## Prerequisites @@ -107,10 +107,11 @@ The PXF Hadoop connectors expose the following profiles to read, and in many cas |-------------|------|---------|-----|-----| | HDFS | delimited single line [text](hdfs_text.html#profile_text) | hdfs:text | n/a | Read, Write | | HDFS | delimited single line comma-separated values of [text](hdfs_text.html#profile_text) | hdfs:csv | n/a | Read, Write | +| HDFS | multi-byte or multi-character delimited single line [csv](hdfs_text.html#multibyte_delim) | hdfs:csv | n/a | Read | | HDFS | fixed width single line [text](hdfs_fixedwidth.html) | hdfs:fixedwidth | n/a | Read, Write | | HDFS | delimited [text with quoted linefeeds](hdfs_text.html#profile_textmulti) | hdfs:text:multi | n/a | Read | | HDFS | [Avro](hdfs_avro.html) | hdfs:avro | n/a | Read, Write | -| HDFS | [JSON](hdfs_json.html) | hdfs:json | n/a | Read | +| HDFS | [JSON](hdfs_json.html) | hdfs:json | n/a | Read, Write | | HDFS | [ORC](hdfs_orc.html) | hdfs:orc | n/a | Read, Write | | HDFS | [Parquet](hdfs_parquet.html) | hdfs:parquet | n/a | Read, Write | | HDFS | AvroSequenceFile | hdfs:AvroSequenceFile | n/a | Read, Write | diff --git a/docs/content/access_objstore.html.md.erb b/docs/content/access_objstore.html.md.erb index b38a9155a0..3053ed09f0 100644 --- a/docs/content/access_objstore.html.md.erb +++ b/docs/content/access_objstore.html.md.erb @@ -51,10 +51,11 @@ The PXF connectors to Azure expose the following profiles to read, and in many c |-----|------|---------| ---------| | delimited single line [plain text](objstore_text.html) | wasbs:text | adl:text | Read, Write | | delimited single line comma-separated values of [plain text](objstore_text.html) | wasbs:csv | adl:csv | Read, Write | +| multi-byte or multi-character delimited single line [csv](objstore_text.html#multibyte_delim) | wasbs:csv | adl:csv | Read | | delimited [text with quoted linefeeds](objstore_text.html) | wasbs:text:multi | adl:text:multi | Read | | fixed width single line [text](objstore_fixedwidth.html) | wasbs:fixedwidth | adl:fixedwidth | Read, Write | | [Avro](objstore_avro.html) | wasbs:avro | adl:avro | Read, Write | -| [JSON](objstore_json.html) | wasbs:json | adl:json | Read | +| [JSON](objstore_json.html) | wasbs:json | adl:json | Read, Write | | [ORC](objstore_orc.html) | wasbs:orc | adl:orc | Read, Write | | [Parquet](objstore_parquet.html) | wasbs:parquet | adl:parquet | Read, Write | | AvroSequenceFile | wasbs:AvroSequenceFile | adl:AvroSequenceFile | Read, Write | @@ -66,6 +67,7 @@ Similarly, the PXF connectors to Google Cloud Storage, and S3-compatible object |-----|------|---------| ---------| | delimited single line [plain text](objstore_text.html) | gs:text | s3:text | Read, Write | | delimited single line comma-separated values of [plain text](objstore_text.html) | gs:csv | s3:csv | Read, Write | +| multi-byte or multi-character delimited single line comma-separated values [csv](objstore_text.html#multibyte_delim) | gs:csv | s3:csv | Read | | delimited [text with quoted linefeeds](objstore_text.html) | gs:text:multi | s3:text:multi | Read | | fixed width single line [text](objstore_fixedwidth.html) | gs:fixedwidth | s3:fixedwidth | Read, Write | | [Avro](objstore_avro.html) | gs:avro | s3:avro | Read, Write | diff --git a/docs/content/cfg_logging.html.md.erb b/docs/content/cfg_logging.html.md.erb index e5583ce3f5..2878182f3a 100644 --- a/docs/content/cfg_logging.html.md.erb +++ b/docs/content/cfg_logging.html.md.erb @@ -32,10 +32,10 @@ To change the PXF log directory, you must update the `$PXF_LOGDIR` property in t **Note:** The new log directory must exist on all Greenplum Database hosts, and must be accessible by the `gpadmin` user. -1. Log in to your Greenplum Database master host: +1. Log in to your Greenplum Database coordinator host: ``` shell - $ ssh gpadmin@ + $ ssh gpadmin@ ``` 1. Use a text editor to uncomment the `export PXF_LOGDIR` line in `$PXF_BASE/conf/pxf-env.sh`, and replace the value with the new PXF log directory. For example: @@ -48,7 +48,7 @@ To change the PXF log directory, you must update the `$PXF_LOGDIR` property in t 2. Use the `pxf cluster sync` command to copy the updated `pxf-env.sh` file to all hosts in the Greenplum Database cluster: ``` shell - gpadmin@gpmaster$ pxf cluster sync + gpadmin@coordinator$ pxf cluster sync ``` 3. Restart PXF on each Greenplum Database host as described in [Restarting PXF](cfginitstart_pxf.html#restart_pxf). @@ -119,10 +119,10 @@ You can change the log level for the PXF Service running on a specific Greenplum To change the log level for the PXF service running on every host in the Greenplum Database cluster: -1. Log in to the Greenplum Database master host: +1. Log in to the Greenplum Database coordinator host: ``` shell - $ ssh gpadmin@ + $ ssh gpadmin@ ``` 1. Use a text editor to uncomment the following line in the `$PXF_BASE/conf/pxf-application.properties` file and set the desired log level. For example, to change the log level from `info` (the default) to `debug`: @@ -134,13 +134,13 @@ To change the log level for the PXF service running on every host in the Greenpl 1. Use the `pxf cluster sync` command to copy the updated `pxf-application.properties` file to all hosts in the Greenplum Database cluster. For example: ``` shell - gpadmin@gpmaster$ pxf cluster sync + gpadmin@coordinator$ pxf cluster sync ``` 1. Restart PXF on each Greenplum Database host: ``` shell - gpadmin@gpmaster$ pxf cluster restart + gpadmin@coordinator$ pxf cluster restart ``` 1. Perform operations that exercise the PXF Service, and then collect and examine the information in `$PXF_LOGDIR/pxf-service.log`. diff --git a/docs/content/cfg_mem.html.md.erb b/docs/content/cfg_mem.html.md.erb index fc66de70eb..d892db46db 100644 --- a/docs/content/cfg_mem.html.md.erb +++ b/docs/content/cfg_mem.html.md.erb @@ -11,16 +11,16 @@ Each PXF Service running on a Greenplum Database host is configured with a defau Perform the following procedure to increase the heap size for the PXF Service running on each host in your Greenplum Database cluster. -1. Log in to your Greenplum Database master host: +1. Log in to your Greenplum Database coordinator host: ``` shell - $ ssh gpadmin@ + $ ssh gpadmin@ ``` 2. Edit the `$PXF_BASE/conf/pxf-env.sh` file. For example: ``` shell - gpadmin@gpmaster$ vi $PXF_BASE/conf/pxf-env.sh + gpadmin@coordinator$ vi $PXF_BASE/conf/pxf-env.sh ``` 3. Locate the `PXF_JVM_OPTS` setting in the `pxf-env.sh` file, and update the `-Xmx` and/or `-Xms` options to the desired value. For example: @@ -34,7 +34,7 @@ Perform the following procedure to increase the heap size for the PXF Service ru 4. Use the `pxf cluster sync` command to copy the updated `pxf-env.sh` file to the Greenplum Database cluster. For example: ``` shell - gpadmin@gpmaster$ pxf cluster sync + gpadmin@coordinator$ pxf cluster sync ``` 5. Restart PXF on each Greenplum Database host as described in [Restarting PXF](cfginitstart_pxf.html#restart_pxf). @@ -96,16 +96,16 @@ Refer to the configuration [procedure](#pxf-cfgoom_proc) below for the instructi Auto-termination of the PXF Service on OOM is deactivated by default. Heap dump generation on OOM is deactivated by default. To configure one or both of these properties, perform the following procedure: -1. Log in to your Greenplum Database master host: +1. Log in to your Greenplum Database coordinator host: ``` shell - $ ssh gpadmin@ + $ ssh gpadmin@ ``` 2. Edit the `$PXF_BASE/conf/pxf-env.sh` file. For example: ``` shell - gpadmin@gpmaster$ vi $PXF_BASE/conf/pxf-env.sh + gpadmin@coordinator$ vi $PXF_BASE/conf/pxf-env.sh ``` 3. If you want to configure (i.e. turn off, or turn back on) auto-termination of the PXF Service on OOM, locate the `PXF_OOM_KILL` property in the `pxf-env.sh` file. If the setting is commented out, uncomment it, and then update the value. For example, to turn off this behavior, set the value to `false`: @@ -133,7 +133,7 @@ Auto-termination of the PXF Service on OOM is deactivated by default. Heap dump 6. Use the `pxf cluster sync` command to copy the updated `pxf-env.sh` file to the Greenplum Database cluster. For example: ``` shell - gpadmin@gpmaster$ pxf cluster sync + gpadmin@coordinator$ pxf cluster sync ``` 7. Restart PXF on each Greenplum Database host as described in [Restarting PXF](cfginitstart_pxf.html#restart_pxf). @@ -151,16 +151,16 @@ If you plan to run large workloads on a large number of files in an external Hiv Perform the following procedure to set the maximum number of Tomcat threads for the PXF Service running on each host in your Greenplum Database deployment. -1. Log in to your Greenplum Database master host: +1. Log in to your Greenplum Database coordinator host: ``` shell - $ ssh gpadmin@ + $ ssh gpadmin@ ``` 2. Edit the `$PXF_BASE/conf/pxf-application.properties` file. For example: ``` shell - gpadmin@gpmaster$ vi $PXF_BASE/conf/pxf-application.properties + gpadmin@coordinator$ vi $PXF_BASE/conf/pxf-application.properties ``` 3. Locate the `pxf.max.threads` setting in the `pxf-application.properties` file. If the setting is commented out, uncomment it, and then update to the desired value. For example, to reduce the maximum number of Tomcat threads to 100: @@ -174,7 +174,7 @@ Perform the following procedure to set the maximum number of Tomcat threads for 4. Use the `pxf cluster sync` command to copy the updated `pxf-application.properties` file to the Greenplum Database cluster. For example: ``` shell - gpadmin@gpmaster$ pxf cluster sync + gpadmin@coordinator$ pxf cluster sync ``` 5. Restart PXF on each Greenplum Database host as described in [Restarting PXF](cfginitstart_pxf.html#restart_pxf). diff --git a/docs/content/cfg_server.html.md.erb b/docs/content/cfg_server.html.md.erb index 25a5410175..fae771b19d 100644 --- a/docs/content/cfg_server.html.md.erb +++ b/docs/content/cfg_server.html.md.erb @@ -25,7 +25,7 @@ The configuration information for a PXF server resides in one or more `/templates/` directory after you install PXF: ``` -gpadmin@gpmaster$ ls /templates +gpadmin@coordinator$ ls /templates adl-site.xml hbase-site.xml jdbc-site.xml pxf-site.xml yarn-site.xml core-site.xml hdfs-site.xml mapred-site.xml s3-site.xml gs-site.xml hive-site.xml minio-site.xml wasbs-site.xml @@ -92,7 +92,7 @@ PXF includes a template file named `pxf-site.xml` for PXF-specific configuration - Kerberos and/or user impersonation settings for server configurations - a base directory for file access -- the action of PXF when it detects an overflow condition while writing numeric Parquet data +- the action of PXF when it detects an overflow condition while writing numeric ORC or Parquet data
Note: The Kerberos and user impersonation settings in this file may apply only to Hadoop and JDBC server configurations; they do not apply to file system or object store server configurations.
@@ -116,6 +116,7 @@ You configure properties in the `pxf-site.xml` file for a PXF server when one or | pxf.fs.basePath | Identifies the base path or share point on the remote file system. This property is applicable when the server configuration is used with a profile that accesses a file. | None; this property is commented out by default. | | pxf.ppd.hive1 | Specifies whether or not predicate pushdown is enabled for queries on external tables that specify the `hive`, `hive:rc`, or `hive:orc` profiles. | True; predicate pushdown is enabled. | | pxf.sasl.connection.retries | Specifies the maximum number of times that PXF retries a SASL connection request after a refused connection returns a `GSS initiate failed` error. | 5 | +| pxf.orc.write.decimal.overflow | Specifies how PXF handles numeric data that exceeds the maximum precision of 38 and [overflows](hdfs_orc.html#overflow) when writing to an ORC file. Valid values are: round, error, or ignore | round | | pxf.parquet.write.decimal.overflow | Specifies how PXF handles numeric data that exceeds the maximum precision of 38 and [overflows](hdfs_parquet.html#overflow) when writing to a Parquet file. Valid values are: round, error, or ignore | round |
1 Should you need to, you can override this setting on a per-table basis by specifying the `&PPD=` option in the `LOCATION` clause when you create the external table. diff --git a/docs/content/cfghostport.html.md.erb b/docs/content/cfghostport.html.md.erb index 5959180a54..ddb9d2e222 100644 --- a/docs/content/cfghostport.html.md.erb +++ b/docs/content/cfghostport.html.md.erb @@ -1,48 +1,135 @@ --- -title: Service Host and Port +title: Service Listen Address, Host, and Port --- -By default, a PXF Service started on a Greenplum host listens on port number `5888` on `localhost`. You can configure PXF to start on a different port number, or use a different hostname or IP address. To change the default configuration, you will set one or both of the environment variables identified below: +In the default deployment topology, since PXF 6.7.0, the PXF Service starts on a Greenplum host and listens on `localhost:5888`. With this configuration, the PXF Service listens for local traffic on the Greenplum host. You can configure PXF to listen on a different listen address. You can also configure PXF to listen on a different port number, or to run on a different host. To change the default configuration, you set one or more of the properties identified below: -| Environment Variable | Description | -|-------------------------|-------------------| -| PXF_HOST | The name of the host or IP address. The default host name is `localhost`. | -| PXF_PORT | The port number on which the PXF Service listens for requests on the host. The default port number is `5888`. | +| Property | Type | Description | Default | +|-------------------------|-------------------|-----| +| server.address | `pxf-application.properties` property | The PXF server listen address. | `localhost` | +| PXF_HOST | Environment variable | The name or IP address of the (non-Greenpum) host on which the PXF Service is running. | `localhost` | +| PXF_PORT | Environment variable | The port number on which the PXF server listens for requests on the host. | `5888` | -Set the environment variables in the `gpadmin` user's `.bashrc` shell login file on each Greenplum host. -
Note: You must restart both Greenplum Database and PXF when you configure the service host and/or port in this manner. Consider performing this configuration during a scheduled down time.
+## Configuring the Listen Address -## Procedure +The `server.address` property identifies the IP address or hostname of the network interface on which the PXF service listens. The default PXF service listen address is `localhost`. You may choose to change the listen address to allow traffic from other hosts to send requests to PXF (for example, when you have chosen the [alternate deployment topology](deployment_topos.html#alt_topo) or to retrieve PXF monitoring data). -Perform the following procedure to configure the PXF Service host and/or port number on one or more Greenplum Database hosts: +Perform the following procedure to change the PXF listen address: -1. Log in to your Greenplum Database master host: +1. Log in to your Greenplum Database coordinator host: ``` shell - $ ssh gpadmin@ + $ ssh gpadmin@ + ``` + +1. Locate the `pxf-application.properties` file in your PXF installation. If you did not relocate `$PXF_BASE`, the file resides here: + + ``` pre + /usr/local/pxf-gp6/conf/pxf-application.properties + ``` + +1. Open the file in the editor of your choice, uncomment and set the following line: + + ``` pre + server.address= + ``` + + Changing the listen address to `0.0.0.0` allows PXF to listen for requests from all hosts. + +1. Save the file and exit the editor. + +1. Synchronize the PXF configuration and then restart PXF: + + ``` shell + gpadmin@coordinator$ pxf cluster sync + gpadmin@coordinator$ pxf cluster restart + ``` + +## Configuring the Port Number + +
Note: You must restart both Greenplum Database and PXF when you configure the service port number in this manner. Consider performing this configuration during a scheduled down time.
+ +Perform the following procedure to configure the port number of the PXF server on one or more Greenplum Database hosts: + +1. Log in to your Greenplum Database coordinator host: + + ``` shell + $ ssh gpadmin@ ``` 2. For each Greenplum Database host: - 1. Identify the host name or IP address of the PXF Service. - 2. Identify the port number on which you want the PXF Service to run. - 3. Log in to the Greenplum Database host: + 1. Identify the port number on which you want the PXF Service to listen. + 1. Log in to the Greenplum Database host: ``` shell $ ssh gpadmin@ ``` - 4. Open the `~/.bashrc` file in the editor of your choice. - 5. Set the `PXF_HOST` and/or `PXF_PORT` environment variables. For example, to set the PXF Service port number to 5998, add the following to the `.bashrc` file: + 1. Open the `~/.bashrc` file in the editor of your choice. + 1. Set the `PXF_PORT` environment variable. For example, to set the PXF Service port number to 5998, add the following to the `.bashrc` file: ``` shell export PXF_PORT=5998 ``` - 4. Save the file and exit the editor. + 1. Save the file and exit the editor. + +1. Source the `.bashrc` that file you just updated: + + ``` shell + gpadmin@coordinator$ source ~/.bashrc + ``` 3. Restart Greenplum Database as described in [Restarting Greenplum Database](https://docs.vmware.com/en/VMware-Greenplum/6/greenplum-database/admin_guide-managing-startstop.html#restarting-greenplum-database) in the Greenplum Documentation. -4. Restart PXF on each Greenplum Database host as described in [Restarting PXF](cfginitstart_pxf.html#restart_pxf). +4. Restart PXF on each Greenplum Database host: + + ``` shell + gpadmin@coordinator$ pxf cluster restart + ``` + +5. Verify that PXF is running on the reconfigured port by invoking `http://:/actuator/health` to view PXF monitoring information as described in [About PXF Service Runtime Monitoring](monitor_pxf.html#about_rtm). + + +## Configuring the Host + +If you have chosen the [alternate deployment topology](deployment_topos.html#alt_topo) for PXF, you must set the `PXF_HOST` environment variable on each Greenplum segment host to inform Greenplum of the location of the PXF service. You must also set the listen address as described in [Configuring the Listen Address](#listen_address). + +Perform the following procedure to configure the PXF host on each Greenplum Database segment host: + +
Note: You must restart Greenplum Database when you configure the host in this manner. Consider performing this configuration during a scheduled down time.
+ +1. Log in to your Greenplum Database coordinator host: + + ``` shell + $ ssh gpadmin@ + ``` + +2. For each Greenplum Database segment host: + + 1. Identify the host name or IP address of a PXF Server. + 3. Log in to the Greenplum Database segment host: + + ``` shell + $ ssh gpadmin@ + ``` + 4. Open the `~/.bashrc` file in the editor of your choice. + 5. Set the `PXF_HOST` environment variable. For example, to set the PXF host to `pxfalthost1`, add the following to the `.bashrc` file: + + ``` shell + export PXF_HOST=pxfalthost1 + ``` + 4. Save the file and exit the editor. + +1. Source the `.bashrc` that file you just updated: + + ``` shell + gpadmin@coordinator$ source ~/.bashrc + ``` + +1. Configure the listen address of the PXF Service as described in [Configuring the Listen Address](#listen_address). + +3. Restart Greenplum Database as described in [Restarting Greenplum Database](https://docs.vmware.com/en/VMware-Greenplum/6/greenplum-database/admin_guide-managing-startstop.html#restarting-greenplum-database) in the Greenplum Documentation. -5. Verify that PXF is running on the reconfigured host and/or port by invoking `http://:/actuator/health` to view PXF monitoring information as described in [About PXF Service Runtime Monitoring](monitor_pxf.html#about_rtm). +5. Verify that PXF is running on the reconfigured host by invoking `http://:/actuator/health` to view PXF monitoring information as described in [About PXF Service Runtime Monitoring](monitor_pxf.html#about_rtm). diff --git a/docs/content/cfginitstart_pxf.html.md.erb b/docs/content/cfginitstart_pxf.html.md.erb index 42a356ef95..2a3ab73ccd 100644 --- a/docs/content/cfginitstart_pxf.html.md.erb +++ b/docs/content/cfginitstart_pxf.html.md.erb @@ -48,16 +48,16 @@ Before you start PXF in your Greenplum Database cluster, ensure that: Perform the following procedure to start PXF on each host in your Greenplum Database cluster. -1. Log in to the Greenplum Database master host: +1. Log in to the Greenplum Database coordinator host: ``` shell - $ ssh gpadmin@ + $ ssh gpadmin@ ``` 3. Run the `pxf cluster start` command to start PXF on each host: ```shell - gpadmin@gpmaster$ pxf cluster start + gpadmin@coordinator$ pxf cluster start ``` ## Stopping PXF @@ -72,16 +72,16 @@ Before you stop PXF in your Greenplum Database cluster, ensure that your Greenpl Perform the following procedure to stop PXF on each host in your Greenplum Database cluster. -1. Log in to the Greenplum Database master host: +1. Log in to the Greenplum Database coordinator host: ``` shell - $ ssh gpadmin@ + $ ssh gpadmin@ ``` 3. Run the `pxf cluster stop` command to stop PXF on each host: ```shell - gpadmin@gpmaster$ pxf cluster stop + gpadmin@coordinator$ pxf cluster stop ``` ## Restarting PXF @@ -98,15 +98,15 @@ Before you restart PXF in your Greenplum Database cluster, ensure that your Gree Perform the following procedure to restart PXF in your Greenplum Database cluster. -1. Log in to the Greenplum Database master host: +1. Log in to the Greenplum Database coordinator host: ``` shell - $ ssh gpadmin@ + $ ssh gpadmin@ ``` 2. Restart PXF: ```shell - gpadmin@gpmaster$ pxf cluster restart + gpadmin@coordinator$ pxf cluster restart ``` diff --git a/docs/content/client_instcfg.html.md.erb b/docs/content/client_instcfg.html.md.erb index 266f0f5def..fda8778101 100644 --- a/docs/content/client_instcfg.html.md.erb +++ b/docs/content/client_instcfg.html.md.erb @@ -8,19 +8,19 @@ PXF is compatible with Cloudera, Hortonworks Data Platform, MapR, and generic Ap ## Prerequisites -Configuring PXF Hadoop connectors involves copying configuration files from your Hadoop cluster to the Greenplum Database master host. If you are using the MapR Hadoop distribution, you must also copy certain JAR files to the master host. Before you configure the PXF Hadoop connectors, ensure that you can copy files from hosts in your Hadoop cluster to the Greenplum Database master. +Configuring PXF Hadoop connectors involves copying configuration files from your Hadoop cluster to the Greenplum Database coordinator host. If you are using the MapR Hadoop distribution, you must also copy certain JAR files to the coordinator host. Before you configure the PXF Hadoop connectors, ensure that you can copy files from hosts in your Hadoop cluster to the Greenplum Database coordinator. ## Procedure -Perform the following procedure to configure the desired PXF Hadoop-related connectors on the Greenplum Database master host. After you configure the connectors, you will use the `pxf cluster sync` command to copy the PXF configuration to the Greenplum Database cluster. +Perform the following procedure to configure the desired PXF Hadoop-related connectors on the Greenplum Database coordinator host. After you configure the connectors, you will use the `pxf cluster sync` command to copy the PXF configuration to the Greenplum Database cluster. -In this procedure, you use the `default`, or create a new PXF server configuration. You copy Hadoop configuration files to the server configuration directory on the Greenplum Database master host. You identify Kerberos and user impersonation settings required for access, if applicable. You may also copy libraries to `$PXF_BASE/lib` for MapR support. You then synchronize the PXF configuration on the master host to the standby master host and segment hosts. +In this procedure, you use the `default`, or create a new PXF server configuration. You copy Hadoop configuration files to the server configuration directory on the Greenplum Database coordinator host. You identify Kerberos and user impersonation settings required for access, if applicable. You may also copy libraries to `$PXF_BASE/lib` for MapR support. You then synchronize the PXF configuration on the coordinator host to the standby coordinator host and segment hosts. -1. Log in to your Greenplum Database master host: +1. Log in to your Greenplum Database coordinator host: ``` shell - $ ssh gpadmin@ + $ ssh gpadmin@ ``` 2. Identify the name of your PXF Hadoop server configuration. @@ -28,55 +28,55 @@ In this procedure, you use the `default`, or create a new PXF server configurati 3. If you are not using the `default` PXF server, create the `$PXF_BASE/servers/` directory. For example, use the following command to create a Hadoop server configuration named `hdp3`: ``` shell - gpadmin@gpmaster$ mkdir $PXF_BASE/servers/hdp3 + gpadmin@coordinator$ mkdir $PXF_BASE/servers/hdp3 ```` 4. Change to the server directory. For example: ```shell - gpadmin@gpmaster$ cd $PXF_BASE/servers/default + gpadmin@coordinator$ cd $PXF_BASE/servers/default ``` Or, ```shell - gpadmin@gpmaster$ cd $PXF_BASE/servers/hdp3 + gpadmin@coordinator$ cd $PXF_BASE/servers/hdp3 ``` 2. PXF requires information from `core-site.xml` and other Hadoop configuration files. Copy the `core-site.xml`, `hdfs-site.xml`, `mapred-site.xml`, and `yarn-site.xml` Hadoop configuration files from your Hadoop cluster NameNode host to the current host using your tool of choice. Your file paths may differ based on the Hadoop distribution in use. For example, these commands use `scp` to copy the files: ``` shell - gpadmin@gpmaster$ scp hdfsuser@namenode:/etc/hadoop/conf/core-site.xml . - gpadmin@gpmaster$ scp hdfsuser@namenode:/etc/hadoop/conf/hdfs-site.xml . - gpadmin@gpmaster$ scp hdfsuser@namenode:/etc/hadoop/conf/mapred-site.xml . - gpadmin@gpmaster$ scp hdfsuser@namenode:/etc/hadoop/conf/yarn-site.xml . + gpadmin@coordinator$ scp hdfsuser@namenode:/etc/hadoop/conf/core-site.xml . + gpadmin@coordinator$ scp hdfsuser@namenode:/etc/hadoop/conf/hdfs-site.xml . + gpadmin@coordinator$ scp hdfsuser@namenode:/etc/hadoop/conf/mapred-site.xml . + gpadmin@coordinator$ scp hdfsuser@namenode:/etc/hadoop/conf/yarn-site.xml . ``` -3. If you plan to use the PXF Hive connector to access Hive table data, similarly copy the Hive configuration to the Greenplum Database master host. For example: +3. If you plan to use the PXF Hive connector to access Hive table data, similarly copy the Hive configuration to the Greenplum Database coordinator host. For example: ``` shell - gpadmin@gpmaster$ scp hiveuser@hivehost:/etc/hive/conf/hive-site.xml . + gpadmin@coordinator$ scp hiveuser@hivehost:/etc/hive/conf/hive-site.xml . ``` -4. If you plan to use the PXF HBase connector to access HBase table data, similarly copy the HBase configuration to the Greenplum Database master host. For example: +4. If you plan to use the PXF HBase connector to access HBase table data, similarly copy the HBase configuration to the Greenplum Database coordinator host. For example: ``` shell - gpadmin@gpmaster$ scp hbaseuser@hbasehost:/etc/hbase/conf/hbase-site.xml . + gpadmin@coordinator$ scp hbaseuser@hbasehost:/etc/hbase/conf/hbase-site.xml . ``` -2. If you are using PXF with the MapR Hadoop distribution, you must copy certain JAR files from your MapR cluster to the Greenplum Database master host. (Your file paths may differ based on the version of MapR in use.) For example, these commands use `scp` to copy the files: +2. If you are using PXF with the MapR Hadoop distribution, you must copy certain JAR files from your MapR cluster to the Greenplum Database coordinator host. (Your file paths may differ based on the version of MapR in use.) For example, these commands use `scp` to copy the files: ``` shell - gpadmin@gpmaster$ cd $PXF_BASE/lib - gpadmin@gpmaster$ scp mapruser@maprhost:/opt/mapr/hadoop/hadoop-2.7.0/share/hadoop/common/lib/maprfs-5.2.2-mapr.jar . - gpadmin@gpmaster$ scp mapruser@maprhost:/opt/mapr/hadoop/hadoop-2.7.0/share/hadoop/common/lib/hadoop-auth-2.7.0-mapr-1707.jar . - gpadmin@gpmaster$ scp mapruser@maprhost:/opt/mapr/hadoop/hadoop-2.7.0/share/hadoop/common/hadoop-common-2.7.0-mapr-1707.jar . + gpadmin@coordinator$ cd $PXF_BASE/lib + gpadmin@coordinator$ scp mapruser@maprhost:/opt/mapr/hadoop/hadoop-2.7.0/share/hadoop/common/lib/maprfs-5.2.2-mapr.jar . + gpadmin@coordinator$ scp mapruser@maprhost:/opt/mapr/hadoop/hadoop-2.7.0/share/hadoop/common/lib/hadoop-auth-2.7.0-mapr-1707.jar . + gpadmin@coordinator$ scp mapruser@maprhost:/opt/mapr/hadoop/hadoop-2.7.0/share/hadoop/common/hadoop-common-2.7.0-mapr-1707.jar . ``` 5. Synchronize the PXF configuration to the Greenplum Database cluster: ``` shell - gpadmin@gpmaster$ pxf cluster sync + gpadmin@coordinator$ pxf cluster sync ``` 4. PXF accesses Hadoop services on behalf of Greenplum Database end users. By default, PXF tries to access HDFS, Hive, and HBase using the identity of the Greenplum Database user account that logs into Greenplum Database. In order to support this functionality, you must configure proxy settings for Hadoop, as well as for Hive and HBase if you intend to use those PXF connectors. Follow procedures in [Configuring User Impersonation and Proxying](pxfuserimpers.html) to configure user impersonation and proxying for Hadoop services, or to turn off PXF user impersonation. @@ -91,8 +91,8 @@ In this procedure, you use the `default`, or create a new PXF server configurati If you update your Hadoop, Hive, or HBase configuration while the PXF Service is running, you must copy the updated configuration to the `$PXF_BASE/servers/` directory and re-sync the PXF configuration to your Greenplum Database cluster. For example: ``` shell -gpadmin@gpmaster$ cd $PXF_BASE/servers/ -gpadmin@gpmaster$ scp hiveuser@hivehost:/etc/hive/conf/hive-site.xml . -gpadmin@gpmaster$ pxf cluster sync +gpadmin@coordinator$ cd $PXF_BASE/servers/ +gpadmin@coordinator$ scp hiveuser@hivehost:/etc/hive/conf/hive-site.xml . +gpadmin@coordinator$ pxf cluster sync ``` diff --git a/docs/content/config_files.html.md.erb b/docs/content/config_files.html.md.erb index a6d6de95d3..23f6b582cb 100644 --- a/docs/content/config_files.html.md.erb +++ b/docs/content/config_files.html.md.erb @@ -21,13 +21,13 @@ Procedure: 1. Synchronize the PXF configuration to all hosts in the Greenplum Database cluster: ``` shell - gpadmin@gpmaster$ pxf cluster sync + gpadmin@coordinator$ pxf cluster sync ``` 1. (Re)start PXF on all Greenplum hosts: ``` shell - gpadmin@gpmaster$ pxf cluster restart + gpadmin@coordinator$ pxf cluster restart ``` @@ -46,6 +46,7 @@ The `pxf-application.properties` file exposes these PXF Service application conf | pxf.task.pool.max-size | The maximum allowed number of core streaming threads. | pxf.max.threads if set, or 200 | | [pxf.log.level](cfg_logging.html) | The log level for the PXF Service. | info | | pxf.fragmenter-cache.expiration | The amount of time after which an entry expires and is removed from the fragment cache. | 10s (10 seconds) | +| [server.address](cfghostport.html) | The PXF server listen address. | localhost | To change the value of a PXF Service application property, you may first need to add the property to, or uncomment the property in, the `pxf-application.properties` file before you can set the new value. @@ -76,5 +77,5 @@ The [Logging](cfg_logging.html) advanced configuration topic describes how to en ## pxf-profiles.xml -PXF defines its default profiles in the [`pxf-profiles-default.xml`](https://github.com/greenplum-db/pxf/blob/master/server/pxf-service/src/main/resources/pxf-profiles-default.xml) file. If you choose to add a custom profile, you configure the profile in `pxf-profiles.xml`. +PXF defines its default profiles in the [`pxf-profiles-default.xml`](https://github.com/greenplum-db/pxf/blob/main/server/pxf-service/src/main/resources/pxf-profiles-default.xml) file. If you choose to add a custom profile, you configure the profile in `pxf-profiles.xml`. diff --git a/docs/content/deployment_topos.html.md.erb b/docs/content/deployment_topos.html.md.erb new file mode 100644 index 0000000000..573b2e3464 --- /dev/null +++ b/docs/content/deployment_topos.html.md.erb @@ -0,0 +1,17 @@ +--- +title: About the PXF Deployment Topology +--- + +The default PXF deployment topology is co-located; you install PXF on each Greenplum host, and the PXF Service starts and runs on each Greenplum segment host. + +You manage the PXF services deployed in a co-located topology using the [pxf cluster](ref/pxf-cluster.html) commands. + + +## Alternate Deployment Topology + +Running the PXF Service on non-Greenplum hosts is an alternate deployment topology. If you choose this topology, you must install PXF on both the non-Greenplum hosts and on all Greenplum hosts. + +In the alternate deployment topology, you manage the PXF services individually using the [pxf](ref/pxf.html) command on each host; you can not use the `pxf cluster` commands to collectively manage the PXF services in this topology. + +If you choose the alternate deployment topology, you must explicitly configure each Greenplum host to identify the host and listen address on which the PXF Service is running. These procedures are described in [Configuring the Host](cfghostport.html#host) and [Configuring the Listen Address](cfghostport.html#listen_address). + diff --git a/docs/content/hdfs_fileasrow.html.md.erb b/docs/content/hdfs_fileasrow.html.md.erb index 82e522b6cb..23cf3b4f1e 100644 --- a/docs/content/hdfs_fileasrow.html.md.erb +++ b/docs/content/hdfs_fileasrow.html.md.erb @@ -6,7 +6,7 @@ You can use the PXF HDFS connector to read one or more multi-line text files in PXF supports reading only text and JSON files in this manner. -**Note**: Refer to the [Reading JSON Data from HDFS](hdfs_json.html) topic if you want to use PXF to read JSON files that include more than one record. +**Note**: Refer to the [Reading and Writing JSON Data in HDFS](hdfs_json.html) topic if you want to use PXF to read JSON files that include more than one record. ## Prerequisites diff --git a/docs/content/hdfs_fixedwidth.html.md.erb b/docs/content/hdfs_fixedwidth.html.md.erb index 48d6ec0f2c..bbdc961466 100644 --- a/docs/content/hdfs_fixedwidth.html.md.erb +++ b/docs/content/hdfs_fixedwidth.html.md.erb @@ -144,12 +144,11 @@ The specific keywords and values used in the [CREATE EXTERNAL TABLE](https://doc | line_delim | The line delimiter character in the data. Preface the \ with an `E` when the value is an escape sequence. Examples: `line_delim=E'\n'`, `line_delim 'aaa'`. The default value is `'\n'`. | | DISTRIBUTED BY | If you want to load data from an existing Greenplum Database table into the writable external table, consider specifying the same distribution policy or `` on both tables. Doing so will avoid extra motion of data between segments on the load operation. | -Writable external tables that you create using the `hdfs:fixedwidth` profile can optionally use record or block compression. You specify the compression type and codec via options in the `CREATE WRITABLE EXTERNAL TABLE` `LOCATION` clause: +Writable external tables that you create using the `hdfs:fixedwidth` profile can optionally use record or block compression. You specify the compression codec via an option in the `CREATE WRITABLE EXTERNAL TABLE` `LOCATION` clause: | Write Option | Value Description | |-------|-------------------------------------| | COMPRESSION_CODEC | The compression codec alias. Supported compression codecs for writing fixed-width text data include: `default`, `bzip2`, `gzip`, and `uncompressed`. If this option is not provided, Greenplum Database performs no data compression. | -| COMPRESSION_TYPE | The compression type to employ; supported values are `RECORD` (the default) or `BLOCK`. | ## Example: Writing Fixed-Width Text Data to HDFS diff --git a/docs/content/hdfs_json.html.md.erb b/docs/content/hdfs_json.html.md.erb index 3a817a50e2..d6d830d757 100644 --- a/docs/content/hdfs_json.html.md.erb +++ b/docs/content/hdfs_json.html.md.erb @@ -1,5 +1,5 @@ --- -title: Reading JSON Data from HDFS +title: Reading and Writing JSON Data in HDFS --- -Use the PXF HDFS Connector to read JSON-format data. This section describes how to use PXF to access JSON data in HDFS, including how to create and query an external table that references a JSON file in the HDFS data store. +Use the PXF HDFS Connector to read and write JSON-format data. This section describes how to use PXF and external tables to access and write JSON data in HDFS. ## Prerequisites -Ensure that you have met the PXF Hadoop [Prerequisites](access_hdfs.html#hadoop_prereq) before you attempt to read data from HDFS. +Ensure that you have met the PXF Hadoop [Prerequisites](access_hdfs.html#hadoop_prereq) before you attempt to read data from or write data to HDFS. ## Working with JSON Data -JSON is a text-based data-interchange format. JSON data is typically stored in a file with a `.json` suffix. +JSON is a text-based data-interchange format. A JSON data file contains one or more JSON objects. A JSON object is a collection of unordered name/value pairs. A value can be a string, a number, true, false, null, an object, or an array. You can define nested JSON objects and arrays. -A `.json` file will contain a collection of objects. A JSON object is a collection of unordered name/value pairs. A value can be a string, a number, true, false, null, or an object or an array. You can define nested JSON objects and arrays. +JSON data is typically stored in a file with a `.json` or `.jsonl` (JSON Lines) suffix as described in the sections below. + +### About the PXF JSON Data Access Modes + +PXF supports two data access modes for JSON files. The default mode expects one full JSON record per row (JSONL). PXF also supports an access mode that expects one JSON object per file where the JSON records may (but are not required to) span multiple lines. + +#### Single Object Per Row + +A JSON file can contain a single JSON object per row, where each row represents a database tuple. A JSON file that PXF reads that contains a single object per row may have any or no suffix. When writing, PXF creates the file with a `.jsonl` suffix. + +Excerpt of sample single-object-per-row JSON data file: + +``` pre +{"id":1,"color":"red"} +{"id":2,"color":"yellow"} +{"id":3,"color":"green"} +``` + +Refer to [JSON Lines](https://jsonlines.org/) for detailed information about this JSON syntax. + +#### Single Object Per File + +A JSON file can also contain a single, named, root level JSON object whose value is an array of JSON objects. When reading, the array may contain objects with arbitrary complexity and nesting, and PXF forms database tuples from objects that have a property named the same as that specified for the `IDENTIFIER` (discussed below). When writing, each JSON object in the array represents a database tuple. JSON files of this type have the `.json` suffix. + +In the following example JSON data file, the root-level `records` object is an array of three objects (tuples): + +``` pre +{"records":[ +{"id":1,"color":"red"} +,{"id":2,"color":"yellow"} +,{"id":3,"color":"green"} +]} +``` + +The records in the single JSON object may also span multiple lines: + +``` pre +{ + "records":[ + { + "id":1, + "color":"red" + }, + { + "id":2, + "color":"yellow" + }, + { + "id":3, + "color":"green" + } + ] +} +``` + +Refer to [Introducing JSON](http://www.json.org/) for detailed information about this JSON syntax. + +## Data Type Mapping + +To represent JSON data in Greenplum Database, map data values that use a primitive data type to Greenplum Database columns of the same type. JSON supports complex data types including projections and arrays. + +### Read Mapping + +PXF uses the following data type mapping when reading JSON data: + +| JSON Data Type | PXF/Greenplum Data Type | +|-------------------|-------------------------------------------| +| boolean | boolean | +| number | { bigint | float8 | integer | numeric | real | smallint } | +| string | text | +| string (base64-encoded value) | bytea | +| string (date, time, timestamp, timestamptz in a text format that Greenplum understands)1 | { date | time | timestamp | timestamptz } | +| Array (one dimension) of type boolean[] | boolean[] | +| Array (one dimension) of type number[] | { bigint[] | float8[] | integer[] | numeric[] | real[] | smallint[] } | +| Array (one dimension) of type string[] (base64-encoded value) | bytea[] | +| Array (one dimension) of type string[] (date, time, timestamp in a text format that Greenplum understands)1 | { date[] | time[] | timestamp[] | timestamptz[] } | +| Array (one dimension) of type string[] | text[] | +| Array of other types | text[] | +| Object | Use dot `.` notation to specify each level of projection (nesting) to a member of a primitive or Array type. | +1 PXF returns an error if Greenplum cannot convert the date or time string to the target type. + +When reading, you can use N-level projection to map members of nested objects and arrays to primitive data types. + +### Write Mapping + +PXF supports writing primitive types and single dimension arrays of primitive types. PXF supports writing other complex types to JSON as string. + +PXF uses the following data type mapping when writing JSON data: + +| PXF/Greenplum Data Type | JSON Data Type | +|-------------------|----------------------| +| bigint, float8, integer, numeric, real, smallint | number | +| boolean | boolean | +| bpchar, text, varchar | string | +| bytea | string (base64-encoded value) | +| date, time, timestamp, timestamptz | string | +| boolean[] | boolean[] | +| bigint[], float8[], int[], numeric[], real[], smallint[] | number[] | +| bytea[] | string[] (base64-encoded value) | +| date[], time[], timestamp[], timestamptz[] | string[] | + +## About Using Projection (Read) + +In the example JSON data file excerpt below, `user` is an object composed of fields named `id` and `location`: -Sample JSON data file content: ``` json { @@ -53,7 +155,7 @@ Sample JSON data file content: } ``` -In the sample above, `user` is an object composed of fields named `id` and `location`. To specify the nested fields in the `user` object as Greenplum Database external table columns, use `.` projection: +To specify the nested fields in the `user` object directly as Greenplum Database external table columns, use `.` projection: ``` pre user.id @@ -62,49 +164,78 @@ user.location `coordinates` is an object composed of a text field named `type` and an array of integers named `values`. -To retrieve all of the elements of the `values` array in a single column, define the corresponding Greenplum Database external table column as type `TEXT[]`. +To read all of the elements of the `values` array in a single column, define the corresponding Greenplum Database external table column as type `int[]`. ``` pre -"coordinates.values" TEXT[] +"coordinates.values" int[] +``` + +## Creating the External Table + +Use the `hdfs:json` profile to read or write JSON-format data in HDFS. The following syntax creates a Greenplum Database external table that references such a file: + +``` sql +CREATE [WRITABLE] EXTERNAL TABLE + ( [, ...] | LIKE ) +LOCATION ('pxf://?PROFILE=hdfs:json[&SERVER=][&=[...]]') +FORMAT 'CUSTOM' (FORMATTER='pxfwritable_import'|'pxfwritable_export') +[DISTRIBUTED BY ( [, ... ] ) | DISTRIBUTED RANDOMLY]; ``` -Refer to [Introducing JSON](http://www.json.org/) for detailed information on JSON syntax. +The specific keywords and values used in the Greenplum Database [CREATE EXTERNAL TABLE](https://docs.vmware.com/en/VMware-Greenplum/6/greenplum-database/ref_guide-sql_commands-CREATE_EXTERNAL_TABLE.html) command are described in the table below. + +| Keyword | Value | +|-------|-------------------------------------| +| \ | The path to the directory or file in the HDFS data store. When the `` configuration includes a [`pxf.fs.basePath`](cfg_server.html#pxf-fs-basepath) property setting, PXF considers \ to be relative to the base path specified. Otherwise, PXF considers it to be an absolute path. \ must not specify a relative path nor include the dollar sign (`$`) character. | +| PROFILE | The `PROFILE` keyword must specify `hdfs:json`. | +| SERVER=\ | The named server configuration that PXF uses to access the data. PXF uses the `default` server if not specified. | +| \ | \s for read and write operations are identified below.| +| FORMAT 'CUSTOM' | Use `FORMAT` '`CUSTOM`' with `(FORMATTER='pxfwritable_export')` (write) or `(FORMATTER='pxfwritable_import')` (read). | + + +PXF supports reading from and writing to JSON files that contain either an object per row (the default) or that contain a JSON single object. When the JSON file(s) that you want to read or write contains a single object, you must provide an `IDENTIFIER` \ and value. Use this option to identify the name of a field whose parent JSON object you want PXF to return or write as an individual tuple. -### JSON to Greenplum Database Data Type Mapping +The `hdfs:json` profile supports the following custom **read** options: -To represent JSON data in Greenplum Database, map data values that use a primitive data type to Greenplum Database columns of the same type. JSON supports complex data types including projections and arrays. Use N-level projection to map members of nested objects and arrays to primitive data types. +| Option Keyword | Description | +|-------|-----------------------| +| IDENTIFIER=\ | When the JSON data that you are reading is comprised of a single JSON object, you must specify an `IDENTIFIER` to identify the name of the field whose parent JSON object you want PXF to return as an individual tuple. | +| SPLIT_BY_FILE=\ | Specify how PXF splits the data in \. The default value is `false`, PXF creates multiple splits for each file that it will process in parallel. When set to `true`, PXF creates and processes a single split per file. | +| IGNORE_MISSING_PATH=\ | Specify the action to take when \ is missing or invalid. The default value is `false`, PXF returns an error in this situation. When the value is `true`, PXF ignores missing path errors and returns an empty fragment. | -The following table summarizes external mapping rules for JSON data. +
Note: When a nested object in a single object JSON file includes a field with the same name as that of a parent object field and the field name is also specified as the IDENTIFIER, there is a possibility that PXF could return incorrect results. Should you need to, you can work around this edge case by compressing the JSON file, and using PXF to read the compressed file.
-Table 1. JSON Mapping +The `hdfs:json` profile supports the following custom **write** options: - +| Option | Value Description | +|-------|-------------------------------------| +| ROOT=\ | When writing to a single JSON object, identifies the name of the root-level object attribute. | +| COMPRESSION_CODEC | The compression codec alias. Supported compression codecs for writing json data include: `default`, `bzip2`, `gzip`, and `uncompressed`. If this option is not provided, Greenplum Database performs no data compression. | +| DISTRIBUTED BY | If you are loading data from an existing Greenplum Database table into the writable external table, consider specifying the same distribution policy or `` on both tables. Doing so will avoid extra motion of data between segments on the load operation. | -| JSON Data Type | PXF/Greenplum Data Type | -|-------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Primitive type (integer, float, string, boolean, null) | Use the corresponding Greenplum Database built-in data type; see [Greenplum Database Data Types](https://docs.vmware.com/en/VMware-Greenplum/6/greenplum-database/ref_guide-data_types.html). | -| Array | Use `TEXT[]` to retrieve the JSON array as a Greenplum text array. | -| Object | Use dot `.` notation to specify each level of projection (nesting) to a member of a primitive or Array type. | +When you specify compression for a JSON write operation, PXF names the files that it writes `..`. For example: `jan_sales.jsonl.gz`. -### JSON Data Read Modes +## Read Examples -PXF supports two data read modes. The default mode expects one full JSON record per line. PXF also supports a read mode operating on JSON records that span multiple lines. +### Example Data Sets -In upcoming examples, you will use both read modes to operate on a sample data set. The schema of the sample data set defines objects with the following member names and value data types: +In upcoming read examples, you use both JSON access modes to operate on a sample data set. The schema of the sample data set defines objects with the following member names and value data types: - - "created_at" - text - - "id_str" - text - - "user" - object - - "id" - integer - - "location" - text - - "coordinates" - object (optional) - - "type" - text - - "values" - array - - [0] - integer - - [1] - integer +- "created_at" - text +- "id_str" - text +- "user" - object + - "id" - integer + - "location" - text +- "coordinates" - object (optional) -The single-JSON-record-per-line data set follows: + - "type" - text + - "values" - array + + - [0] - integer + - [1] - integer + +The data set for the single-object-per-row (JSONL) access mode follows: ``` pre {"created_at":"FriJun0722:45:03+00002013","id_str":"343136551322136576","user":{"id":395504494,"location":"NearCornwall"},"coordinates":{"type":"Point","values": [ 6, 50 ]}}, @@ -112,7 +243,7 @@ The single-JSON-record-per-line data set follows: {"created_at":"FriJun0722:45:02+00002013","id_str":"343136547136233472","user":{"id":287819058,"location":""}, "coordinates": null} ``` -This is the data set for the multi-line JSON record data set: +The data set for the single-object-per-file JSON access mode follows: ``` json { @@ -149,137 +280,88 @@ This is the data set for the multi-line JSON record data set: You will create JSON files for the sample data sets and add them to HDFS in the next section. -## Loading the Sample JSON Data to HDFS +### Loading the Sample JSON Data to HDFS -The PXF HDFS connector reads native JSON stored in HDFS. Before you can use Greenplum Database to query JSON format data, the data must reside in your HDFS data store. +The PXF HDFS connector can read and write native JSON stored in HDFS. -Copy and paste the single line JSON record sample data set above to a file named `singleline.json`. Similarly, copy and paste the multi-line JSON record data set to a file named `multiline.json`. +Copy and paste the object-per-row JSON sample data set above to a file named `objperrow.jsonl`. Similarly, copy and paste the single object per file JSON record data set to a file named `singleobj.json`. -**Note**: Ensure that there are **no** blank lines in your JSON files. +> **Note** Ensure that there are **no** blank lines in your JSON files. Copy the JSON data files that you just created to your HDFS data store. Create the `/data/pxf_examples` directory if you did not do so in a previous exercise. For example: ``` shell $ hdfs dfs -mkdir /data/pxf_examples -$ hdfs dfs -put singleline.json /data/pxf_examples -$ hdfs dfs -put multiline.json /data/pxf_examples -``` - -Once the data is loaded to HDFS, you can use Greenplum Database and PXF to query and analyze the JSON data. - - -## Creating the External Table - -Use the `hdfs:json` profile to read JSON-format files from HDFS. The following syntax creates a Greenplum Database readable external table that references such a file: - -``` sql -CREATE EXTERNAL TABLE - ( [, ...] | LIKE ) -LOCATION ('pxf://?PROFILE=hdfs:json[&SERVER=][&=[...]]') -FORMAT 'CUSTOM' (FORMATTER='pxfwritable_import'); +$ hdfs dfs -put objperrow.jsonl /data/pxf_examples/ +$ hdfs dfs -put singleobj.json /data/pxf_examples/ ``` -The specific keywords and values used in the Greenplum Database [CREATE EXTERNAL TABLE](https://docs.vmware.com/en/VMware-Greenplum/6/greenplum-database/ref_guide-sql_commands-CREATE_EXTERNAL_TABLE.html) command are described in the table below. - -| Keyword | Value | -|-------|-------------------------------------| -| \ | The path to the directory or file in the HDFS data store. When the `` configuration includes a [`pxf.fs.basePath`](cfg_server.html#pxf-fs-basepath) property setting, PXF considers \ to be relative to the base path specified. Otherwise, PXF considers it to be an absolute path. \ must not specify a relative path nor include the dollar sign (`$`) character. | -| PROFILE | The `PROFILE` keyword must specify `hdfs:json`. | -| SERVER=\ | The named server configuration that PXF uses to access the data. PXF uses the `default` server if not specified. | -| \ | \s are discussed below.| -| FORMAT 'CUSTOM' | Use `FORMAT` `'CUSTOM'` with the `hdfs:json` profile. The `CUSTOM` `FORMAT` requires that you specify `(FORMATTER='pxfwritable_import')`. | - - -PXF supports single- and multi- line JSON records. When you want to read multi-line JSON records, you must provide an `IDENTIFIER` \ and value. Use this \ to identify the name of a field whose parent JSON object you want to be returned as individual tuples. - -The `hdfs:json` profile supports the following \s: - -| Option Keyword |   Syntax,  Example(s)   | Description | -|-------|--------------|-----------------------| -| IDENTIFIER | `&IDENTIFIER=`
`&IDENTIFIER=created_at`| You must include the `IDENTIFIER` keyword and \ in the `LOCATION` string only when you are accessing JSON data comprised of multi-line records. Use the \ to identify the name of the field whose parent JSON object you want to be returned as individual tuples. | -| SPLIT_BY_FILE | `&SPLIT_BY_FILE=` | Specify how PXF splits the data in \. The default value is `false`, PXF creates multiple splits for each file that it will process in parallel. When set to `true`, PXF creates and processes a single split per file. | -| IGNORE_MISSING_PATH | `&IGNORE_MISSING_PATH=` | Specify the action to take when \ is missing or invalid. The default value is `false`, PXF returns an error in this situation. When the value is `true`, PXF ignores missing path errors and returns an empty fragment. | - -
Note: When a nested object in a multi-line record JSON file includes a field with the same name as that of a parent object field and the field name is also specified as the IDENTIFIER, there is a possibility that PXF could return incorrect results. Should you need to, you can work around this edge case by compressing the JSON file, and having PXF read the compressed file.
- +Once the data is loaded to HDFS, you can use Greenplum Database and PXF to query and add to the JSON data. -## Example: Reading a JSON File with Single Line Records +### Example: Single Object Per Row (Read) -Use the following [CREATE EXTERNAL TABLE](https://docs.vmware.com/en/VMware-Greenplum/6/greenplum-database/ref_guide-sql_commands-CREATE_EXTERNAL_TABLE.html) SQL command to create a readable external table that references the single-line-per-record JSON data file and uses the PXF default server. +Use the following [CREATE EXTERNAL TABLE](https://docs.vmware.com/en/VMware-Greenplum/6/greenplum-database/ref_guide-sql_commands-CREATE_EXTERNAL_TABLE.html) SQL command to create a readable external table that references the single-object-per-row JSON data file and uses the PXF default server. ```sql -CREATE EXTERNAL TABLE singleline_json_tbl( +CREATE EXTERNAL TABLE objperrow_json_tbl( created_at TEXT, id_str TEXT, "user.id" INTEGER, "user.location" TEXT, - "coordinates.values" TEXT[] + "coordinates.values" INTEGER[] ) -LOCATION('pxf://data/pxf_examples/singleline.json?PROFILE=hdfs:json') +LOCATION('pxf://data/pxf_examples/objperrow.jsonl?PROFILE=hdfs:json') FORMAT 'CUSTOM' (FORMATTER='pxfwritable_import'); ``` -Notice the use of `.` projection to access the nested fields in the `user` and `coordinates` objects. +This table reads selected fields in the JSON file. Notice the use of `.` projection to access the nested fields in the `user` and `coordinates` objects. -To query the JSON data in the external table: +To view the JSON data in the file, query the external table: ``` sql -SELECT * FROM singleline_json_tbl; +SELECT * FROM objperrow_json_tbl; ``` To access specific elements of the `coordinates.values` array, you can specify the array subscript number in square brackets: ```sql -SELECT "coordinates.values"[1], "coordinates.values"[2] FROM singleline_json_tbl; -``` - -To access the array elements as some type other than `TEXT`, you can either cast the whole column: - -```sql -SELECT "coordinates.values"::int[] FROM singleline_json_tbl; -``` - -or cast specific elements: - -```sql -SELECT "coordinates.values"[1]::int, "coordinates.values"[2]::float FROM singleline_json_tbl; +SELECT "coordinates.values"[1], "coordinates.values"[2] FROM objperrow_json_tbl; ``` +### Example: Single Object Per File (Read) -## Example: Reading a JSON file with Multi-Line Records - -The SQL command to create a readable external table from the multi-line-per-record JSON file is very similar to that of the single line data set above. You must additionally specify the `LOCATION` clause `IDENTIFIER` keyword and an associated value when you want to read multi-line JSON records. For example: +The SQL command to create a readable external table for a single object JSON file is very similar to that of the single object per row data set above. You must additionally specify the `LOCATION` clause `IDENTIFIER` keyword and an associated value. For example: ``` sql -CREATE EXTERNAL TABLE multiline_json_tbl( +CREATE EXTERNAL TABLE singleobj_json_tbl( created_at TEXT, id_str TEXT, "user.id" INTEGER, "user.location" TEXT, - "coordinates.values" TEXT[] + "coordinates.values" INTEGER[] ) -LOCATION('pxf://data/pxf_examples/multiline.json?PROFILE=hdfs:json&IDENTIFIER=created_at') +LOCATION('pxf://data/pxf_examples/singleobj.json?PROFILE=hdfs:json&IDENTIFIER=created_at') FORMAT 'CUSTOM' (FORMATTER='pxfwritable_import'); ``` `created_at` identifies the member name of the first field in the JSON record `record_obj` in the sample data schema. -To query the JSON data in this external table: +To view the JSON data in the file, query the external table: ``` sql -SELECT * FROM multiline_json_tbl; +SELECT * FROM singleobj_json_tbl; ``` -## Other Methods to Read a JSON Array +### Other Methods to Read a JSON Array Starting in version 6.2.0, PXF supports reading a JSON array into a `TEXT[]` column. PXF still supports the old methods of using array element projection or a single text-type column to read a JSON array. These access methods are described here. -### Using Array Element Projection +#### Using Array Element Projection PXF supports accessing specific elements of a JSON array using the syntax `[n]` in the table definition to identify the specific element. ```sql -CREATE EXTERNAL TABLE singleline_json_tbl_aep( +CREATE EXTERNAL TABLE objperrow_json_tbl_aep( created_at TEXT, id_str TEXT, "user.id" INTEGER, @@ -287,7 +369,7 @@ CREATE EXTERNAL TABLE singleline_json_tbl_aep( "coordinates.values[0]" INTEGER, "coordinates.values[1]" INTEGER ) -LOCATION('pxf://data/pxf_examples/singleline.json?PROFILE=hdfs:json') +LOCATION('pxf://data/pxf_examples/objperrow.jsonl?PROFILE=hdfs:json') FORMAT 'CUSTOM' (FORMATTER='pxfwritable_import'); ``` @@ -296,24 +378,24 @@ FORMAT 'CUSTOM' (FORMATTER='pxfwritable_import'); If your existing external table definition uses array element projection and you want to read the array into a `TEXT[]` column, you can use the `ALTER EXTERNAL TABLE` command to update the table definition. For example: ```sql -ALTER EXTERNAL TABLE singleline_json_tbl_aep DROP COLUMN "coordinates.values[0]", DROP COLUMN "coordinates.values[1]", ADD COLUMN "coordinates.values" TEXT[]; +ALTER EXTERNAL TABLE objperrow_json_tbl_aep DROP COLUMN "coordinates.values[0]", DROP COLUMN "coordinates.values[1]", ADD COLUMN "coordinates.values" TEXT[]; ``` If you choose to alter the external table definition in this manner, be sure to update any existing queries on the external table to account for the changes to column name and type. -### Specifying a Single Text-type Column +#### Specifying a Single Text-type Column PXF supports accessing all of the elements within an array as a single string containing the serialized JSON array by defining the corresponding Greenplum table column with one of the following data types: `TEXT`, `VARCHAR`, or `BPCHAR`. ```sql -CREATE EXTERNAL TABLE singleline_json_tbl_stc( +CREATE EXTERNAL TABLE objperrow_json_tbl_stc( created_at TEXT, id_str TEXT, "user.id" INTEGER, "user.location" TEXT, "coordinates.values" TEXT ) -LOCATION('pxf://data/pxf_examples/singleline.json?PROFILE=hdfs:json') +LOCATION('pxf://data/pxf_examples/objperrow.jsonl?PROFILE=hdfs:json') FORMAT 'CUSTOM' (FORMATTER='pxfwritable_import'); ``` @@ -322,7 +404,7 @@ If you retrieve the JSON array in a single text-type column and wish to convert ```sql SELECT user.id, ARRAY(SELECT json_array_elements_text(coordinates.values::json))::int[] AS coords -FROM singleline_json_tbl_stc; +FROM objperrow_json_tbl_stc; ``` **Note**: This conversion is possible only when you are using PXF with Greenplum Database 6.x; the function `json_array_elements_text()` is not available in Greenplum 5.x. @@ -330,8 +412,147 @@ FROM singleline_json_tbl_stc; If your external table definition uses a single text-type column for a JSON array and you want to read the array into a `TEXT[]` column, you can use the `ALTER EXTERNAL TABLE` command to update the table definition. For example: ```sql -ALTER EXTERNAL TABLE singleline_json_tbl_stc ALTER COLUMN "coordinates.values" TYPE TEXT[]; +ALTER EXTERNAL TABLE objperrow_json_tbl_stc ALTER COLUMN "coordinates.values" TYPE TEXT[]; ``` If you choose to alter the external table definition in this manner, be sure to update any existing queries on the external table to account for the change in column type. +## Writing JSON Data + +To write JSON data, you create a writable external table that references the name of a directory on HDFS. When you insert records into the writable external table, PXF writes the block(s) of data that you insert to one or more files in the directory that you specified. In the default case (single object per row), PXF writes the data to a `.jsonl` file. When you specify a `ROOT` attribute (single object per file), PXF writes to a `.json` file. + +> **Note** When writing JSON data, PXF supports only scalar or one dimensional arrays of Greenplum data types. PXF does not support column projection when writing JSON data. + +Writable external tables can only be used for `INSERT` operations. If you want to query the data that you inserted, you must create a separate readable external table that references the HDFS directory and read from that table. + +The write examples use a data schema similar to that of the read examples. + +### Example: Single Object Per Row (Write) + +In this example, we add data to a directory named `jsopr`. + +Use the following [CREATE EXTERNAL TABLE](https://docs.vmware.com/en/VMware-Greenplum/6/greenplum-database/ref_guide-sql_commands-CREATE_EXTERNAL_TABLE.html) SQL command to create a writable external table that writes JSON data in single-object-per-row format and uses the PXF default server. + +```sql +CREATE WRITABLE EXTERNAL TABLE add_objperrow_json_tbl( + created_at TEXT, + id_str TEXT, + id INTEGER, + location TEXT, + coordinates INTEGER[] +) +LOCATION('pxf://data/pxf_examples/jsopr?PROFILE=hdfs:json') +FORMAT 'CUSTOM' (FORMATTER='pxfwritable_export'); +``` + +Write data to the table: + +``` sql +INSERT INTO add_objperrow_json_tbl VALUES ( 'SunJun0912:59:07+00002013', '343136551111111111', 311111111, 'FarAway', '{ 6, 50 }' ); +INSERT INTO add_objperrow_json_tbl VALUES ( 'MonJun1002:12:06+00002013', '343136557777777777', 377777777, 'NearHere', '{ 13, 93 }' ); +``` + +Read the data that you just wrote. Recall that you must first create a readable external table: + +``` sql +CREATE EXTERNAL TABLE jsopr_tbl( + created_at TEXT, + id_str TEXT, + id INTEGER, + location TEXT, + coordinates INTEGER[] +) +LOCATION('pxf://data/pxf_examples/jsopr?PROFILE=hdfs:json') +FORMAT 'CUSTOM' (FORMATTER='pxfwritable_import'); +``` + +Query the table: + +``` sql +SELECT * FROM jsopr_tbl; + + created_at | id_str | id | location | coordinates +---------------------------+--------------------+-----------+----------+------------- + MonJun1002:12:06+00002013 | 343136557777777777 | 377777777 | NearHere | {13,93} + SunJun0912:59:07+00002013 | 343136551111111111 | 311111111 | FarAway | {6,50} +(2 rows) +``` + +View the files added to HDFS: + +``` +$ hdfs dfs -cat /data/pxf_examples/jsopr/* +{"created_at":"SunJun0912:59:07+00002013","id_str":"343136551111111111","id":311111111,"location":"FarAway","coordinates":[6,50]} +{"created_at":"MonJun1002:12:06+00002013","id_str":"343136557777777777","id":377777777,"location":"NearHere","coordinates":[13,93]} +``` + +Notice that PXF creates a flat JSON structure. + +### Example: Single Object Per File (Write) + +Use the following [CREATE EXTERNAL TABLE](https://docs.vmware.com/en/VMware-Greenplum/6/greenplum-database/ref_guide-sql_commands-CREATE_EXTERNAL_TABLE.html) SQL command to create a writable external table that writes JSON data in single object format and uses the PXF default server. + +You must specify the `ROOT` keyword and associated value in the `LOCATION` clause. For example: + +``` sql +CREATE WRITABLE EXTERNAL TABLE add_singleobj_json_tbl( + created_at TEXT, + id_str TEXT, + id INTEGER, + location TEXT, + coordinates INTEGER[] +) +LOCATION('pxf://data/pxf_examples/jso?PROFILE=hdfs:json&ROOT=root') +FORMAT 'CUSTOM' (FORMATTER='pxfwritable_export'); +``` + +`root` identifies the name of the root attribute of the single object. + +Write data to the table: + +``` sql +INSERT INTO add_singleobj_json_tbl VALUES ( 'SunJun0912:59:07+00002013', '343136551111111111', 311111111, 'FarAway', '{ 6, 50 }' ); +INSERT INTO add_singleobj_json_tbl VALUES ( 'WedJun1212:37:02+00002013', '333333333333333333', 333333333, 'NetherWorld', '{ 9, 63 }' ); +``` + +Read the data that you just wrote. Recall that you must first create a new readable external table: + +``` sql +CREATE EXTERNAL TABLE jso_tbl( + created_at TEXT, + id_str TEXT, + id INTEGER, + location TEXT, + coordinates INTEGER[] +) +LOCATION('pxf://data/pxf_examples/jso?PROFILE=hdfs:json&IDENTIFIER=created_at') +FORMAT 'CUSTOM' (FORMATTER='pxfwritable_import'); + +The column names that you specify in the create command must match those of the writable external table. And recall that to read a JSON file that contains a single object, you must specify the `IDENTIFIER` option. + +Query the table to read the data: + +``` sql +SELECT * FROM jso_tbl; + + created_at | id_str | id | location | coo +rdinates +---------------------------+--------------------+-----------+--------------+---- +--------- + WedJun1212:37:02+00002013 | 333333333333333333 | 333333333 | NetherWorld | {9,63} + SunJun0912:59:07+00002013 | 343136551111111111 | 311111111 | FarAway | {6,50} +(2 rows) +``` + +View the files added to HDFS: + +``` +$ hdfs dfs -cat /data/pxf_examples/jso/* +{"root":[ +{"created_at":"SunJun0912:59:07+00002013","id_str":"343136551111111111","id":311111111,"location":"FarAway","coordinates":[6,50]} +]} +{"root":[ +{"created_at":"WedJun1212:37:02+00002013","id_str":"333333333333333333","id":333333333,"location":"NetherWorld","coordinates":[9,63]} +]} +``` + diff --git a/docs/content/hdfs_orc.html.md.erb b/docs/content/hdfs_orc.html.md.erb index 16f946cf2d..f429ee2be4 100644 --- a/docs/content/hdfs_orc.html.md.erb +++ b/docs/content/hdfs_orc.html.md.erb @@ -217,10 +217,10 @@ Procedure: $ hdfs dfs -put /tmp/sampledata.orc /data/pxf_examples/orc_example/ ``` -1. Log in to the Greenplum Database master host and connect to a database. This command connects to the database named `testdb` as the `gpadmin` user: +1. Log in to the Greenplum Database coordinator host and connect to a database. This command connects to the database named `testdb` as the `gpadmin` user: ``` shell - gpadmin@gpmaster$ psql -d testdb + gpadmin@coordinator$ psql -d testdb ``` 1. Create an external table named `sample_orc` that references the `/data/pxf_examples/orc_example/sampledata.orc` file on HDFS. This command creates the table with the column names specified in the ORC schema, and uses the `default` PXF server: @@ -301,3 +301,27 @@ In this example, you create a writable external table to write some data to the postgres=# SELECT * FROM sample_orc ORDER BY num_orders; ``` +## Understanding Overflow Conditions When Writing Numeric Data + +PXF uses the `HiveDecimal` class to write numeric ORC data. In versions prior to 6.7.0, PXF limited only the precision of a numeric type to a maximum of 38. In versions 6.7.0 and later, PXF must meet both precision and scale requirements before writing numeric ORC data. + +When you define a `NUMERIC` column in an external table without specifying a precision or scale, PXF internally maps the column to a `DECIMAL(38, 10)`. + +PXF handles the following precision overflow conditions: + +- You define a `NUMERIC` column in the external table, and the integer digit count of a value exceeds the maximum supported precision of 38. For example, `1234567890123456789012345678901234567890.12345`, which has an integer digit count of 45. +- You define a `NUMERIC()` column with a `` greater than 38. For example, `NUMERIC(55)`. +- You define a `NUMERIC` column in the external table, and the integer digit count of a value is greater than 28 (38-10). For example, `123456789012345678901234567890.12345`, which has an integer digit count of 30. + +If you define a `NUMERIC(, )` column and the integer digit count of a value is greater than ` - `, PXF returns an error. For example, you define a `NUMERIC(20,4)` column and the value is `12345678901234567.12`, which has an integer digit count of 19, which is greater than 20-4=16. + +PXF can take one of three actions when it detects an overflow while writing numeric data to an ORC file: round the value (the default), return an error, or ignore the overflow. The `pxf.orc.write.decimal.overflow` property in the `pxf-site.xml` server configuration governs PXF's action in this circumstance; valid values for this property follow: + +| Value | PXF Action | +|-------|-------------------------------------| +| `round` | When PXF encounters an overflow, it attempts to round the value to meet both precision and scale requirements before writing. PXF reports an error if rounding fails. This may potentially leave an incomplete data set in the external system. `round` is the default. | +| `error` | PXF reports an error when it encounters an overflow, and the transaction fails. | +| `ignore` | PXF attempts to round the value to meet only the precision requirement and ignores validation of precision and scale; otherwise PXF writes a NULL value. (This was PXF's behavior prior to version 6.7.0.) | + +PXF logs a warning when it detects an overflow and the `pxf.orc.write.decimal.overflow` property is set to `ignore`. + diff --git a/docs/content/hdfs_parquet.html.md.erb b/docs/content/hdfs_parquet.html.md.erb index d45be2472c..26ee4817e9 100644 --- a/docs/content/hdfs_parquet.html.md.erb +++ b/docs/content/hdfs_parquet.html.md.erb @@ -251,19 +251,21 @@ PXF uses the `HiveDecimal` class to write numeric Parquet data. `HiveDecimal` li When you define a `NUMERIC` column in an external table without specifying a precision or scale, PXF internally maps the column to a `DECIMAL(38, 18)`. -A precision overflow condition can result when: +PXF handles the following precision overflow conditions: -- You define a `NUMERIC` column in the external table, and the integer digit count of a value exceeds maximum precision 38. For example, `1234567890123456789012345678901234567890.12345`, which has an integer digit count of 45. +- You define a `NUMERIC` column in the external table, and the integer digit count of a value exceeds the maximum supported precision of 38. For example, `1234567890123456789012345678901234567890.12345`, which has an integer digit count of 45. - You define a `NUMERIC()` column with a `` greater than 38. For example, `NUMERIC(55)`. -- You define a `NUMERIC(, )` column and the integer digit count of a value is greater than ` - `. For example, you define a `NUMERIC(20,4)` column and the value is `12345678901234567.12`, which has an integer digit count of 19, which is greater than 20-4=16. +- You define a `NUMERIC` column in the external table, and the integer digit count of a value is greater than 20 (38-18). For example, `123456789012345678901234567890.12345`, which has an integer digit count of 30. + +If you define a `NUMERIC(, )` column and the integer digit count of a value is greater than ` - `, PXF returns an error. For example, you define a `NUMERIC(20,4)` column and the value is `12345678901234567.12`, which has an integer digit count of 19, which is greater than 20-4=16. PXF can take one of three actions when it detects an overflow while writing numeric data to a Parquet file: round the value (the default), return an error, or ignore the overflow. The `pxf.parquet.write.decimal.overflow` property in the `pxf-site.xml` server configuration governs PXF's action in this circumstance; valid values for this property follow: | Value | PXF Action | |-------|-------------------------------------| -| `round` | When PXF encounters an overflow, it attempts to round the value before writing and logs a warning. PXF reports an error if rounding fails. This may potentially leave an incomplete data set in the external system. `round` is the default. | +| `round` | When PXF encounters an overflow, it attempts to round the value to meet both precision and scale requirements before writing. PXF reports an error if rounding fails. This may potentially leave an incomplete data set in the external system. `round` is the default. | | `error` | PXF reports an error when it encounters an overflow, and the transaction fails. | -| `ignore` | PXF writes a NULL value. (This was PXF's behavior prior to version 6.6.0.) | +| `ignore` | PXF attempts to round the value to meet both precision and scale requirements; otherwise PXF writes a NULL value. (This was PXF's behavior prior to version 6.6.0.) | -PXF always logs an warning when it detects an overflow, regardless of the `pxf.parquet.write.decimal.overflow` property setting. +PXF logs a warning when it detects an overflow and the `pxf.parquet.write.decimal.overflow` property is set to `ignore`. diff --git a/docs/content/hdfs_seqfile.html.md.erb b/docs/content/hdfs_seqfile.html.md.erb index 55cd8bdb74..37ebe47c2e 100644 --- a/docs/content/hdfs_seqfile.html.md.erb +++ b/docs/content/hdfs_seqfile.html.md.erb @@ -202,28 +202,28 @@ Perform the following procedure to create the Java class and writable table. (Your Hadoop library classpath may differ.) -3. Copy the `pxfex-customwritable.jar` file to the Greenplum Database master host. For example: +3. Copy the `pxfex-customwritable.jar` file to the Greenplum Database coordinator host. For example: ``` shell - $ scp pxfex-customwritable.jar gpadmin@gpmaster:/home/gpadmin + $ scp pxfex-customwritable.jar gpadmin@coordinator:/home/gpadmin ``` -4. Log in to your Greenplum Database master host: +4. Log in to your Greenplum Database coordinator host: ``` shell - $ ssh gpadmin@ + $ ssh gpadmin@ ```` 5. Copy the `pxfex-customwritable.jar` JAR file to the user runtime library directory, and note the location. For example, if `PXF_BASE=/usr/local/pxf-gp6`: ``` shell - gpadmin@gpmaster$ cp /home/gpadmin/pxfex-customwritable.jar /usr/local/pxf-gp6/lib/pxfex-customwritable.jar + gpadmin@coordinator$ cp /home/gpadmin/pxfex-customwritable.jar /usr/local/pxf-gp6/lib/pxfex-customwritable.jar ``` 5. Synchronize the PXF configuration to the Greenplum Database cluster: ``` shell - gpadmin@gpmaster$ pxf cluster sync + gpadmin@coordinator$ pxf cluster sync ``` 3. Restart PXF on each Greenplum Database host as described in [Restarting PXF](cfginitstart_pxf.html#restart_pxf). diff --git a/docs/content/hdfs_text.html.md.erb b/docs/content/hdfs_text.html.md.erb index 1787733a3d..b5ca885b2c 100644 --- a/docs/content/hdfs_text.html.md.erb +++ b/docs/content/hdfs_text.html.md.erb @@ -247,12 +247,11 @@ The specific keywords and values used in the [CREATE EXTERNAL TABLE](https://doc | delimiter | The delimiter character in the data. For `FORMAT` `'CSV'`, the default \ is a comma (`,`). Preface the \ with an `E` when the value is an escape sequence. Examples: `(delimiter=E'\t')`, `(delimiter ':')`. | | DISTRIBUTED BY | If you want to load data from an existing Greenplum Database table into the writable external table, consider specifying the same distribution policy or `` on both tables. Doing so will avoid extra motion of data between segments on the load operation. | -Writable external tables that you create using the `hdfs:text` or the `hdfs:csv` profiles can optionally use record or block compression. You specify the compression type and codec via custom options in the `CREATE EXTERNAL TABLE` `LOCATION` clause. The `hdfs:text` and `hdfs:csv` profiles support the following custom write options: +Writable external tables that you create using the `hdfs:text` or the `hdfs:csv` profiles can optionally use record or block compression. You specify the compression codec via a custom option in the `CREATE EXTERNAL TABLE` `LOCATION` clause. The `hdfs:text` and `hdfs:csv` profiles support the following custom write option: | Option | Value Description | |-------|-------------------------------------| | COMPRESSION_CODEC | The compression codec alias. Supported compression codecs for writing text data include: `default`, `bzip2`, `gzip`, and `uncompressed`. If this option is not provided, Greenplum Database performs no data compression. | -| COMPRESSION_TYPE | The compression type to employ; supported values are `RECORD` (the default) or `BLOCK`. | ### Example: Writing Text Data to HDFS @@ -365,3 +364,148 @@ Perform the following procedure to create Greenplum Database writable external t To query data from the newly-created HDFS directory named `pxfwritable_hdfs_textsimple2`, you can create a readable external Greenplum Database table as described above that references this HDFS directory and specifies `FORMAT 'CSV' (delimiter=':')`. +## About Setting the External Table Encoding + +When the external file encoding differs from the database encoding, you must set the external table `ENCODING` to match that of the data file. For example, if the database encoding is `UTF8` and the file encoding is `LATIN1`, create the external table as follows: + +``` +CREATE EXTERNAL TABLE pxf_csv_latin1(location text, month text, num_orders int, total_sales float8) + LOCATION ('pxf://data/pxf_examples/pxf_hdfs_simple.txt?PROFILE=hdfs:csv') +FORMAT 'CSV' ENCODING 'LATIN1'; +``` + +## About Reading Data Containing Multi-Byte or Multi-Character Delimiters + +You can use only a `*:csv` PXF profile to read data that contains a multi-byte delimiter or multiple delimiter characters. The syntax for creating a readable external table for such data follows: + +``` sql +CREATE EXTERNAL TABLE + ( [, ...] | LIKE ) +LOCATION ('pxf://?PROFILE=hdfs:csv[&SERVER=][&IGNORE_MISSING_PATH=][&SKIP_HEADER_COUNT=][&NEWLINE=]') +FORMAT 'CUSTOM' (FORMATTER='pxfdelimited_import'