Skip to content

Commit

Permalink
Merge pull request #101 from arenadata/feature/ADBDEV-5743
Browse files Browse the repository at this point in the history
ADBDEV-5743: [Java] Extend support for year > 4 digits and BC/AD dating notation
  • Loading branch information
iamlapa authored Jul 12, 2024
2 parents 6933f5e + 90594d5 commit b43cf67
Show file tree
Hide file tree
Showing 23 changed files with 602 additions and 364 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -401,18 +401,35 @@ public void afterPropertiesSet() {
poolQualifier = configuration.get(JDBC_POOL_QUALIFIER_PROPERTY_NAME);
}

// Optional parameter to determine if the year might contain more than 4 digits in `date` or 'timestamp'.
// The default value is false.
// We need to check the legacy parameter name for backward compatability with the open source project
String dateWideRangeConfig = configuration.get(JDBC_DATE_WIDE_RANGE_LEGACY);
isDateWideRange = getIsDateWideRange(context);
}

/**
* Determine if the year might contain more than 4 digits in 'date' or 'timestamp' using the legacy parameter name.
* Optional parameter. The default value is false.
* We need to check the legacy parameter name for backward compatability with the open source project
*
* @param context To get "jdbc.date.wide-range" parameter
* @return true if the year might contain more than 4 digits
*/
public static boolean getIsDateWideRange(RequestContext context) {
Configuration configuration = context.getConfiguration();
String dateWideRangeConfig = configuration != null ? configuration.get(JDBC_DATE_WIDE_RANGE_LEGACY) : null;
String dateWideRangeContext = context.getOption(JDBC_DATE_WIDE_RANGE_LEGACY);
if (Objects.nonNull(dateWideRangeContext) || Objects.nonNull(dateWideRangeConfig)) {
log.warn("'{}' is a deprecated name of the parameter. Use 'date_wide_range' in the external table definition or " +
"'{}' in the jdbc-site.xml configuration file", JDBC_DATE_WIDE_RANGE_LEGACY, JDBC_DATE_WIDE_RANGE);
isDateWideRange = isDateWideRange(dateWideRangeContext);
} else {
isDateWideRange = configuration.getBoolean(JDBC_DATE_WIDE_RANGE, false);
log.warn(
"'{}' is a deprecated name of the parameter. Use 'date_wide_range' in the external table definition or " +
"'{}' in the jdbc-site.xml configuration file",
JDBC_DATE_WIDE_RANGE_LEGACY,
JDBC_DATE_WIDE_RANGE
);
if (Objects.nonNull(dateWideRangeContext)) {
return Boolean.parseBoolean(dateWideRangeContext);
} else {
return configuration.getBoolean(JDBC_DATE_WIDE_RANGE_LEGACY, false);
}
}
return configuration != null && configuration.getBoolean(JDBC_DATE_WIDE_RANGE, false);
}

/**
Expand Down Expand Up @@ -660,18 +677,4 @@ private Map<String, String> getPropsWithPrefix(Configuration configuration, Stri
}
return configMap;
}

/**
* Determine if the year might contain more than 4 digits in 'date' or 'timestamp' using the legacy parameter name.
*
* @param dateWideRangeContext value of the parameter from the context
* @return true if the year might contain more than 4 digits
*/
private boolean isDateWideRange(String dateWideRangeContext) {
if (Objects.nonNull(dateWideRangeContext)) {
return Boolean.parseBoolean(dateWideRangeContext);
} else {
return configuration.getBoolean(JDBC_DATE_WIDE_RANGE_LEGACY, false);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ public class JdbcPartitionFragmenter extends BaseFragmenter {
private String column;
private String range;
private String interval;
protected boolean isDateWideRange;

@Override
public void afterPropertiesSet() {
Expand All @@ -54,6 +55,7 @@ public void afterPropertiesSet() {

range = context.getOption("RANGE");
interval = context.getOption("INTERVAL");
isDateWideRange = JdbcBasePlugin.getIsDateWideRange(context);
}

/**
Expand All @@ -67,7 +69,9 @@ public List<Fragment> getFragments() {
if (partitionType == null) {
fragments.add(new Fragment(context.getDataSource()));
} else {
List<JdbcFragmentMetadata> fragmentsMetadata = partitionType.getFragmentsMetadata(column, range, interval);
List<JdbcFragmentMetadata> fragmentsMetadata = partitionType.getFragmentsMetadata(
column, range, interval, isDateWideRange
);
for (JdbcFragmentMetadata fragmentMetadata : fragmentsMetadata) {
fragments.add(new Fragment(context.getDataSource(), fragmentMetadata));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
* under the License.
*/

import org.greenplum.pxf.api.GreenplumDateTime;
import io.arenadata.security.encryption.client.service.DecryptClient;
import org.greenplum.pxf.api.OneField;
import org.greenplum.pxf.api.OneRow;
Expand All @@ -42,87 +41,24 @@
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeFormatterBuilder;
import java.time.format.SignStyle;
import java.time.temporal.ChronoField;
import java.util.EnumSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.UUID;

import static java.time.format.DateTimeFormatter.ISO_LOCAL_TIME;
import static java.time.format.DateTimeFormatter.ISO_OFFSET_TIME;
import static org.greenplum.pxf.api.GreenplumDateTime.DATETIME_FORMATTER;
import static org.greenplum.pxf.api.GreenplumDateTime.DATE_FORMATTER;
import static org.greenplum.pxf.plugins.jdbc.utils.DateTimeEraFormatters.LOCAL_DATE_FORMATTER;
import static org.greenplum.pxf.plugins.jdbc.utils.DateTimeEraFormatters.LOCAL_DATE_TIME_FORMATTER;
import static org.greenplum.pxf.plugins.jdbc.utils.DateTimeEraFormatters.OFFSET_DATE_TIME_FORMATTER;
import static org.greenplum.pxf.plugins.jdbc.utils.DateTimeEraFormatters.getLocalDate;
import static org.greenplum.pxf.plugins.jdbc.utils.DateTimeEraFormatters.getLocalDateTime;

/**
* JDBC tables resolver
*/
public class JdbcResolver extends JdbcBasePlugin implements Resolver {
// Signifies the ERA format
final private static String DATE_TIME_FORMATTER_SPECIFIER = " G";
/**
* LOCAL_DATE_GET_FORMATTER is used to format LocalDate to String.
* Examples: 2023-01-10 -> "2023-01-10 AD"; +12345-02-01 -> "12345-02-01 AD"; -0009-12-11 -> "0010-12-11 BC"
*/
private static final DateTimeFormatter LOCAL_DATE_GET_FORMATTER = (new DateTimeFormatterBuilder())
.appendValue(ChronoField.YEAR_OF_ERA, 4, 9, SignStyle.NORMAL).appendLiteral("-")
.appendValue(ChronoField.MONTH_OF_YEAR, 2).appendLiteral('-')
.appendValue(ChronoField.DAY_OF_MONTH, 2)
.appendPattern(DATE_TIME_FORMATTER_SPECIFIER)
.toFormatter();

/**
* LOCAL_DATE_TIME_GET_FORMATTER is used to format LocalDateTime to String.
* Examples: 2018-10-19T10:11 -> "2018-10-19 10:11:00 AD"; +123456-10-19T11:12:13 -> "123456-10-19 11:12:13 AD";
* -1233-10-19T10:11:15.456 -> "1234-10-19 10:11:15.456 BC"
*/
private static final DateTimeFormatter LOCAL_DATE_TIME_GET_FORMATTER = (new DateTimeFormatterBuilder())
.appendValue(ChronoField.YEAR_OF_ERA, 4, 9, SignStyle.NORMAL).appendLiteral("-")
.appendValue(ChronoField.MONTH_OF_YEAR, 2).appendLiteral('-')
.appendValue(ChronoField.DAY_OF_MONTH, 2).appendLiteral(" ")
.append(ISO_LOCAL_TIME)
.appendPattern(DATE_TIME_FORMATTER_SPECIFIER)
.toFormatter();


/**
* OFFSET_DATE_TIME_GET_FORMATTER is used to format OffsetDateTime to String.
* Examples: 1956-02-01T07:15:16Z -> "1956-02-01 07:15:16Z AD"; +12345-02-01T10:15:16Z -> "12345-02-01 10:15:16Z AD";
* -1999-02-01T04:15:16Z -> "2000-02-01 04:15:16Z BC"
*/
private static final DateTimeFormatter OFFSET_DATE_TIME_GET_FORMATTER = (new DateTimeFormatterBuilder())
.appendValue(ChronoField.YEAR_OF_ERA, 4, 9, SignStyle.NORMAL).appendLiteral("-")
.appendValue(ChronoField.MONTH_OF_YEAR, 2).appendLiteral('-')
.appendValue(ChronoField.DAY_OF_MONTH, 2).appendLiteral(" ")
.append(ISO_OFFSET_TIME)
.appendPattern(DATE_TIME_FORMATTER_SPECIFIER)
.toFormatter();

/**
* LOCAL_DATE_SET_FORMATTER is used to format String to LocalDate.
* Examples: "1977-12-11" -> 1977-12-11; "456789-12-11" -> +456789-12-11; "0010-12-11 BC" -> -0009-12-11
*/
private static final DateTimeFormatter LOCAL_DATE_SET_FORMATTER = (new DateTimeFormatterBuilder())
.appendValue(ChronoField.YEAR_OF_ERA, 1, 9, SignStyle.NORMAL).appendLiteral('-')
.appendValue(ChronoField.MONTH_OF_YEAR, 2).appendLiteral('-')
.appendValue(ChronoField.DAY_OF_MONTH, 2)
.optionalStart().appendPattern(DATE_TIME_FORMATTER_SPECIFIER).optionalEnd()
.toFormatter();

/**
* LOCAL_DATE_TIME_SET_FORMATTER is used to transfer String to LocalDateTime.
* Examples: "1980-08-10 17:10:20" -> 1980-08-10T17:10:20; "123456-10-19 11:12:13" -> +123456-10-19T11:12:13;
* "1234-10-19 10:11:15.456 BC" -> -1233-10-19T10:11:15.456
*/
private static final DateTimeFormatter LOCAL_DATE_TIME_SET_FORMATTER = (new DateTimeFormatterBuilder())
.appendValue(ChronoField.YEAR_OF_ERA, 1, 9, SignStyle.NORMAL).appendLiteral('-')
.appendValue(ChronoField.MONTH_OF_YEAR, 1, 2, SignStyle.NORMAL).appendLiteral('-')
.appendValue(ChronoField.DAY_OF_MONTH, 1, 2, SignStyle.NORMAL).appendLiteral(" ")
.append(ISO_LOCAL_TIME)
.optionalStart().appendPattern(DATE_TIME_FORMATTER_SPECIFIER).optionalEnd()
.toFormatter();

private static final Set<DataType> DATATYPES_SUPPORTED = EnumSet.of(
DataType.VARCHAR,
DataType.BPCHAR,
Expand Down Expand Up @@ -216,25 +152,25 @@ public List<OneField> getFields(OneRow row) throws SQLException {
case DATE:
if (isDateWideRange) {
LocalDate localDate = result.getObject(colName, LocalDate.class);
value = localDate != null ? localDate.format(LOCAL_DATE_GET_FORMATTER) : null;
value = localDate != null ? localDate.format(LOCAL_DATE_FORMATTER) : null;
} else {
Date date = result.getDate(colName);
value = date != null ? date.toLocalDate().format(GreenplumDateTime.DATE_FORMATTER) : null;
value = date != null ? date.toLocalDate().format(DATE_FORMATTER) : null;
}
break;
case TIMESTAMP:
if (isDateWideRange) {
LocalDateTime localDateTime = result.getObject(colName, LocalDateTime.class);
value = localDateTime != null ? localDateTime.format(LOCAL_DATE_TIME_GET_FORMATTER) : null;
value = localDateTime != null ? localDateTime.format(LOCAL_DATE_TIME_FORMATTER) : null;
} else {
Timestamp timestamp = result.getTimestamp(colName);
value = timestamp != null ? timestamp.toLocalDateTime().format(GreenplumDateTime.DATETIME_FORMATTER) : null;
value = timestamp != null ? timestamp.toLocalDateTime().format(DATETIME_FORMATTER) : null;
}
break;
case TIMESTAMP_WITH_TIME_ZONE:
if (isDateWideRange) {
OffsetDateTime offsetDateTime = result.getObject(colName, OffsetDateTime.class);
value = offsetDateTime != null ? offsetDateTime.format(OFFSET_DATE_TIME_GET_FORMATTER) : null;
value = offsetDateTime != null ? offsetDateTime.format(OFFSET_DATE_TIME_FORMATTER) : null;
} else {
throw new UnsupportedOperationException(
String.format("Field type '%s' (column '%s') is not supported",
Expand Down Expand Up @@ -468,32 +404,4 @@ public static void decodeOneRowToPreparedStatement(OneRow row, PreparedStatement
}
}
}

/**
* Convert a string to LocalDate class with formatter
*
* @param rawVal the LocalDate in a string format
* @return LocalDate
*/
private LocalDate getLocalDate(String rawVal) {
try {
return LocalDate.parse(rawVal, LOCAL_DATE_SET_FORMATTER);
} catch (Exception e) {
throw new IllegalArgumentException("Failed to convert date '" + rawVal + "' to LocalDate class: " + e.getMessage(), e);
}
}

/**
* Convert a string to LocalDateTime class with formatter
*
* @param rawVal the LocalDateTime in a string format
* @return LocalDateTime
*/
private LocalDateTime getLocalDateTime(String rawVal) {
try {
return LocalDateTime.parse(rawVal, LOCAL_DATE_TIME_SET_FORMATTER);
} catch (Exception e) {
throw new IllegalArgumentException("Failed to convert timestamp '" + rawVal + "' to the LocalDateTime class: " + e.getMessage(), e);
}
}
}
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package org.greenplum.pxf.plugins.jdbc.partitioning;

import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;

Expand All @@ -10,39 +9,20 @@
* <p>
* All partitions use some column as a partition column. It is processed by this class.
*/
@NoArgsConstructor
@Getter
@RequiredArgsConstructor
public abstract class BasePartition implements JdbcFragmentMetadata {

/**
* Column name to use as a partition column. Must not be null
*/
@Getter
@NonNull
protected String column;
protected final String column;

/**
* Generate a range-based SQL constraint
*
* @param quotedColumn column name (used as is, thus it should be quoted if necessary)
* @param range range to base constraint on
* @return a pure SQL constraint (without WHERE)
*/
String generateRangeConstraint(String quotedColumn, String[] range) {
StringBuilder sb = new StringBuilder(quotedColumn);

if (range.length == 1) {
sb.append(" = ").append(range[0]);
} else if (range[0] == null) {
sb.append(" < ").append(range[1]);
} else if (range[1] == null) {
sb.append(" >= ").append(range[0]);
} else {
sb.append(" >= ").append(range[0])
.append(" AND ")
.append(quotedColumn).append(" < ").append(range[1]);
protected String getQuotedColumn(String quoteString) {
if (quoteString == null) {
throw new RuntimeException("Quote string cannot be null");
}

return sb.toString();
return quoteString + column + quoteString;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
package org.greenplum.pxf.plugins.jdbc.partitioning;

import lombok.NonNull;

/**
* A base class for partition of any type.
* <p>
* All partitions use some column as a partition column. It is processed by this class.
*/
public abstract class BaseRangePartition extends BasePartition {
public BaseRangePartition(@NonNull String column) {
super(column);
}

/**
* Generate a range-based SQL constraint
*
* @param quotedColumn column name (used as is, thus it should be quoted if necessary)
* @param start range start to base constraint on
* @param end range end to base constraint on
* @return a pure SQL constraint (without WHERE)
*/
String generateRangeConstraint(String quotedColumn, String start, String end) {
StringBuilder sb = new StringBuilder(quotedColumn);

if (start == null) {
sb.append(" < ").append(end);
} else if (end == null) {
sb.append(" >= ").append(start);
} else {
sb.append(" >= ").append(start)
.append(" AND ")
.append(quotedColumn).append(" < ").append(end);
}

return sb.toString();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
package org.greenplum.pxf.plugins.jdbc.partitioning;

import lombok.NonNull;

/**
* A base class for partition of any type.
* <p>
* All partitions use some column as a partition column. It is processed by this class.
*/
public abstract class BaseValuePartition extends BasePartition {
public BaseValuePartition(@NonNull String column) {
super(column);
}

/**
* Generate a range-based SQL constraint
*
* @param quotedColumn column name (used as is, thus it should be quoted if necessary)
* @param value value to base constraint on
* @return a pure SQL constraint (without WHERE)
*/
String generateConstraint(String quotedColumn, String value) {
return quotedColumn + " = " + value;
}
}
Loading

0 comments on commit b43cf67

Please sign in to comment.