Skip to content

[native] Fail-fast for file formats unsupported by hive connector #25147

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
+35 −8
Open

[native] Fail-fast for file formats unsupported by hive connector #25147

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion presto-hive-hadoop2/src/test/java/com/facebook/presto/hive/s3select/S3SelectTestHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@
import com.facebook.presto.spi.connector.ConnectorPageSourceProvider;
import com.facebook.presto.spi.connector.ConnectorSplitManager;
import com.facebook.presto.testing.MaterializedResult;
import com.facebook.presto.testing.TestingConnectorContext;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.net.HostAndPort;
Expand Down Expand Up @@ -199,7 +200,8 @@ public S3SelectTestHelper(String host,
config.getRecursiveDirWalkerEnabled(),
new ConfigBasedCacheQuotaRequirementProvider(cacheConfig),
new HiveEncryptionInformationProvider(ImmutableSet.of()),
new HivePartitionSkippabilityChecker());
new HivePartitionSkippabilityChecker(),
new TestingConnectorContext().getConnectorSystemConfig());
pageSourceProvider = new HivePageSourceProvider(
config,
hdfsEnvironment,
Expand Down
24 changes: 21 additions & 3 deletions presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import com.facebook.presto.spi.ColumnHandle;
import com.facebook.presto.spi.ConnectorSession;
import com.facebook.presto.spi.ConnectorSplitSource;
import com.facebook.presto.spi.ConnectorSystemConfig;
import com.facebook.presto.spi.ConnectorTableLayoutHandle;
import com.facebook.presto.spi.FixedSplitSource;
import com.facebook.presto.spi.PrestoException;
Expand Down Expand Up @@ -96,6 +97,8 @@
import static com.facebook.presto.hive.HiveSessionProperties.isDynamicSplitSizesEnabled;
import static com.facebook.presto.hive.HiveSessionProperties.isOfflineDataDebugModeEnabled;
import static com.facebook.presto.hive.HiveSessionProperties.isPartitionStatisticsBasedOptimizationEnabled;
import static com.facebook.presto.hive.HiveStorageFormat.DWRF;
import static com.facebook.presto.hive.HiveStorageFormat.ORC;
import static com.facebook.presto.hive.HiveStorageFormat.PARQUET;
import static com.facebook.presto.hive.HiveStorageFormat.getHiveStorageFormat;
import static com.facebook.presto.hive.HiveType.getPrimitiveType;
Expand Down Expand Up @@ -151,6 +154,7 @@ public class HiveSplitManager
private final CacheQuotaRequirementProvider cacheQuotaRequirementProvider;
private final HiveEncryptionInformationProvider encryptionInformationProvider;
private final PartitionSkippabilityChecker partitionSkippabilityChecker;
private final ConnectorSystemConfig connectorSystemConfig;

@Inject
public HiveSplitManager(
Expand All @@ -163,7 +167,8 @@ public HiveSplitManager(
@ForHiveClient ExecutorService executorService,
CoercionPolicy coercionPolicy,
HiveEncryptionInformationProvider encryptionInformationProvider,
PartitionSkippabilityChecker partitionSkippabilityChecker)
PartitionSkippabilityChecker partitionSkippabilityChecker,
ConnectorSystemConfig connectorSystemConfig)
{
this(
hiveTransactionManager,
Expand All @@ -181,7 +186,8 @@ public HiveSplitManager(
hiveClientConfig.getRecursiveDirWalkerEnabled(),
cacheQuotaRequirementProvider,
encryptionInformationProvider,
partitionSkippabilityChecker);
partitionSkippabilityChecker,
connectorSystemConfig);
}

public HiveSplitManager(
Expand All @@ -200,7 +206,8 @@ public HiveSplitManager(
boolean recursiveDfsWalkerEnabled,
CacheQuotaRequirementProvider cacheQuotaRequirementProvider,
HiveEncryptionInformationProvider encryptionInformationProvider,
PartitionSkippabilityChecker partitionSkippabilityChecker)
PartitionSkippabilityChecker partitionSkippabilityChecker,
ConnectorSystemConfig connectorSystemConfig)
{
this.hiveTransactionManager = requireNonNull(hiveTransactionManager, "hiveTransactionManager is null");
this.namenodeStats = requireNonNull(namenodeStats, "namenodeStats is null");
Expand All @@ -219,6 +226,7 @@ public HiveSplitManager(
this.cacheQuotaRequirementProvider = requireNonNull(cacheQuotaRequirementProvider, "cacheQuotaRequirementProvider is null");
this.encryptionInformationProvider = requireNonNull(encryptionInformationProvider, "encryptionInformationProvider is null");
this.partitionSkippabilityChecker = requireNonNull(partitionSkippabilityChecker, "partitionSkippabilityChecker is null");
this.connectorSystemConfig = requireNonNull(connectorSystemConfig, "connectorSystemConfig is null");
}

@Override
Expand Down Expand Up @@ -250,6 +258,16 @@ public ConnectorSplitSource getSplits(
session.getRuntimeStats());
Table table = layout.getTable(metastore, metastoreContext);

if (connectorSystemConfig.isNativeExecution()) {
StorageFormat storageFormat = table.getStorage().getStorageFormat();
Optional<HiveStorageFormat> hiveStorageFormat = getHiveStorageFormat(storageFormat);
if (hiveStorageFormat.isPresent() && !(hiveStorageFormat.equals(Optional.of(DWRF))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's add this in the Hive configs. By default, it is empty, which means whatever is available in Hive is fine. It can be a set of comma separated values.

|| hiveStorageFormat.equals(Optional.of(ORC)) || hiveStorageFormat.equals(Optional.of(PARQUET)))) {
throw new HiveNotReadableException(tableName, Optional.empty(),
format("FileFormat %s unsupported in Presto C++.", hiveStorageFormat.get()));
}
}

if (!isOfflineDataDebugModeEnabled(session)) {
// verify table is not marked as non-readable
String tableNotReadable = table.getParameters().get(OBJECT_NOT_READABLE);
Expand Down
4 changes: 3 additions & 1 deletion presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,7 @@
import com.facebook.presto.sql.planner.TypeProvider;
import com.facebook.presto.testing.MaterializedResult;
import com.facebook.presto.testing.MaterializedRow;
import com.facebook.presto.testing.TestingConnectorContext;
import com.facebook.presto.testing.TestingConnectorSession;
import com.facebook.presto.testing.TestingNodeManager;
import com.google.common.base.Predicate;
Expand Down Expand Up @@ -1067,7 +1068,8 @@ protected final void setup(String databaseName, HiveClientConfig hiveClientConfi
false,
new ConfigBasedCacheQuotaRequirementProvider(cacheConfig),
encryptionInformationProvider,
new HivePartitionSkippabilityChecker());
new HivePartitionSkippabilityChecker(),
new TestingConnectorContext().getConnectorSystemConfig());
pageSinkProvider = new HivePageSinkProvider(
getDefaultHiveFileWriterFactories(hiveClientConfig, metastoreClientConfig),
hdfsEnvironment,
Expand Down
4 changes: 3 additions & 1 deletion presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveFileSystem.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@
import com.facebook.presto.spi.security.ConnectorIdentity;
import com.facebook.presto.sql.gen.JoinCompiler;
import com.facebook.presto.testing.MaterializedResult;
import com.facebook.presto.testing.TestingConnectorContext;
import com.facebook.presto.testing.TestingConnectorSession;
import com.facebook.presto.testing.TestingNodeManager;
import com.google.common.collect.ImmutableList;
Expand Down Expand Up @@ -247,7 +248,8 @@ protected void setup(String host, int port, String databaseName, BiFunction<Hive
config.getRecursiveDirWalkerEnabled(),
new ConfigBasedCacheQuotaRequirementProvider(cacheConfig),
new HiveEncryptionInformationProvider(ImmutableSet.of()),
new HivePartitionSkippabilityChecker());
new HivePartitionSkippabilityChecker(),
new TestingConnectorContext().getConnectorSystemConfig());
pageSinkProvider = new HivePageSinkProvider(
getDefaultHiveFileWriterFactories(config, metastoreClientConfig),
hdfsEnvironment,
Expand Down
7 changes: 5 additions & 2 deletions presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSplitManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import com.facebook.presto.spi.WarningCollector;
import com.facebook.presto.spi.connector.ConnectorSplitManager.SplitSchedulingContext;
import com.facebook.presto.spi.connector.ConnectorTransactionHandle;
import com.facebook.presto.testing.TestingConnectorContext;
import com.facebook.presto.testing.TestingConnectorSession;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
Expand Down Expand Up @@ -552,7 +553,8 @@ private void assertRedundantColumnDomains(Range predicateRange, PartitionStatist
false,
new ConfigBasedCacheQuotaRequirementProvider(new CacheConfig()),
new HiveEncryptionInformationProvider(ImmutableList.of()),
new HivePartitionSkippabilityChecker());
new HivePartitionSkippabilityChecker(),
new TestingConnectorContext().getConnectorSystemConfig());

HiveColumnHandle partitionColumn = new HiveColumnHandle(
"ds",
Expand Down Expand Up @@ -700,7 +702,8 @@ public void testEncryptionInformation()
false,
new ConfigBasedCacheQuotaRequirementProvider(new CacheConfig()),
encryptionInformationProvider,
new HivePartitionSkippabilityChecker());
new HivePartitionSkippabilityChecker(),
new TestingConnectorContext().getConnectorSystemConfig());

HiveColumnHandle partitionColumn = new HiveColumnHandle(
"ds",
Expand Down
Loading