diff --git a/presto-docs/src/main/sphinx/admin/properties-session.rst b/presto-docs/src/main/sphinx/admin/properties-session.rst index 77b7e8942ee5c..a98f53fcc554f 100644 --- a/presto-docs/src/main/sphinx/admin/properties-session.rst +++ b/presto-docs/src/main/sphinx/admin/properties-session.rst @@ -581,3 +581,14 @@ The corresponding configuration property is :ref:`admin/properties:\`\`experimen .. warning:: Materialized views are experimental. The SPI and behavior may change in future releases. + +``table_finish_info_json_length_limit`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** ``int`` +* **Default value:** ``10,000,000`` + +The maximum length of the JSON-serialized ConnectorOutputMetadata string in +TableFinishInfo. If the length is exceeded, then the Metadata is omitted. + +When set to a non-positive value, the length limit is not enforced. diff --git a/presto-main-base/src/main/java/com/facebook/presto/SystemSessionProperties.java b/presto-main-base/src/main/java/com/facebook/presto/SystemSessionProperties.java index de115c56da024..94db6097b6cf5 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/SystemSessionProperties.java +++ b/presto-main-base/src/main/java/com/facebook/presto/SystemSessionProperties.java @@ -27,12 +27,12 @@ import com.facebook.presto.execution.warnings.WarningCollectorConfig; import com.facebook.presto.memory.MemoryManagerConfig; import com.facebook.presto.memory.NodeMemoryConfig; +import com.facebook.presto.operator.FeaturesConfig; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.eventlistener.CTEInformation; import com.facebook.presto.spi.security.ViewSecurity; import com.facebook.presto.spi.session.PropertyMetadata; import com.facebook.presto.spiller.NodeSpillConfig; -import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.analyzer.FeaturesConfig.AggregationIfToFilterRewriteStrategy; import com.facebook.presto.sql.analyzer.FeaturesConfig.AggregationPartitioningMergingStrategy; import com.facebook.presto.sql.analyzer.FeaturesConfig.CteMaterializationStrategy; @@ -133,6 +133,7 @@ public final class SystemSessionProperties public static final String SCALE_WRITERS = "scale_writers"; public static final String WRITER_MIN_SIZE = "writer_min_size"; public static final String OPTIMIZED_SCALE_WRITER_PRODUCER_BUFFER = "optimized_scale_writer_producer_buffer"; + public static final String TABLE_FINISH_INFO_JSON_LENGTH_LIMIT = "table_finish_info_json_length_limit"; public static final String PUSH_TABLE_WRITE_THROUGH_UNION = "push_table_write_through_union"; public static final String EXECUTION_POLICY = "execution_policy"; public static final String DICTIONARY_AGGREGATION = "dictionary_aggregation"; @@ -374,7 +375,7 @@ public SystemSessionProperties() new QueryManagerConfig(), new TaskManagerConfig(), new MemoryManagerConfig(), - new FeaturesConfig(), + new com.facebook.presto.sql.analyzer.FeaturesConfig(), new FunctionsConfig(), new NodeMemoryConfig(), new WarningCollectorConfig(), @@ -382,7 +383,8 @@ public SystemSessionProperties() new NodeSpillConfig(), new TracingConfig(), new CompilerConfig(), - new HistoryBasedOptimizationConfig()); + new HistoryBasedOptimizationConfig(), + new FeaturesConfig()); } @Inject @@ -390,7 +392,7 @@ public SystemSessionProperties( QueryManagerConfig queryManagerConfig, TaskManagerConfig taskManagerConfig, MemoryManagerConfig memoryManagerConfig, - FeaturesConfig featuresConfig, + com.facebook.presto.sql.analyzer.FeaturesConfig featuresConfig, FunctionsConfig functionsConfig, NodeMemoryConfig nodeMemoryConfig, WarningCollectorConfig warningCollectorConfig, @@ -398,7 +400,8 @@ public SystemSessionProperties( NodeSpillConfig nodeSpillConfig, TracingConfig tracingConfig, CompilerConfig compilerConfig, - HistoryBasedOptimizationConfig historyBasedOptimizationConfig) + HistoryBasedOptimizationConfig historyBasedOptimizationConfig, + FeaturesConfig tableFinishConfig) { sessionProperties = ImmutableList.of( integerProperty( @@ -560,6 +563,11 @@ public SystemSessionProperties( "Optimize scale writer creation based on producer buffer", featuresConfig.isOptimizedScaleWriterProducerBuffer(), true), + integerProperty( + TABLE_FINISH_INFO_JSON_LENGTH_LIMIT, + "Maximum number of characters in connector output metadata JSON in table finish info", + tableFinishConfig.getTableFinishInfoJsonLengthLimit(), + false), booleanProperty( PUSH_TABLE_WRITE_THROUGH_UNION, "Parallelize writes when using UNION ALL in queries that write data", diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/FeaturesConfig.java b/presto-main-base/src/main/java/com/facebook/presto/operator/FeaturesConfig.java new file mode 100644 index 0000000000000..6c1cfb387281a --- /dev/null +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/FeaturesConfig.java @@ -0,0 +1,37 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator; + +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; +import jakarta.validation.constraints.NotNull; + +public class FeaturesConfig +{ + private int tableFinishInfoJsonLengthLimit = 10_000_000; + + @NotNull + public int getTableFinishInfoJsonLengthLimit() + { + return tableFinishInfoJsonLengthLimit; + } + + @Config("table-finish-info-json-length-limit") + @ConfigDescription("Maximum number of characters in connector output metadata JSON in table finish info") + public FeaturesConfig setTableFinishInfoJsonLengthLimit(int tableFinishInfoJsonLengthLimit) + { + this.tableFinishInfoJsonLengthLimit = tableFinishInfoJsonLengthLimit; + return this; + } +} diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/TableFinishInfo.java b/presto-main-base/src/main/java/com/facebook/presto/operator/TableFinishInfo.java index c4038f20146d8..b911c183ddb83 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/TableFinishInfo.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/TableFinishInfo.java @@ -14,7 +14,6 @@ package com.facebook.presto.operator; import com.facebook.airlift.json.JsonCodec; -import com.facebook.airlift.units.DataSize; import com.facebook.airlift.units.Duration; import com.facebook.drift.annotations.ThriftConstructor; import com.facebook.drift.annotations.ThriftField; @@ -26,15 +25,12 @@ import java.util.Optional; import static com.facebook.airlift.json.JsonCodec.jsonCodec; -import static com.facebook.airlift.units.DataSize.Unit.MEGABYTE; -import static java.lang.Math.toIntExact; import static java.util.Objects.requireNonNull; @ThriftStruct public class TableFinishInfo implements OperatorInfo { - private static final int JSON_LENGTH_LIMIT = toIntExact(new DataSize(10, MEGABYTE).toBytes()); private static final JsonCodec INFO_CODEC = jsonCodec(Object.class); private final String serializedConnectorOutputMetadata; @@ -42,21 +38,16 @@ public class TableFinishInfo private final Duration statisticsWallTime; private final Duration statisticsCpuTime; - public TableFinishInfo(Optional metadata, Duration statisticsWallTime, Duration statisticsCpuTime) + public TableFinishInfo(Optional metadata, Duration statisticsWallTime, Duration statisticsCpuTime, int jsonLengthLimit) { String serializedConnectorOutputMetadata = null; boolean jsonLengthLimitExceeded = false; if (metadata.isPresent()) { - Optional serializedMetadata = INFO_CODEC.toJsonWithLengthLimit(metadata.get().getInfo(), JSON_LENGTH_LIMIT); - if (!serializedMetadata.isPresent()) { - serializedConnectorOutputMetadata = null; - jsonLengthLimitExceeded = true; - } - else { - serializedConnectorOutputMetadata = serializedMetadata.get(); - jsonLengthLimitExceeded = false; - } + Optional serializedMetadata = getSerializedMetadataWithLimit(metadata.get(), jsonLengthLimit); + serializedConnectorOutputMetadata = serializedMetadata.orElse(null); + jsonLengthLimitExceeded = !serializedMetadata.isPresent(); } + this.serializedConnectorOutputMetadata = serializedConnectorOutputMetadata; this.jsonLengthLimitExceeded = jsonLengthLimitExceeded; this.statisticsWallTime = requireNonNull(statisticsWallTime, "statisticsWallTime is null"); @@ -110,4 +101,13 @@ public boolean isFinal() { return true; } + + private static Optional getSerializedMetadataWithLimit(ConnectorOutputMetadata metadata, int jsonLengthLimit) + { + Object metadataInfo = metadata.getInfo(); + if (jsonLengthLimit <= 0) { + return Optional.of(INFO_CODEC.toJson(metadataInfo)); + } + return INFO_CODEC.toJsonWithLengthLimit(metadataInfo, jsonLengthLimit); + } } diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/TableFinishOperator.java b/presto-main-base/src/main/java/com/facebook/presto/operator/TableFinishOperator.java index ddf2c0e102a42..0c04f77e87234 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/TableFinishOperator.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/TableFinishOperator.java @@ -47,6 +47,7 @@ import java.util.function.Supplier; import static com.facebook.airlift.units.Duration.succinctNanos; +import static com.facebook.presto.SystemSessionProperties.TABLE_FINISH_INFO_JSON_LENGTH_LIMIT; import static com.facebook.presto.SystemSessionProperties.isStatisticsCpuTimerEnabled; import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.VarbinaryType.VARBINARY; @@ -114,6 +115,7 @@ public Operator createOperator(DriverContext driverContext) OperatorContext context = driverContext.addOperatorContext(operatorId, planNodeId, TableFinishOperator.class.getSimpleName()); Operator statisticsAggregationOperator = statisticsAggregationOperatorFactory.createOperator(driverContext); boolean statisticsCpuTimerEnabled = !(statisticsAggregationOperator instanceof DevNullOperator) && isStatisticsCpuTimerEnabled(session); + int jsonLengthLimit = session.getSystemProperty(TABLE_FINISH_INFO_JSON_LENGTH_LIMIT, Integer.class); return new TableFinishOperator( context, tableFinisher, @@ -122,7 +124,8 @@ public Operator createOperator(DriverContext driverContext) descriptor, statisticsCpuTimerEnabled, memoryTrackingEnabled, - tableCommitContextCodec); + tableCommitContextCodec, + jsonLengthLimit); } @Override @@ -156,6 +159,7 @@ private enum State private final TableFinisher tableFinisher; private final Operator statisticsAggregationOperator; private final StatisticAggregationsDescriptor descriptor; + private final int jsonLengthLimit; private State state = State.RUNNING; private final AtomicReference> outputMetadata = new AtomicReference<>(Optional.empty()); @@ -181,18 +185,20 @@ public TableFinishOperator( StatisticAggregationsDescriptor descriptor, boolean statisticsCpuTimerEnabled, boolean memoryTrackingEnabled, - JsonCodec tableCommitContextCodec) + JsonCodec tableCommitContextCodec, + int jsonLengthLimit) { this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); this.tableFinisher = requireNonNull(tableFinisher, "tableCommitter is null"); this.statisticsAggregationOperator = requireNonNull(statisticsAggregationOperator, "statisticsAggregationOperator is null"); this.descriptor = requireNonNull(descriptor, "descriptor is null"); + this.jsonLengthLimit = jsonLengthLimit; this.statisticsCpuTimerEnabled = statisticsCpuTimerEnabled; this.memoryTrackingEnabled = memoryTrackingEnabled; this.tableCommitContextCodec = requireNonNull(tableCommitContextCodec, "tableCommitContextCodec is null"); this.lifespanAndStageStateTracker = new LifespanAndStageStateTracker(pageSinkCommitter, operatorRetainedMemoryBytes); this.systemMemoryContext = operatorContext.localSystemMemoryContext(); - this.tableFinishInfoSupplier = createTableFinishInfoSupplier(outputMetadata, statisticsTiming); + this.tableFinishInfoSupplier = createTableFinishInfoSupplier(outputMetadata, statisticsTiming, jsonLengthLimit); operatorContext.setInfoSupplier(tableFinishInfoSupplier); } @@ -322,14 +328,15 @@ TableFinishInfo getInfo() return tableFinishInfoSupplier.get(); } - private static Supplier createTableFinishInfoSupplier(AtomicReference> outputMetadata, OperationTiming statisticsTiming) + private static Supplier createTableFinishInfoSupplier(AtomicReference> outputMetadata, OperationTiming statisticsTiming, int jsonLengthLimit) { requireNonNull(outputMetadata, "outputMetadata is null"); requireNonNull(statisticsTiming, "statisticsTiming is null"); return () -> new TableFinishInfo( outputMetadata.get(), succinctNanos(statisticsTiming.getWallNanos()), - succinctNanos(statisticsTiming.getCpuNanos())); + succinctNanos(statisticsTiming.getCpuNanos()), + jsonLengthLimit); } @Override diff --git a/presto-main-base/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java b/presto-main-base/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java index ed686664b10d2..6e553eaf09990 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java +++ b/presto-main-base/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java @@ -123,6 +123,7 @@ import com.facebook.presto.operator.PagesIndex; import com.facebook.presto.operator.SourceOperatorFactory; import com.facebook.presto.operator.TableCommitContext; +import com.facebook.presto.operator.FeaturesConfig; import com.facebook.presto.operator.TaskContext; import com.facebook.presto.operator.index.IndexJoinLookupStats; import com.facebook.presto.server.NodeStatusNotificationManager; @@ -174,7 +175,6 @@ import com.facebook.presto.sql.analyzer.BuiltInQueryPreparer; import com.facebook.presto.sql.analyzer.BuiltInQueryPreparer.BuiltInPreparedQuery; import com.facebook.presto.sql.analyzer.BuiltInQueryPreparerProvider; -import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.analyzer.FunctionsConfig; import com.facebook.presto.sql.analyzer.JavaFeaturesConfig; import com.facebook.presto.sql.analyzer.QueryExplainer; @@ -373,30 +373,30 @@ public class LocalQueryRunner public LocalQueryRunner(Session defaultSession) { - this(defaultSession, new FeaturesConfig(), new FunctionsConfig(), new NodeSpillConfig(), false, false); + this(defaultSession, new com.facebook.presto.sql.analyzer.FeaturesConfig(), new FunctionsConfig(), new NodeSpillConfig(), false, false); } - public LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, FunctionsConfig functionsConfig) + public LocalQueryRunner(Session defaultSession, com.facebook.presto.sql.analyzer.FeaturesConfig featuresConfig, FunctionsConfig functionsConfig) { this(defaultSession, featuresConfig, functionsConfig, new NodeSpillConfig(), false, false); } - public LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, FunctionsConfig functionsConfig, NodeSpillConfig nodeSpillConfig, boolean withInitialTransaction, boolean alwaysRevokeMemory) + public LocalQueryRunner(Session defaultSession, com.facebook.presto.sql.analyzer.FeaturesConfig featuresConfig, FunctionsConfig functionsConfig, NodeSpillConfig nodeSpillConfig, boolean withInitialTransaction, boolean alwaysRevokeMemory) { this(defaultSession, featuresConfig, functionsConfig, nodeSpillConfig, withInitialTransaction, alwaysRevokeMemory, new ObjectMapper()); } - public LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, FunctionsConfig functionsConfig, NodeSpillConfig nodeSpillConfig, boolean withInitialTransaction, boolean alwaysRevokeMemory, ObjectMapper objectMapper) + public LocalQueryRunner(Session defaultSession, com.facebook.presto.sql.analyzer.FeaturesConfig featuresConfig, FunctionsConfig functionsConfig, NodeSpillConfig nodeSpillConfig, boolean withInitialTransaction, boolean alwaysRevokeMemory, ObjectMapper objectMapper) { this(defaultSession, featuresConfig, functionsConfig, nodeSpillConfig, withInitialTransaction, alwaysRevokeMemory, 1, objectMapper, new TaskManagerConfig().setTaskConcurrency(4)); } - public LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, FunctionsConfig functionsConfig, NodeSpillConfig nodeSpillConfig, boolean withInitialTransaction, boolean alwaysRevokeMemory, ObjectMapper objectMapper, TaskManagerConfig taskManagerConfig) + public LocalQueryRunner(Session defaultSession, com.facebook.presto.sql.analyzer.FeaturesConfig featuresConfig, FunctionsConfig functionsConfig, NodeSpillConfig nodeSpillConfig, boolean withInitialTransaction, boolean alwaysRevokeMemory, ObjectMapper objectMapper, TaskManagerConfig taskManagerConfig) { this(defaultSession, featuresConfig, functionsConfig, nodeSpillConfig, withInitialTransaction, alwaysRevokeMemory, 1, objectMapper, taskManagerConfig); } - private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, FunctionsConfig functionsConfig, NodeSpillConfig nodeSpillConfig, boolean withInitialTransaction, boolean alwaysRevokeMemory, int nodeCountForStats, ObjectMapper objectMapper, TaskManagerConfig taskManagerConfig) + private LocalQueryRunner(Session defaultSession, com.facebook.presto.sql.analyzer.FeaturesConfig featuresConfig, FunctionsConfig functionsConfig, NodeSpillConfig nodeSpillConfig, boolean withInitialTransaction, boolean alwaysRevokeMemory, int nodeCountForStats, ObjectMapper objectMapper, TaskManagerConfig taskManagerConfig) { requireNonNull(defaultSession, "defaultSession is null"); checkArgument(!defaultSession.getTransactionId().isPresent() || !withInitialTransaction, "Already in transaction"); @@ -456,7 +456,8 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, new NodeSpillConfig(), new TracingConfig(), new CompilerConfig(), - new HistoryBasedOptimizationConfig()).getSessionProperties(), + new HistoryBasedOptimizationConfig(), + new FeaturesConfig()).getSessionProperties(), new JavaFeaturesConfig(), nodeSpillConfig), new SchemaPropertyManager(), @@ -598,7 +599,7 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, dataDefinitionTask = ImmutableMap., DataDefinitionTask>builder() .put(CreateTable.class, new CreateTableTask()) - .put(CreateView.class, new CreateViewTask(jsonCodec(ViewDefinition.class), sqlParser, new FeaturesConfig())) + .put(CreateView.class, new CreateViewTask(jsonCodec(ViewDefinition.class), sqlParser, new com.facebook.presto.sql.analyzer.FeaturesConfig())) .put(CreateMaterializedView.class, new CreateMaterializedViewTask(sqlParser)) .put(CreateType.class, new CreateTypeTask(sqlParser)) .put(CreateFunction.class, new CreateFunctionTask(sqlParser)) @@ -631,12 +632,12 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, public static LocalQueryRunner queryRunnerWithInitialTransaction(Session defaultSession) { checkArgument(!defaultSession.getTransactionId().isPresent(), "Already in transaction!"); - return new LocalQueryRunner(defaultSession, new FeaturesConfig(), new FunctionsConfig(), new NodeSpillConfig(), true, false); + return new LocalQueryRunner(defaultSession, new com.facebook.presto.sql.analyzer.FeaturesConfig(), new FunctionsConfig(), new NodeSpillConfig(), true, false); } public static LocalQueryRunner queryRunnerWithFakeNodeCountForStats(Session defaultSession, int nodeCount) { - return new LocalQueryRunner(defaultSession, new FeaturesConfig(), new FunctionsConfig(), new NodeSpillConfig(), false, false, nodeCount, new ObjectMapper(), new TaskManagerConfig().setTaskConcurrency(4)); + return new LocalQueryRunner(defaultSession, new com.facebook.presto.sql.analyzer.FeaturesConfig(), new FunctionsConfig(), new NodeSpillConfig(), false, false, nodeCount, new ObjectMapper(), new TaskManagerConfig().setTaskConcurrency(4)); } @Override @@ -1152,7 +1153,7 @@ public List getPlanOptimizers(boolean noExchange) public List getPlanOptimizers(boolean noExchange, boolean nativeExecutionEnabled) { - FeaturesConfig featuresConfig = new FeaturesConfig() + com.facebook.presto.sql.analyzer.FeaturesConfig featuresConfig = new com.facebook.presto.sql.analyzer.FeaturesConfig() .setDistributedIndexJoinsEnabled(false) .setOptimizeHashGeneration(true) .setNativeExecutionEnabled(nativeExecutionEnabled); diff --git a/presto-main-base/src/test/java/com/facebook/presto/execution/TestCreateMaterializedViewTask.java b/presto-main-base/src/test/java/com/facebook/presto/execution/TestCreateMaterializedViewTask.java index 31227f45233f6..35c19ddbd2d3b 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/execution/TestCreateMaterializedViewTask.java +++ b/presto-main-base/src/test/java/com/facebook/presto/execution/TestCreateMaterializedViewTask.java @@ -32,6 +32,7 @@ import com.facebook.presto.metadata.FunctionAndTypeManager; import com.facebook.presto.metadata.SessionPropertyManager; import com.facebook.presto.metadata.TablePropertyManager; +import com.facebook.presto.operator.FeaturesConfig; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ColumnMetadata; import com.facebook.presto.spi.ConnectorId; @@ -51,7 +52,6 @@ import com.facebook.presto.spi.security.AccessControl; import com.facebook.presto.spi.security.AllowAllAccessControl; import com.facebook.presto.spiller.NodeSpillConfig; -import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.analyzer.FunctionsConfig; import com.facebook.presto.sql.analyzer.JavaFeaturesConfig; import com.facebook.presto.sql.analyzer.SemanticException; @@ -342,7 +342,7 @@ public void testCreateMaterializedViewWithInvalidDefaultViewSecurityMode() private static SessionPropertyManager createSessionPropertyManager() { - FeaturesConfig featuresConfig = new FeaturesConfig() + com.facebook.presto.sql.analyzer.FeaturesConfig featuresConfig = new com.facebook.presto.sql.analyzer.FeaturesConfig() .setAllowLegacyMaterializedViewsToggle(true); return SessionPropertyManager.createTestingSessionPropertyManager( @@ -358,7 +358,8 @@ private static SessionPropertyManager createSessionPropertyManager() new NodeSpillConfig(), new TracingConfig(), new CompilerConfig(), - new HistoryBasedOptimizationConfig()).getSessionProperties(), + new HistoryBasedOptimizationConfig(), + new FeaturesConfig()).getSessionProperties(), featuresConfig, new JavaFeaturesConfig(), new NodeSpillConfig()); diff --git a/presto-main-base/src/test/java/com/facebook/presto/execution/scheduler/TestAdaptivePhasedExecutionPolicy.java b/presto-main-base/src/test/java/com/facebook/presto/execution/scheduler/TestAdaptivePhasedExecutionPolicy.java index 3a9ca81197856..b357797b80a82 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/execution/scheduler/TestAdaptivePhasedExecutionPolicy.java +++ b/presto-main-base/src/test/java/com/facebook/presto/execution/scheduler/TestAdaptivePhasedExecutionPolicy.java @@ -28,6 +28,7 @@ import com.facebook.presto.failureDetector.NoOpFailureDetector; import com.facebook.presto.memory.MemoryManagerConfig; import com.facebook.presto.memory.NodeMemoryConfig; +import com.facebook.presto.operator.FeaturesConfig; import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.QueryId; import com.facebook.presto.spi.TableHandle; @@ -41,7 +42,6 @@ import com.facebook.presto.spi.plan.TableScanNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.spiller.NodeSpillConfig; -import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.analyzer.FunctionsConfig; import com.facebook.presto.sql.planner.CompilerConfig; import com.facebook.presto.sql.planner.PlanFragment; @@ -98,7 +98,7 @@ public void testCreateExecutionSchedule() new QueryManagerConfig(), new TaskManagerConfig(), new MemoryManagerConfig(), - new FeaturesConfig().setMaxStageCountForEagerScheduling(5), + new com.facebook.presto.sql.analyzer.FeaturesConfig().setMaxStageCountForEagerScheduling(5), new FunctionsConfig(), new NodeMemoryConfig(), new WarningCollectorConfig(), @@ -106,7 +106,8 @@ public void testCreateExecutionSchedule() new NodeSpillConfig(), new TracingConfig(), new CompilerConfig(), - new HistoryBasedOptimizationConfig()))).build(); + new HistoryBasedOptimizationConfig(), + new FeaturesConfig()))).build(); AdaptivePhasedExecutionPolicy policy = new AdaptivePhasedExecutionPolicy(); Collection schedulers = getStageExecutionAndSchedulers(4); assertTrue(policy.createExecutionSchedule(session, schedulers) instanceof AllAtOnceExecutionSchedule); diff --git a/presto-main-base/src/test/java/com/facebook/presto/operator/TestFeaturesConfig.java b/presto-main-base/src/test/java/com/facebook/presto/operator/TestFeaturesConfig.java new file mode 100644 index 0000000000000..45c04eab8c7f9 --- /dev/null +++ b/presto-main-base/src/test/java/com/facebook/presto/operator/TestFeaturesConfig.java @@ -0,0 +1,46 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator; + +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.Map; + +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; + +public class TestFeaturesConfig +{ + @Test + public void testDefaults() + { + assertRecordedDefaults(recordDefaults(FeaturesConfig.class) + .setTableFinishInfoJsonLengthLimit(10_000_000)); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = new ImmutableMap.Builder() + .put("table-finish-info-json-length-limit", "5000000") + .build(); + + FeaturesConfig expected = new FeaturesConfig() + .setTableFinishInfoJsonLengthLimit(5_000_000); + + assertFullMapping(properties, expected); + } +} diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestAnalyzer.java b/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestAnalyzer.java index a74d59c1e2d03..1ff26333a6176 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestAnalyzer.java +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestAnalyzer.java @@ -22,6 +22,7 @@ import com.facebook.presto.execution.warnings.WarningCollectorConfig; import com.facebook.presto.memory.MemoryManagerConfig; import com.facebook.presto.memory.NodeMemoryConfig; +import com.facebook.presto.operator.FeaturesConfig; import com.facebook.presto.spi.PrestoWarning; import com.facebook.presto.spi.StandardWarningCode; import com.facebook.presto.spi.WarningCollector; @@ -236,7 +237,7 @@ public void testWindowOrderByAnalysis() new QueryManagerConfig(), new TaskManagerConfig(), new MemoryManagerConfig(), - new FeaturesConfig().setAllowWindowOrderByLiterals(false), + new com.facebook.presto.sql.analyzer.FeaturesConfig().setAllowWindowOrderByLiterals(false), new FunctionsConfig(), new NodeMemoryConfig(), new WarningCollectorConfig(), @@ -244,7 +245,8 @@ public void testWindowOrderByAnalysis() new NodeSpillConfig(), new TracingConfig(), new CompilerConfig(), - new HistoryBasedOptimizationConfig()))).build(); + new HistoryBasedOptimizationConfig(), + new FeaturesConfig()))).build(); assertFails(session, WINDOW_FUNCTION_ORDERBY_LITERAL, "SELECT SUM(x) OVER (PARTITION BY y ORDER BY 1) AS s\n" + "FROM (values (1,10), (2, 10)) AS T(x, y)"); @@ -646,7 +648,7 @@ public void testTooManyGroupingElements() new QueryManagerConfig(), new TaskManagerConfig(), new MemoryManagerConfig(), - new FeaturesConfig().setMaxGroupingSets(2048), + new com.facebook.presto.sql.analyzer.FeaturesConfig().setMaxGroupingSets(2048), new FunctionsConfig(), new NodeMemoryConfig(), new WarningCollectorConfig(), @@ -654,7 +656,8 @@ public void testTooManyGroupingElements() new NodeSpillConfig(), new TracingConfig(), new CompilerConfig(), - new HistoryBasedOptimizationConfig()))).build(); + new HistoryBasedOptimizationConfig(), + new FeaturesConfig()))).build(); analyze(session, "SELECT a, b, c, d, e, f, g, h, i, j, k, SUM(l)" + "FROM (VALUES (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12))\n" + "t (a, b, c, d, e, f, g, h, i, j, k, l)\n" + diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/sanity/TestValidateStreamingJoins.java b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/sanity/TestValidateStreamingJoins.java index caa37100245c7..28c3604d19927 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/sanity/TestValidateStreamingJoins.java +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/sanity/TestValidateStreamingJoins.java @@ -24,6 +24,7 @@ import com.facebook.presto.memory.MemoryManagerConfig; import com.facebook.presto.memory.NodeMemoryConfig; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.operator.FeaturesConfig; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.TableHandle; @@ -32,7 +33,6 @@ import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spiller.NodeSpillConfig; -import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.analyzer.FunctionsConfig; import com.facebook.presto.sql.planner.CompilerConfig; import com.facebook.presto.sql.planner.assertions.BasePlanTest; @@ -81,7 +81,7 @@ public void setup() new QueryManagerConfig(), new TaskManagerConfig(), new MemoryManagerConfig(), - new FeaturesConfig().setSpillerSpillPaths("/path/to/nowhere"), + new com.facebook.presto.sql.analyzer.FeaturesConfig().setSpillerSpillPaths("/path/to/nowhere"), new FunctionsConfig(), new NodeMemoryConfig(), new WarningCollectorConfig(), @@ -89,7 +89,8 @@ public void setup() new NodeSpillConfig(), new TracingConfig(), new CompilerConfig(), - new HistoryBasedOptimizationConfig()))) + new HistoryBasedOptimizationConfig(), + new FeaturesConfig()))) .setCatalog("local") .setSchema("tiny") .setSystemProperty("spill_enabled", "true") @@ -207,7 +208,7 @@ private void validatePlan(Function planProvider, boolean PlanNode planNode = planProvider.apply(builder); getQueryRunner().inTransaction(testSession, session -> { session.getCatalog().ifPresent(catalog -> metadata.getCatalogHandle(session, catalog)); - new ValidateStreamingJoins(new FeaturesConfig().setNativeExecutionEnabled(nativeExecutionEnabled)).validate(planNode, session, metadata, WarningCollector.NOOP); + new ValidateStreamingJoins(new com.facebook.presto.sql.analyzer.FeaturesConfig().setNativeExecutionEnabled(nativeExecutionEnabled)).validate(planNode, session, metadata, WarningCollector.NOOP); return null; }); } diff --git a/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java b/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java index 63beb77c6bcf8..6f645e20f9b42 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java @@ -137,6 +137,7 @@ import com.facebook.presto.operator.PagesIndex; import com.facebook.presto.operator.RpcShuffleClientProvider; import com.facebook.presto.operator.TableCommitContext; +import com.facebook.presto.operator.FeaturesConfig; import com.facebook.presto.operator.TaskMemoryReservationSummary; import com.facebook.presto.operator.ThriftShuffleClientProvider; import com.facebook.presto.operator.index.IndexJoinLookupStats; @@ -209,7 +210,6 @@ import com.facebook.presto.sql.analyzer.BuiltInQueryAnalyzer; import com.facebook.presto.sql.analyzer.BuiltInQueryPreparer; import com.facebook.presto.sql.analyzer.BuiltInQueryPreparerProvider; -import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.analyzer.FeaturesConfig.SingleStreamSpillerChoice; import com.facebook.presto.sql.analyzer.ForMetadataExtractor; import com.facebook.presto.sql.analyzer.FunctionsConfig; @@ -342,7 +342,7 @@ else if (serverConfig.isCoordinator()) { install(new InternalCommunicationModule()); configBinder(binder).bindConfig(ServerConfig.class); - configBinder(binder).bindConfig(FeaturesConfig.class); + configBinder(binder).bindConfig(com.facebook.presto.sql.analyzer.FeaturesConfig.class); configBinder(binder).bindConfig(FunctionsConfig.class); configBinder(binder).bindConfig(JavaFeaturesConfig.class); @@ -536,8 +536,8 @@ public ListeningExecutorService createResourceManagerExecutor(ResourceManagerCon moduleBinder.bind(ResourceGroupService.class).to(NoopResourceGroupService.class).in(Scopes.SINGLETON); })); - FeaturesConfig featuresConfig = buildConfigObject(FeaturesConfig.class); - FeaturesConfig.TaskSpillingStrategy taskSpillingStrategy = featuresConfig.getTaskSpillingStrategy(); + com.facebook.presto.sql.analyzer.FeaturesConfig featuresConfig = buildConfigObject(com.facebook.presto.sql.analyzer.FeaturesConfig.class); + com.facebook.presto.sql.analyzer.FeaturesConfig.TaskSpillingStrategy taskSpillingStrategy = featuresConfig.getTaskSpillingStrategy(); switch (taskSpillingStrategy) { case PER_TASK_MEMORY_THRESHOLD: binder.bind(TaskThresholdMemoryRevokingScheduler.class).in(Scopes.SINGLETON); @@ -628,6 +628,7 @@ public ListeningExecutorService createResourceManagerExecutor(ResourceManagerCon addressSelectorBinder.bind(AddressSelector.class).annotatedWith(annotation).to(FixedAddressSelector.class))); configBinder(binder).bindConfig(ExchangeClientConfig.class); + configBinder(binder).bindConfig(FeaturesConfig.class); binder.bind(ExchangeExecutionMBean.class).in(Scopes.SINGLETON); newExporter(binder).export(ExchangeExecutionMBean.class).withGeneratedName(); @@ -812,14 +813,14 @@ public ListeningExecutorService createResourceManagerExecutor(ResourceManagerCon configBinder(binder).bindConfig(NodeSpillConfig.class); install(installModuleIf( - FeaturesConfig.class, + com.facebook.presto.sql.analyzer.FeaturesConfig.class, config -> config.getSingleStreamSpillerChoice() == SingleStreamSpillerChoice.LOCAL_FILE, moduleBinder -> moduleBinder .bind(SingleStreamSpillerFactory.class) .to(FileSingleStreamSpillerFactory.class) .in(Scopes.SINGLETON))); install(installModuleIf( - FeaturesConfig.class, + com.facebook.presto.sql.analyzer.FeaturesConfig.class, config -> config.getSingleStreamSpillerChoice() == SingleStreamSpillerChoice.TEMP_STORAGE, moduleBinder -> moduleBinder .bind(SingleStreamSpillerFactory.class) diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java index d7e41e512f830..499c67679e7e6 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java @@ -98,6 +98,7 @@ import com.facebook.presto.operator.OperatorStats; import com.facebook.presto.operator.PagesIndex; import com.facebook.presto.operator.TableCommitContext; +import com.facebook.presto.operator.FeaturesConfig; import com.facebook.presto.operator.TaskMemoryReservationSummary; import com.facebook.presto.operator.index.IndexJoinLookupStats; import com.facebook.presto.resourcemanager.NoopResourceGroupService; @@ -173,7 +174,6 @@ import com.facebook.presto.sql.analyzer.BuiltInQueryAnalyzer; import com.facebook.presto.sql.analyzer.BuiltInQueryPreparer; import com.facebook.presto.sql.analyzer.BuiltInQueryPreparerProvider; -import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.analyzer.ForMetadataExtractor; import com.facebook.presto.sql.analyzer.FunctionsConfig; import com.facebook.presto.sql.analyzer.JavaFeaturesConfig; @@ -270,10 +270,10 @@ protected void setup(Binder binder) configBinder(binder).bindConfig(SimpleTtlNodeSelectorConfig.class); configBinder(binder).bindConfig(QueryManagerConfig.class); configBinder(binder).bindConfigGlobalDefaults(QueryManagerConfig.class, PrestoSparkSettingsRequirements::setDefaults); - configBinder(binder).bindConfig(FeaturesConfig.class); + configBinder(binder).bindConfig(com.facebook.presto.sql.analyzer.FeaturesConfig.class); configBinder(binder).bindConfig(FunctionsConfig.class); configBinder(binder).bindConfig(JavaFeaturesConfig.class); - configBinder(binder).bindConfigGlobalDefaults(FeaturesConfig.class, PrestoSparkSettingsRequirements::setDefaults); + configBinder(binder).bindConfigGlobalDefaults(com.facebook.presto.sql.analyzer.FeaturesConfig.class, PrestoSparkSettingsRequirements::setDefaults); configBinder(binder).bindConfig(MemoryManagerConfig.class); configBinder(binder).bindConfig(TaskManagerConfig.class); configBinder(binder).bindConfig(TransactionManagerConfig.class); @@ -289,6 +289,7 @@ protected void setup(Binder binder) configBinder(binder).bindConfig(TracingConfig.class); configBinder(binder).bindConfig(PlanCheckerProviderManagerConfig.class); configBinder(binder).bindConfig(SecurityConfig.class); + configBinder(binder).bindConfig(FeaturesConfig.class); // json codecs jsonCodecBinder(binder).bindJsonCodec(ViewDefinition.class); @@ -363,7 +364,7 @@ protected void setup(Binder binder) MapBinder mapBinder = newMapBinder(binder, String.class, WorkerSessionPropertyProvider.class); - FeaturesConfig featuresConfig = buildConfigObject(FeaturesConfig.class); + com.facebook.presto.sql.analyzer.FeaturesConfig featuresConfig = buildConfigObject(com.facebook.presto.sql.analyzer.FeaturesConfig.class); if (featuresConfig.isNativeExecutionEnabled()) { mapBinder.addBinding("native-worker").to(NativeWorkerSessionPropertyProvider.class) .in(Scopes.SINGLETON);