Changes for EXPLAIN VERBOSE

.gitignore

@@ -89,3 +89,5 @@ fdb-relational-core/src/main/antlr/*.tokens
 
 # output during release build process
 /mixed-mode-results.md
+
+.DS_Store

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/explain/ExplainPlanVisitor.java

@@ -722,6 +722,15 @@ public static String toStringForExternalExplain(@Nonnull final RecordQueryPlan p
         return toStringForExternalExplain(plan, ExplainLevel.ALL_DETAILS, Integer.MAX_VALUE);
     }
 
+    @Nonnull
+    public static String toStringForExternalExplain(@Nonnull final RecordQueryPlan plan, final boolean isVerboseExplainLevel) {
+        final var visitor = new ExplainPlanVisitor(Integer.MAX_VALUE);
+        final var explainTokens = visitor.visit(plan);
+        final var explainLevel = isVerboseExplainLevel ? ExplainLevel.ALL_DETAILS : ExplainLevel.STRUCTURE;
+        return explainTokens.render(explainLevel, new DefaultExplainFormatter(ExplainSelfContainedSymbolMap::new), explainTokens.getMaxLength(explainLevel))
+                .toString();
+    }
+
     @Nonnull
     public static String toStringForExternalExplain(@Nonnull final RecordQueryPlan plan, final int maxExplainLevel,
                                                     final int maxSize) {

fdb-relational-core/src/main/antlr/RelationalLexer.g4

@@ -223,6 +223,7 @@ USAGE:                               'USAGE';
 USE:                                 'USE';
 USING:                               'USING';
 VALUES:                              'VALUES';
+VERBOSE:                             'VERBOSE';
 WHEN:                                'WHEN';
 WHERE:                               'WHERE';
 WHILE:                               'WHILE';
@@ -1352,4 +1353,4 @@ fragment DECIMAL_TYPE_MODIFIER:      (INT_TYPE_MODIFIER | LONG_TYPE_MODIFIER);
 
 ERROR_RECOGNITION
     : . { this.notifyListeners(new LexerNoViableAltException(this, _input, _tokenStartCharIndex, null)); }
-    ;
+    ;

fdb-relational-core/src/main/antlr/RelationalParser.g4

@@ -650,7 +650,8 @@ helpStatement
 // details
 
 describeObjectClause
-    : (
+    : (VERBOSE)?
+      (
         query | deleteStatement | insertStatement
         | updateStatement | executeContinuationStatement
       )                                                             #describeStatements

fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/AstNormalizer.java

@@ -244,6 +244,12 @@ public Void visitFullDescribeStatement(@Nonnull RelationalParser.FullDescribeSta
         return visitChildren(ctx);
     }
 
+    @Override
+    public Void visitDescribeStatements(@Nonnull RelationalParser.DescribeStatementsContext ctx) {
+        queryHasherContextBuilder.setIsVerboseExplainLevel(ctx.VERBOSE() != null);
+        return visitChildren(ctx);
+    }
+
     @Override
     public Void visitLimitClause(@Nonnull RelationalParser.LimitClauseContext ctx) {
         if (ctx.offset != null) {

fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/MutablePlanGenerationContext.java

@@ -21,7 +21,6 @@
 package com.apple.foundationdb.relational.recordlayer.query;
 
 import com.apple.foundationdb.annotation.API;
-
 import com.apple.foundationdb.record.ExecuteProperties;
 import com.apple.foundationdb.record.PlanHashable;
 import com.apple.foundationdb.record.query.expressions.Comparisons;
@@ -43,7 +42,6 @@
 import com.apple.foundationdb.relational.recordlayer.metadata.DataTypeUtils;
 import com.apple.foundationdb.relational.util.Assert;
 import com.apple.foundationdb.relational.util.SpotBugsSuppressWarnings;
-
 import com.google.common.collect.ImmutableList;
 import com.google.protobuf.ZeroCopyByteString;
 
@@ -91,6 +89,8 @@ public class MutablePlanGenerationContext implements QueryExecutionContext {
 
     private boolean forExplain;
 
+    private boolean isVerboseExplainLevel;
+
     @Nullable
     private byte[] continuation;
 
@@ -279,6 +279,7 @@ public MutablePlanGenerationContext(@Nonnull PreparedParams preparedParams,
         constantObjectValues = new LinkedList<>();
         shouldProcessLiteral = true;
         forExplain = false;
+        isVerboseExplainLevel = false;
         setContinuation(null);
         equalityConstraints = ImmutableList.builder();
     }
@@ -354,6 +355,10 @@ public boolean isForExplain() {
         return forExplain;
     }
 
+    @Override
+    public boolean isVerboseExplainLevel() {
+        return isVerboseExplainLevel;
+    }
 
     @Nonnull
     public QueryPlanConstraint getPlanConstraintsForLiteralReferences() {
@@ -390,6 +395,10 @@ public void setForExplain(boolean forExplain) {
         this.forExplain = forExplain;
     }
 
+    public void setIsVerboseExplainLevel(boolean isVerboseExplainLevel) {
+        this.isVerboseExplainLevel = isVerboseExplainLevel;
+    }
+
     @Nonnull
     public Value processQueryLiteral(@Nonnull Type type, @Nullable Object literal, int tokenIndex) {
         return processQueryLiteralOrParameter(type, literal, null, null, tokenIndex);

fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/NormalizedQueryExecutionContext.java

@@ -44,6 +44,8 @@ public final class NormalizedQueryExecutionContext implements QueryExecutionCont
 
     private final boolean isForExplain;
 
+    private final boolean isVerboseExplainLevel;
+
     private final int parameterHash;
 
     @Nonnull
@@ -53,10 +55,12 @@ private NormalizedQueryExecutionContext(@Nonnull Literals literals,
                                             @Nullable byte[] continuation,
                                             int parameterHash,
                                             boolean isForExplain,
+                                            boolean isVerboseExplainLevel,
                                             @Nonnull final PlanHashable.PlanHashMode planHashMode) {
         this.literals = literals;
         this.continuation = continuation;
         this.isForExplain = isForExplain;
+        this.isVerboseExplainLevel = isVerboseExplainLevel;
         this.parameterHash = parameterHash;
         this.planHashMode = planHashMode;
     }
@@ -90,6 +94,11 @@ public boolean isForExplain() {
         return isForExplain;
     }
 
+    @Override
+    public boolean isVerboseExplainLevel() {
+        return isVerboseExplainLevel;
+    }
+
     @Nonnull
     @Override
     public PlanHashable.PlanHashMode getPlanHashMode() {
@@ -107,6 +116,8 @@ public static final class Builder {
 
         private boolean isForExplain;
 
+        private boolean isVerboseExplainLevel;
+
         @Nullable
         private byte[] continuation;
 
@@ -120,6 +131,7 @@ private Builder() {
             this.isForExplain = false;
             this.continuation = null;
             this.planHashMode = null;
+            this.isVerboseExplainLevel = false;
         }
 
         @Nonnull
@@ -146,6 +158,12 @@ public Builder setForExplain(boolean isForExplain) {
             return this;
         }
 
+        @Nonnull
+        public Builder setIsVerboseExplainLevel(boolean isVerboseExplainLevel) {
+            this.isVerboseExplainLevel = isVerboseExplainLevel;
+            return this;
+        }
+
         @Nonnull
         public Builder setPlanHashMode(@Nonnull PlanHashable.PlanHashMode planHashMode) {
             this.planHashMode = planHashMode;
@@ -155,7 +173,7 @@ public Builder setPlanHashMode(@Nonnull PlanHashable.PlanHashMode planHashMode)
         @Nonnull
         public NormalizedQueryExecutionContext build() {
             return new NormalizedQueryExecutionContext(literalsBuilder.build(), continuation,
-                    parameterHash, isForExplain,
+                    parameterHash, isForExplain, isVerboseExplainLevel,
                     Objects.requireNonNull(planHashMode));
         }
     }

fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/QueryExecutionContext.java

@@ -60,6 +60,8 @@ default EvaluationContext getEvaluationContext() {
 
     boolean isForExplain(); // todo (yhatem) remove.
 
+    boolean isVerboseExplainLevel();
+
     @Nonnull
     PlanHashable.PlanHashMode getPlanHashMode();
 }

fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/QueryPlan.java

@@ -200,7 +200,7 @@ public PhysicalQueryPlan withExecutionContext(@Nonnull final QueryExecutionConte
         public String explain() {
             final var executeProperties = queryExecutionContext.getExecutionPropertiesBuilder();
             List<String> explainComponents = new ArrayList<>();
-            explainComponents.add(ExplainPlanVisitor.toStringForExternalExplain(recordQueryPlan));
+            explainComponents.add(ExplainPlanVisitor.toStringForExternalExplain(recordQueryPlan, queryExecutionContext.isVerboseExplainLevel()));
             if (executeProperties.getReturnedRowLimit() != ReadTransaction.ROW_LIMIT_UNLIMITED) {
                 explainComponents.add("(limit=" + executeProperties.getReturnedRowLimit() + ")");
             }

fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/QueryVisitor.java

@@ -536,6 +536,11 @@ public Object visitExecuteContinuationStatement(@Nonnull RelationalParser.Execut
     @Override
     public QueryPlan.LogicalQueryPlan visitFullDescribeStatement(@Nonnull RelationalParser.FullDescribeStatementContext ctx) {
         getDelegate().getPlanGenerationContext().setForExplain(ctx.EXPLAIN() != null);
+        if (!ctx.describeObjectClause().getTokens(RelationalLexer.VERBOSE).isEmpty()) {
+            getDelegate().getPlanGenerationContext().setIsVerboseExplainLevel(true);
+        } else {
+            getDelegate().getPlanGenerationContext().setIsVerboseExplainLevel(false);
+        }
         final var logicalOperator = Assert.castUnchecked(ctx.describeObjectClause().accept(this), LogicalOperator.class);
         return QueryPlan.LogicalQueryPlan.of(logicalOperator.getQuantifier().getRangesOver().get(), getDelegate().getPlanGenerationContext(), "TODO");
     }

fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/QueryLoggingTest.java

@@ -245,7 +245,7 @@ void testRelationalConnectionSetLogIsOverriddenByExecuteContinuationQueryOption(
                     rs.next();
                 }
                 Assertions.assertThat(logAppender.getLogEvents()).isNotEmpty();
-                Assertions.assertThat(logAppender.getLastLogEventMessage()).contains("plan=\"COVERING(RECORD_TYPE_COVERING <,> -> [NAME: VALUE[0], REST_NO: KEY[0]]) | MAP (_.NAME AS NAME)\"");
+                Assertions.assertThat(logAppender.getLastLogEventMessage()).contains("plan=\"COVERING(RECORD_TYPE_COVERING) | MAP (_.NAME AS NAME)\"");
             }
         }
     }
@@ -276,7 +276,7 @@ void testRelationalConnectionSetLogWithExecuteContinuation(boolean setLogging) t
                     Assertions.assertThat(logAppender.getLogEvents()).isEmpty();
                 } else {
                     Assertions.assertThat(logAppender.getLogEvents()).isNotEmpty();
-                    Assertions.assertThat(logAppender.getLastLogEventMessage()).contains("plan=\"COVERING(RECORD_TYPE_COVERING <,> -> [NAME: VALUE[0], REST_NO: KEY[0]]) | MAP (_.NAME AS NAME)\"");
+                    Assertions.assertThat(logAppender.getLastLogEventMessage()).contains("plan=\"COVERING(RECORD_TYPE_COVERING) | MAP (_.NAME AS NAME)\"");
                 }
             }
         }

fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/ExplainTests.java

@@ -108,7 +108,7 @@ void explainWithNoContinuationTest() throws Exception {
                 try (final RelationalResultSet resultSet = ps.executeQuery()) {
                     final var assertResult = ResultSetAssert.assertThat(resultSet);
                     assertResult.hasNextRow()
-                            .hasColumn("PLAN", "ISCAN(RECORD_NAME_IDX <,>)")
+                            .hasColumn("PLAN", "ISCAN(RECORD_NAME_IDX)")
                             .hasColumn("PLAN_HASH", -1635569052)
                             .hasColumn("PLAN_CONTINUATION", null);
                     assertResult.hasNoNextRow();
@@ -135,7 +135,7 @@ void explainWithContinuationNoSerializedPlanTest() throws Exception {
                         final var assertResult = ResultSetAssert.assertThat(resultSet);
 
                         assertResult.hasNextRow()
-                                .hasColumn("PLAN", "ISCAN(RECORD_NAME_IDX <,>)")
+                                .hasColumn("PLAN", "ISCAN(RECORD_NAME_IDX)")
                                 .hasColumn("PLAN_HASH", -1635569052);
                         final var continuationInfo = resultSet.getStruct(5);
                         org.junit.jupiter.api.Assertions.assertNotNull(continuationInfo);
@@ -168,7 +168,7 @@ void explainWithContinuationSerializedPlanTest() throws Exception {
                     try (final RelationalResultSet resultSet = ps.executeQuery()) {
                         final var assertResult = ResultSetAssert.assertThat(resultSet);
                         assertResult.hasNextRow()
-                                .hasColumn("PLAN", "ISCAN(RECORD_NAME_IDX <,>)")
+                                .hasColumn("PLAN", "ISCAN(RECORD_NAME_IDX)")
                                 .hasColumn("PLAN_HASH", -1635569052);
                         final var continuationInfo = resultSet.getStruct(5);
                         org.junit.jupiter.api.Assertions.assertNotNull(continuationInfo);
@@ -196,7 +196,7 @@ void explainWithContinuationSerializedPlanWithDifferentQueryTest() throws Except
                     continuation = consumeResultAndGetContinuation(ps, 2);
                 }
 
-                try (RelationalPreparedStatement ps = connection.prepareStatement("EXPLAIN SELECT rest_no FROM RestaurantComplexRecord WITH CONTINUATION ?cont")) {
+                try (RelationalPreparedStatement ps = connection.prepareStatement("EXPLAIN VERBOSE SELECT rest_no FROM RestaurantComplexRecord WITH CONTINUATION ?cont")) {
                     ps.setObject("cont", continuation.serialize());
                     try (final RelationalResultSet resultSet = ps.executeQuery()) {
                         final var assertResult = ResultSetAssert.assertThat(resultSet);
@@ -234,7 +234,7 @@ void explainExecuteStatementTest() throws Exception {
                     try (final RelationalResultSet resultSet = ps.executeQuery()) {
                         final var assertResult = ResultSetAssert.assertThat(resultSet);
                         assertResult.hasNextRow()
-                                .hasColumn("PLAN", "ISCAN(RECORD_NAME_IDX <,>)")
+                                .hasColumn("PLAN", "ISCAN(RECORD_NAME_IDX)")
                                 .hasColumn("PLAN_HASH", -1635569052);
                         final var continuationInfo = resultSet.getStruct(5);
                         org.junit.jupiter.api.Assertions.assertNotNull(continuationInfo);
@@ -251,6 +251,42 @@ void explainExecuteStatementTest() throws Exception {
         }
     }
 
+    @Test
+    void explainVerboseTest() throws Exception {
+        try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate).build()) {
+            executeInsert(ddl);
+            try (RelationalPreparedStatement ps = ddl.setSchemaAndGetConnection().prepareStatement("EXPLAIN VERBOSE SELECT * FROM RestaurantComplexRecord where COUNT(reviews) > 3 and COUNT(reviews) < 100")) {
+                ps.setMaxRows(2);
+                try (final RelationalResultSet resultSet = ps.executeQuery()) {
+                    final var assertResult = ResultSetAssert.assertThat(resultSet);
+                    assertResult.hasNextRow()
+                            .hasColumn("PLAN", "ISCAN(RECORD_NAME_IDX <,>) | FILTER count_star(*) GREATER_THAN promote(@c12 AS LONG) AND count_star(*) LESS_THAN promote(@c19 AS LONG)")
+                            .hasColumn("PLAN_HASH", -1697137247)
+                            .hasColumn("PLAN_CONTINUATION", null);
+                    assertResult.hasNoNextRow();
+                }
+            }
+        }
+    }
+
+    @Test
+    void explainDefaultTest() throws Exception {
+        try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate).build()) {
+            executeInsert(ddl);
+            try (RelationalPreparedStatement ps = ddl.setSchemaAndGetConnection().prepareStatement("EXPLAIN SELECT * FROM RestaurantComplexRecord where COUNT(reviews) > 3 and COUNT(reviews) < 100")) {
+                ps.setMaxRows(2);
+                try (final RelationalResultSet resultSet = ps.executeQuery()) {
+                    final var assertResult = ResultSetAssert.assertThat(resultSet);
+                    assertResult.hasNextRow()
+                            .hasColumn("PLAN", "ISCAN(RECORD_NAME_IDX) | FILTER count_star(*) GREATER_THAN promote(@c11 AS LONG) AND count_star(*) LESS_THAN promote(@c18 AS LONG)")
+                            .hasColumn("PLAN_HASH", -1697137247)
+                            .hasColumn("PLAN_CONTINUATION", null);
+                    assertResult.hasNoNextRow();
+                }
+            }
+        }
+    }
+
     private Continuation consumeResultAndGetContinuation(RelationalPreparedStatement ps, int numRows) throws SQLException {
         Continuation continuation;
         try (final RelationalResultSet resultSet = ps.executeQuery()) {

fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/GroupByQueryTests.java

@@ -185,12 +185,12 @@ void isNullPredicateUsesGroupIndex() throws Exception {
                 Assertions.assertTrue(statement.execute("EXPLAIN SELECT sum(c) FROM T1 WHERE a is null"), "Did not return a result set from a select statement!");
                 try (final RelationalResultSet resultSet = statement.getResultSet()) {
                     ResultSetAssert.assertThat(resultSet).hasNextRow()
-                            .hasColumn("PLAN", "AISCAN(SUM_IDX [[null],[null]] BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP ((_._1 AS _0) AS _0) | ON EMPTY NULL | MAP (_._0._0 AS _0)");
+                            .hasColumn("PLAN", "AISCAN(SUM_IDX -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP ((_._1 AS _0) AS _0) | ON EMPTY NULL | MAP (_._0._0 AS _0)");
                 }
                 Assertions.assertTrue(statement.execute("EXPLAIN SELECT sum(c) FROM T1 WHERE a = 1"), "Did not return a result set from a select statement!");
                 try (final RelationalResultSet resultSet = statement.getResultSet()) {
                     ResultSetAssert.assertThat(resultSet).hasNextRow()
-                            .hasColumn("PLAN", "AISCAN(SUM_IDX [EQUALS promote(@c11 AS LONG)] BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP ((_._1 AS _0) AS _0) | ON EMPTY NULL | MAP (_._0._0 AS _0)");
+                            .hasColumn("PLAN", "AISCAN(SUM_IDX -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP ((_._1 AS _0) AS _0) | ON EMPTY NULL | MAP (_._0._0 AS _0)");
                 }
             }
         }

fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java

@@ -233,7 +233,7 @@ void explainTableScan() throws Exception {
                 try (final RelationalResultSet resultSet = statement.executeQuery("EXPLAIN SELECT * FROM RestaurantComplexRecord WHERE rest_no > 10")) {
                     resultSet.next();
                     String plan = resultSet.getString(1);
-                    assertThat(plan).matches("(.*SCAN.*RESTAURANTCOMPLEXRECORD|.*COVERING.* <,>).*REST_NO GREATER_THAN promote\\(@c8 AS LONG\\).*");
+                    assertThat(plan).matches("(.*RESTAURANTCOMPLEXRECORD|.*COVERING.*).*REST_NO GREATER_THAN promote\\(@c8 AS LONG\\).*");
                 }
             }
         }
@@ -259,7 +259,7 @@ void explainUnhintedIndexScan() throws Exception {
                 try (final RelationalResultSet resultSet = statement.executeQuery("EXPLAIN SELECT * FROM RestaurantComplexRecord AS R WHERE EXISTS (SELECT * FROM R.reviews AS RE WHERE RE.rating >= 9)")) {
                     resultSet.next();
                     String plan = resultSet.getString(1);
-                    assertThat(plan).matches(".*ISCAN.*MV1.*\\[\\[GREATER_THAN_OR_EQUALS promote\\(@c24 AS LONG\\)\\]\\].*");
+                    assertThat(plan).matches(".*ISCAN.*MV1.*");
                 }
             }
         }