From 811881bbfb8d046c46fd9afba3f00f489b96d1c2 Mon Sep 17 00:00:00 2001 From: qianheng Date: Wed, 14 Jan 2026 16:37:34 +0800 Subject: [PATCH] Separate explain mode from format params (#5042) * Separate explain mode from format params and support different combination Signed-off-by: Heng Qian * Fix CI Signed-off-by: Heng Qian * Fix CI Signed-off-by: Heng Qian * Fix CI Signed-off-by: Heng Qian * Support explain format BWC Signed-off-by: Heng Qian * Address comments and increase test coverage Signed-off-by: Heng Qian --------- Signed-off-by: Heng Qian (cherry picked from commit 5dbcd285fc6f05620826c2664abda2cff21a4245) Signed-off-by: Heng Qian --- .../opensearch/sql/ast/statement/Explain.java | 24 +-- .../sql/ast/statement/ExplainMode.java | 29 +++ .../sql/executor/ExecutionEngine.java | 4 +- .../opensearch/sql/executor/QueryService.java | 21 ++- .../sql/executor/execution/AbstractPlan.java | 4 +- .../sql/executor/execution/CommandPlan.java | 4 +- .../sql/executor/execution/ExplainPlan.java | 12 +- .../sql/executor/execution/QueryPlan.java | 6 +- .../executor/execution/QueryPlanFactory.java | 5 +- .../sql/executor/QueryServiceTest.java | 8 +- .../executor/execution/CommandPlanTest.java | 6 +- .../executor/execution/ExplainPlanTest.java | 14 +- .../sql/executor/execution/QueryPlanTest.java | 10 +- docs/user/ppl/interfaces/endpoint.md | 13 +- .../sql/calcite/remote/CalciteExplainIT.java | 70 ++++++-- .../opensearch/sql/ppl/PPLIntegTestCase.java | 38 ++-- .../org/opensearch/sql/util/MatcherUtils.java | 5 +- .../explain_extended_for_standardization.json | 6 - .../explain_extended_for_standardization.yaml | 9 + .../calcite/explain_output_cost.json | 6 + .../calcite/explain_output_cost.yaml | 21 +++ .../calcite/explain_output_extended.json | 7 + .../calcite/explain_output_extended.yaml | 167 ++++++++++++++++++ .../calcite/explain_output_simple.json | 5 + .../calcite/explain_output_simple.yaml | 14 ++ .../calcite/explain_output_standard.json | 6 + .../calcite/explain_output_standard.yaml | 21 +++ .../calcite/explain_skip_script_encoding.json | 1 - .../calcite/explain_skip_script_encoding.yaml | 8 + .../sql/legacy/executor/Format.java | 19 +- .../sql/legacy/request/SqlRequestParam.java | 10 +- .../executor/OpenSearchExecutionEngine.java | 10 +- .../request/PPLQueryRequestFactory.java | 32 +++- .../sql/plugin/rest/RestPPLQueryAction.java | 2 +- .../transport/TransportPPLQueryRequest.java | 6 +- .../org/opensearch/sql/ppl/PPLService.java | 1 + .../sql/ppl/domain/PPLQueryRequest.java | 16 +- .../sql/ppl/parser/AstStatementBuilder.java | 3 +- .../sql/ppl/utils/PPLQueryDataAnonymizer.java | 2 +- .../sql/protocol/response/format/Format.java | 27 ++- .../protocol/response/format/FormatTest.java | 3 +- 41 files changed, 523 insertions(+), 152 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/ast/statement/ExplainMode.java delete mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_extended_for_standardization.json create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_extended_for_standardization.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_output_cost.json create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_output_cost.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_output_extended.json create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_output_extended.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_output_simple.json create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_output_simple.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_output_standard.json create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_output_standard.yaml delete mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.json create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.yaml diff --git a/core/src/main/java/org/opensearch/sql/ast/statement/Explain.java b/core/src/main/java/org/opensearch/sql/ast/statement/Explain.java index 1fa64810df7..40c15c953eb 100644 --- a/core/src/main/java/org/opensearch/sql/ast/statement/Explain.java +++ b/core/src/main/java/org/opensearch/sql/ast/statement/Explain.java @@ -8,7 +8,6 @@ package org.opensearch.sql.ast.statement; -import java.util.Locale; import lombok.EqualsAndHashCode; import lombok.Getter; import org.opensearch.sql.ast.AbstractNodeVisitor; @@ -21,37 +20,20 @@ public class Explain extends Statement { private final Statement statement; private final QueryType queryType; - private final ExplainFormat format; + private final ExplainMode mode; public Explain(Statement statement, QueryType queryType) { this(statement, queryType, null); } - public Explain(Statement statement, QueryType queryType, String format) { + public Explain(Statement statement, QueryType queryType, String mode) { this.statement = statement; this.queryType = queryType; - this.format = Explain.format(format); + this.mode = ExplainMode.of(mode); } @Override public R accept(AbstractNodeVisitor visitor, C context) { return visitor.visitExplain(this, context); } - - public enum ExplainFormat { - SIMPLE, - STANDARD, - EXTENDED, - COST, - /** Formats explain output in yaml format. */ - YAML - } - - public static ExplainFormat format(String format) { - try { - return ExplainFormat.valueOf(format.toUpperCase(Locale.ROOT)); - } catch (Exception e) { - return ExplainFormat.STANDARD; - } - } } diff --git a/core/src/main/java/org/opensearch/sql/ast/statement/ExplainMode.java b/core/src/main/java/org/opensearch/sql/ast/statement/ExplainMode.java new file mode 100644 index 00000000000..9043f05929b --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/statement/ExplainMode.java @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.statement; + +import java.util.Locale; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public enum ExplainMode { + SIMPLE("simple"), + STANDARD("standard"), + EXTENDED("extended"), + COST("cost"); + + @Getter private final String modeName; + + public static ExplainMode of(String mode) { + if (mode == null || mode.isEmpty()) return STANDARD; + try { + return ExplainMode.valueOf(mode.toUpperCase(Locale.ROOT)); + } catch (Exception e) { + return ExplainMode.STANDARD; + } + } +} diff --git a/core/src/main/java/org/opensearch/sql/executor/ExecutionEngine.java b/core/src/main/java/org/opensearch/sql/executor/ExecutionEngine.java index ec1e427e365..e65db7b4065 100644 --- a/core/src/main/java/org/opensearch/sql/executor/ExecutionEngine.java +++ b/core/src/main/java/org/opensearch/sql/executor/ExecutionEngine.java @@ -13,7 +13,7 @@ import lombok.RequiredArgsConstructor; import lombok.ToString; import org.apache.calcite.rel.RelNode; -import org.opensearch.sql.ast.statement.Explain; +import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.calcite.CalcitePlanContext; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.data.model.ExprValue; @@ -53,7 +53,7 @@ default void execute( default void explain( RelNode plan, - Explain.ExplainFormat format, + ExplainMode mode, CalcitePlanContext context, ResponseListener listener) {} diff --git a/core/src/main/java/org/opensearch/sql/executor/QueryService.java b/core/src/main/java/org/opensearch/sql/executor/QueryService.java index 54c0e565234..4180258f3c8 100644 --- a/core/src/main/java/org/opensearch/sql/executor/QueryService.java +++ b/core/src/main/java/org/opensearch/sql/executor/QueryService.java @@ -28,7 +28,7 @@ import org.apache.calcite.tools.Programs; import org.opensearch.sql.analysis.AnalysisContext; import org.opensearch.sql.analysis.Analyzer; -import org.opensearch.sql.ast.statement.Explain; +import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.calcite.CalcitePlanContext; import org.opensearch.sql.calcite.CalciteRelNodeVisitor; @@ -83,11 +83,11 @@ public void explain( UnresolvedPlan plan, QueryType queryType, ResponseListener listener, - Explain.ExplainFormat format) { + ExplainMode mode) { if (shouldUseCalcite(queryType)) { - explainWithCalcite(plan, queryType, listener, format); + explainWithCalcite(plan, queryType, listener, mode); } else { - explainWithLegacy(plan, queryType, listener, format, Optional.empty()); + explainWithLegacy(plan, queryType, listener, mode, Optional.empty()); } } @@ -143,7 +143,7 @@ public void explainWithCalcite( UnresolvedPlan plan, QueryType queryType, ResponseListener listener, - Explain.ExplainFormat format) { + ExplainMode mode) { CalcitePlanContext.run( () -> { try { @@ -160,7 +160,7 @@ public void explainWithCalcite( RelNode calcitePlan = convertToCalcitePlan(relNode, context); - executionEngine.explain(calcitePlan, format, context, listener); + executionEngine.explain(calcitePlan, mode, context, listener); }, settings); return null; @@ -168,7 +168,7 @@ public void explainWithCalcite( } catch (Throwable t) { if (isCalciteFallbackAllowed(t)) { log.warn("Fallback to V2 query engine since got exception", t); - explainWithLegacy(plan, queryType, listener, format, Optional.of(t)); + explainWithLegacy(plan, queryType, listener, mode, Optional.of(t)); } else { if (t instanceof Error) { // Calcite may throw AssertError during query execution. @@ -214,13 +214,12 @@ public void explainWithLegacy( UnresolvedPlan plan, QueryType queryType, ResponseListener listener, - Explain.ExplainFormat format, + ExplainMode mode, Optional calciteFailure) { try { - if (format != null - && (format != Explain.ExplainFormat.STANDARD && format != Explain.ExplainFormat.YAML)) { + if (mode != null && (mode != ExplainMode.STANDARD)) { throw new UnsupportedOperationException( - "Explain mode " + format.name() + " is not supported in v2 engine"); + "Explain mode " + mode.name() + " is not supported in v2 engine"); } executionEngine.explain(plan(analyze(plan, queryType)), listener); } catch (Exception e) { diff --git a/core/src/main/java/org/opensearch/sql/executor/execution/AbstractPlan.java b/core/src/main/java/org/opensearch/sql/executor/execution/AbstractPlan.java index caf23b51b14..a3ecf8d81f4 100644 --- a/core/src/main/java/org/opensearch/sql/executor/execution/AbstractPlan.java +++ b/core/src/main/java/org/opensearch/sql/executor/execution/AbstractPlan.java @@ -10,7 +10,7 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; -import org.opensearch.sql.ast.statement.Explain; +import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.QueryId; @@ -34,5 +34,5 @@ public abstract class AbstractPlan { * @param listener query explain response listener. */ public abstract void explain( - ResponseListener listener, Explain.ExplainFormat format); + ResponseListener listener, ExplainMode mode); } diff --git a/core/src/main/java/org/opensearch/sql/executor/execution/CommandPlan.java b/core/src/main/java/org/opensearch/sql/executor/execution/CommandPlan.java index bd333806413..f990589dc99 100644 --- a/core/src/main/java/org/opensearch/sql/executor/execution/CommandPlan.java +++ b/core/src/main/java/org/opensearch/sql/executor/execution/CommandPlan.java @@ -8,7 +8,7 @@ package org.opensearch.sql.executor.execution; -import org.opensearch.sql.ast.statement.Explain; +import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.executor.ExecutionEngine; @@ -50,7 +50,7 @@ public void execute() { @Override public void explain( - ResponseListener listener, Explain.ExplainFormat format) { + ResponseListener listener, ExplainMode mode) { throw new UnsupportedOperationException("CommandPlan does not support explain"); } } diff --git a/core/src/main/java/org/opensearch/sql/executor/execution/ExplainPlan.java b/core/src/main/java/org/opensearch/sql/executor/execution/ExplainPlan.java index 61d6f2f8e5c..4e53a5a0529 100644 --- a/core/src/main/java/org/opensearch/sql/executor/execution/ExplainPlan.java +++ b/core/src/main/java/org/opensearch/sql/executor/execution/ExplainPlan.java @@ -8,7 +8,7 @@ package org.opensearch.sql.executor.execution; -import org.opensearch.sql.ast.statement.Explain; +import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.QueryId; @@ -18,7 +18,7 @@ public class ExplainPlan extends AbstractPlan { private final AbstractPlan plan; - private final Explain.ExplainFormat format; + private final ExplainMode mode; private final ResponseListener explainListener; @@ -27,22 +27,22 @@ public ExplainPlan( QueryId queryId, QueryType queryType, AbstractPlan plan, - Explain.ExplainFormat format, + ExplainMode mode, ResponseListener explainListener) { super(queryId, queryType); this.plan = plan; - this.format = format; + this.mode = mode; this.explainListener = explainListener; } @Override public void execute() { - plan.explain(explainListener, format); + plan.explain(explainListener, mode); } @Override public void explain( - ResponseListener listener, Explain.ExplainFormat format) { + ResponseListener listener, ExplainMode mode) { throw new UnsupportedOperationException("explain query can not been explained."); } } diff --git a/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlan.java b/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlan.java index bc05fe50d4d..a2363724f8b 100644 --- a/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlan.java +++ b/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlan.java @@ -10,7 +10,7 @@ import java.util.Optional; import org.apache.commons.lang3.NotImplementedException; -import org.opensearch.sql.ast.statement.Explain; +import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.ast.tree.Paginate; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.response.ResponseListener; @@ -72,13 +72,13 @@ public void execute() { @Override public void explain( - ResponseListener listener, Explain.ExplainFormat format) { + ResponseListener listener, ExplainMode mode) { if (pageSize.isPresent()) { listener.onFailure( new NotImplementedException( "`explain` feature for paginated requests is not implemented yet.")); } else { - queryService.explain(plan, getQueryType(), listener, format); + queryService.explain(plan, getQueryType(), listener, mode); } } } diff --git a/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java b/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java index 46f8c77a767..b8dd9b3a8fc 100644 --- a/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java +++ b/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java @@ -14,6 +14,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.opensearch.sql.ast.AbstractNodeVisitor; import org.opensearch.sql.ast.statement.Explain; +import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.ast.statement.Query; import org.opensearch.sql.ast.statement.Statement; import org.opensearch.sql.ast.tree.CloseCursor; @@ -80,7 +81,7 @@ public AbstractPlan create( new QueryPlan( queryId, queryType, new FetchCursor(cursor), queryService, queryResponseListener); return isExplain - ? new ExplainPlan(queryId, queryType, plan, Explain.format(format), explainListener) + ? new ExplainPlan(queryId, queryType, plan, ExplainMode.of(format), explainListener) : plan; } @@ -140,7 +141,7 @@ public AbstractPlan visitExplain( QueryId.queryId(), node.getQueryType(), create(node.getStatement(), NO_CONSUMER_RESPONSE_LISTENER, context.getRight()), - node.getFormat(), + node.getMode(), context.getRight()); } } diff --git a/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java b/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java index 6a3a1c6dceb..b0e03c69a01 100644 --- a/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java @@ -23,7 +23,7 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.sql.analysis.Analyzer; -import org.opensearch.sql.ast.statement.Explain; +import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.common.setting.Settings; @@ -62,7 +62,7 @@ class QueryServiceTest { @Mock private Split split; - private final Explain.ExplainFormat format = Explain.ExplainFormat.STANDARD; + private final ExplainMode mode = ExplainMode.STANDARD; @Test public void executeWithoutContext() { @@ -225,7 +225,7 @@ public void onFailure(Exception e) { fail(); } }, - format); + mode); } void handledByExplainOnFailure() { @@ -243,7 +243,7 @@ public void onFailure(Exception e) { assertTrue(e instanceof IllegalStateException); } }, - format); + mode); } } } diff --git a/core/src/test/java/org/opensearch/sql/executor/execution/CommandPlanTest.java b/core/src/test/java/org/opensearch/sql/executor/execution/CommandPlanTest.java index d9149c8da27..d7cab2c6f10 100644 --- a/core/src/test/java/org/opensearch/sql/executor/execution/CommandPlanTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/execution/CommandPlanTest.java @@ -22,7 +22,7 @@ import org.junit.jupiter.api.DisplayNameGeneration; import org.junit.jupiter.api.DisplayNameGenerator; import org.junit.jupiter.api.Test; -import org.opensearch.sql.ast.statement.Explain; +import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.executor.QueryId; @@ -66,7 +66,7 @@ public void explain_not_supported() { QueryService qs = mock(QueryService.class); ResponseListener listener = mock(ResponseListener.class); ResponseListener explainListener = mock(ResponseListener.class); - Explain.ExplainFormat format = mock(Explain.ExplainFormat.class); + ExplainMode mode = mock(ExplainMode.class); var exception = assertThrows( @@ -78,7 +78,7 @@ public void explain_not_supported() { mock(UnresolvedPlan.class), qs, listener) - .explain(explainListener, format)); + .explain(explainListener, mode)); assertEquals("CommandPlan does not support explain", exception.getMessage()); verify(listener, never()).onResponse(any()); diff --git a/core/src/test/java/org/opensearch/sql/executor/execution/ExplainPlanTest.java b/core/src/test/java/org/opensearch/sql/executor/execution/ExplainPlanTest.java index 7d025f9ad9d..e462b2f4e36 100644 --- a/core/src/test/java/org/opensearch/sql/executor/execution/ExplainPlanTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/execution/ExplainPlanTest.java @@ -19,7 +19,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.sql.ast.statement.Explain; +import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.QueryId; @@ -35,29 +35,27 @@ public class ExplainPlanTest { @Mock private ResponseListener explainListener; - @Mock private Explain.ExplainFormat format; + @Mock private ExplainMode mode; @Test public void execute() { doNothing().when(queryPlan).explain(any(), any()); - ExplainPlan explainPlan = - new ExplainPlan(queryId, queryType, queryPlan, format, explainListener); + ExplainPlan explainPlan = new ExplainPlan(queryId, queryType, queryPlan, mode, explainListener); explainPlan.execute(); - verify(queryPlan, times(1)).explain(explainListener, format); + verify(queryPlan, times(1)).explain(explainListener, mode); } @Test public void explainThrowException() { - ExplainPlan explainPlan = - new ExplainPlan(queryId, queryType, queryPlan, format, explainListener); + ExplainPlan explainPlan = new ExplainPlan(queryId, queryType, queryPlan, mode, explainListener); UnsupportedOperationException unsupportedExplainException = assertThrows( UnsupportedOperationException.class, () -> { - explainPlan.explain(explainListener, format); + explainPlan.explain(explainListener, mode); }); assertEquals("explain query can not been explained.", unsupportedExplainException.getMessage()); } diff --git a/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanTest.java b/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanTest.java index 437448f2853..11d507339c5 100644 --- a/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanTest.java @@ -23,7 +23,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.sql.ast.statement.Explain; +import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.executor.DefaultExecutionEngine; @@ -48,7 +48,7 @@ class QueryPlanTest { @Mock private ResponseListener queryListener; - @Mock private Explain.ExplainFormat format; + @Mock private ExplainMode mode; @Test public void execute_no_page_size() { @@ -61,9 +61,9 @@ public void execute_no_page_size() { @Test public void explain_no_page_size() { QueryPlan query = new QueryPlan(queryId, queryType, plan, queryService, queryListener); - query.explain(explainListener, format); + query.explain(explainListener, mode); - verify(queryService, times(1)).explain(plan, queryType, explainListener, format); + verify(queryService, times(1)).explain(plan, queryType, explainListener, mode); } @Test @@ -127,6 +127,6 @@ public void onFailure(Exception e) { assertTrue(e instanceof NotImplementedException); } }, - format); + mode); } } diff --git a/docs/user/ppl/interfaces/endpoint.md b/docs/user/ppl/interfaces/endpoint.md index 098698f4263..996ec692fef 100644 --- a/docs/user/ppl/interfaces/endpoint.md +++ b/docs/user/ppl/interfaces/endpoint.md @@ -61,7 +61,8 @@ You can send HTTP explain request to endpoint **/_plugins/_ppl/_explain** with y ### Description To translate your query, send it to explain endpoint. The explain output is OpenSearch domain specific language (DSL) in JSON format. You can just copy and paste it to your console to run it against OpenSearch directly. -Explain output could be set different formats: `standard` (the default format), `simple`, `extended`, `dsl`. +Explain API supports various mode: `standard` (the default format), `simple`, `extended`, `cost`. +And the explain output could be shown in different formats: `json` (the default format), `yaml` ### Example 1 default (standard) format Explain query @@ -84,13 +85,13 @@ Expected output: ``` -### Example 2 simple format +### Example 2 simple mode Explain query ```bash ppl curl -sS -H 'Content-Type: application/json' \ --X POST localhost:9200/_plugins/_ppl/_explain?format=simple \ +-X POST localhost:9200/_plugins/_ppl/_explain?mode=simple \ -d '{"query" : "source=state_country | where age>30"}' ``` @@ -104,13 +105,13 @@ Expected output: } ``` -### Example 3 extended format +### Example 3 extended mode Explain query ```bash ppl ignore curl -sS -H 'Content-Type: application/json' \ --X POST localhost:9200/_plugins/_ppl/_explain?format=extended \ +-X POST localhost:9200/_plugins/_ppl/_explain?mode=extended \ -d '{"query" : "source=state_country | head 10 | where age>30"}' ``` @@ -130,7 +131,7 @@ Expected output: YAML explain output is an experimental feature and not intended for production use. The interface and output may change without notice. -Return Explain response format in In `yaml` format. +Return Explain response in `yaml` format. Explain query ```bash ppl ignore diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 9616dcbd7bd..7c51c0076f7 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -25,8 +25,10 @@ import java.util.Locale; import org.junit.Ignore; import org.junit.Test; -import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.ast.statement.ExplainMode; +import org.opensearch.sql.common.setting.Settings.Key; import org.opensearch.sql.ppl.ExplainIT; +import org.opensearch.sql.protocol.response.format.Format; public class CalciteExplainIT extends ExplainIT { @Override @@ -382,9 +384,9 @@ public void testSkipScriptEncodingOnExtendedFormat() throws IOException { String query = "source=opensearch-sql_test_index_account | where address = '671 Bristol Street' and age -" + " 2 = 30 | fields firstname, age, address"; - var result = explainQueryToString(query, true); - String expected = loadFromFile("expectedOutput/calcite/explain_skip_script_encoding.json"); - assertJsonEqualsIgnoreId(expected, result); + var result = explainQueryYaml(query, ExplainMode.EXTENDED); + String expected = loadFromFile("expectedOutput/calcite/explain_skip_script_encoding.yaml"); + assertYamlEqualsIgnoreId(expected, result); } // Only for Calcite, as v2 gets unstable serialized string for function @@ -1720,7 +1722,7 @@ public void testExplainRareCommandUseNull() throws IOException { explainQueryYaml( String.format("source=%s | rare 2 usenull=true state by gender", TEST_INDEX_ACCOUNT))); withSettings( - Settings.Key.PPL_SYNTAX_LEGACY_PREFERRED, + Key.PPL_SYNTAX_LEGACY_PREFERRED, "false", () -> { try { @@ -1747,7 +1749,7 @@ public void testExplainTopCommandUseNull() throws IOException { explainQueryYaml( String.format("source=%s | top 2 usenull=true state by gender", TEST_INDEX_ACCOUNT))); withSettings( - Settings.Key.PPL_SYNTAX_LEGACY_PREFERRED, + Key.PPL_SYNTAX_LEGACY_PREFERRED, "false", () -> { try { @@ -2207,14 +2209,14 @@ public void testAliasTypeField() throws IOException { @Test public void testRexStandardizationForScript() throws IOException { enabledOnlyWhenPushdownIsEnabled(); - assertJsonEqualsIgnoreId( - loadExpectedPlan("explain_extended_for_standardization.json"), - explainQueryToString( + assertYamlEqualsIgnoreId( + loadExpectedPlan("explain_extended_for_standardization.yaml"), + explainQueryYaml( String.format( "source=%s | eval age_range = case(age < 30, 'u30', age >= 30 and age <= 40, 'u40'" + " else 'u100') | stats avg(age) as avg_age by age_range", TEST_INDEX_BANK), - true)); + ExplainMode.EXTENDED)); } @Test @@ -2318,4 +2320,52 @@ public void testNotBetweenPushDownExplain() throws Exception { explainQueryYaml( "source=opensearch-sql_test_index_bank | where age not between 30 and 39")); } + + @Test + public void testExplainInVariousModeAndFormat() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + String query = + "source=opensearch-sql_test_index_account" + + "| where age > 30 " + + "| stats avg(age) AS avg_age by state, city " + + "| sort state " + + "| fields - city " + + "| eval age2 = avg_age + 2 " + + "| dedup age2 " + + "| fields age2"; + ExplainMode[] explainModes = + new ExplainMode[] { + ExplainMode.SIMPLE, ExplainMode.STANDARD, ExplainMode.EXTENDED, ExplainMode.COST + }; + for (ExplainMode explainMode : explainModes) { + String modeName = explainMode.getModeName().toLowerCase(Locale.ROOT); + assertYamlEqualsIgnoreId( + loadExpectedPlan(String.format("explain_output_%s.yaml", modeName)), + explainQueryYaml(query, explainMode)); + assertJsonEqualsIgnoreId( + loadExpectedPlan(String.format("explain_output_%s.json", modeName)), + explainQueryToString(query, explainMode)); + } + } + + @Test + public void testExplainBWC() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + String query = + "source=opensearch-sql_test_index_account" + + "| where age > 30 " + + "| stats avg(age) AS avg_age by state, city " + + "| sort state " + + "| fields - city " + + "| eval age2 = avg_age + 2 " + + "| dedup age2 " + + "| fields age2"; + Format[] formats = new Format[] {Format.SIMPLE, Format.STANDARD, Format.EXTENDED, Format.COST}; + for (Format format : formats) { + String formatName = format.getFormatName().toLowerCase(Locale.ROOT); + assertJsonEqualsIgnoreId( + loadExpectedPlan(String.format("explain_output_%s.json", formatName)), + explainQueryToStringBWC(query, format)); + } + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java index 831a8bda0e9..8b6bd359661 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java @@ -6,7 +6,6 @@ package org.opensearch.sql.ppl; import static org.opensearch.sql.legacy.TestUtils.getResponseBody; -import static org.opensearch.sql.plugin.rest.RestPPLQueryAction.EXPLAIN_API_ENDPOINT; import static org.opensearch.sql.plugin.rest.RestPPLQueryAction.QUERY_API_ENDPOINT; import com.google.common.io.Resources; @@ -27,16 +26,17 @@ import org.opensearch.client.Response; import org.opensearch.client.ResponseException; import org.opensearch.common.collect.MapBuilder; +import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.common.setting.Settings.Key; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.protocol.response.format.Format; import org.opensearch.sql.util.RetryProcessor; /** OpenSearch Rest integration test base for PPL testing. */ public abstract class PPLIntegTestCase extends SQLIntegTestCase { - private static final String EXTENDED_EXPLAIN_API_ENDPOINT = - "/_plugins/_ppl/_explain?format=extended"; - private static final String YAML_EXPLAIN_API_ENDPOINT = "/_plugins/_ppl/_explain?format=yaml"; + private static final String BWC_EXPLAIN_API_ENDPOINT = "/_plugins/_ppl/_explain?format=%s"; + private static final String EXPLAIN_API_ENDPOINT = "/_plugins/_ppl/_explain?format=%s&mode=%s"; private static final Logger LOG = LogManager.getLogger(); @Rule public final RetryProcessor retryProcessor = new RetryProcessor(); public static final Integer DEFAULT_SUBSEARCH_MAXOUT = 10000; @@ -62,25 +62,35 @@ protected String executeQueryToString(String query) throws IOException { /** Deprecated, use {@link #explainQueryYaml(String)} */ @Deprecated protected String explainQueryToString(String query) throws IOException { - return explainQueryToString(query, false); + return explainQueryToString(query, ExplainMode.STANDARD); } protected String explainQueryYaml(String query) throws IOException { - Response response = client().performRequest(buildRequest(query, YAML_EXPLAIN_API_ENDPOINT)); + return explainQueryYaml(query, ExplainMode.STANDARD); + } + + protected String explainQueryYaml(String query, ExplainMode mode) throws IOException { + return explainQuery(query, Format.YAML, mode); + } + + protected String explainQueryToString(String query, ExplainMode mode) throws IOException { + return explainQuery(query, Format.JSON, mode).replace("\\r\\n", "\\n"); + } + + private String explainQuery(String query, Format format, ExplainMode mode) throws IOException { + Response response = + client() + .performRequest(buildRequest(query, String.format(EXPLAIN_API_ENDPOINT, format, mode))); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); - String responseBody = getResponseBody(response, true); - return responseBody; + return getResponseBody(response, true); } - protected String explainQueryToString(String query, boolean extended) throws IOException { + protected String explainQueryToStringBWC(String query, Format format) throws IOException { Response response = client() - .performRequest( - buildRequest( - query, extended ? EXTENDED_EXPLAIN_API_ENDPOINT : EXPLAIN_API_ENDPOINT)); + .performRequest(buildRequest(query, String.format(BWC_EXPLAIN_API_ENDPOINT, format))); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); - String responseBody = getResponseBody(response, true); - return responseBody.replace("\\r\\n", "\\n"); + return getResponseBody(response, true).replace("\\r\\n", "\\n"); } protected String executeCsvQuery(String query, boolean sanitize) throws IOException { diff --git a/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java b/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java index 3fc7634c574..979de6dba2a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java @@ -400,7 +400,8 @@ private static String eliminateTimeStamp(String s) { private static String eliminateRelId(String s) { return s.replaceAll("rel#\\d+", "rel#") .replaceAll("RelSubset#\\d+", "RelSubset#") - .replaceAll("LogicalProject#\\d+", "LogicalProject#"); + .replaceAll("LogicalProject#\\d+", "LogicalProject#") + .replaceAll("id = \\d+", "id = *"); } private static String eliminateRequestOptions(String s) { @@ -460,6 +461,8 @@ private static String cleanUpYaml(String s) { .replaceAll("\\$t?\\d+", "\\$FIELD_INDEX") .replaceAll(" needClean=true,", "") .replaceAll(" searchDone=false,", "") + .replaceAll("id = \\d+", "id = *") + .replaceAll(" searchDone=false,", "") .replaceAll("\\([^)]*\\.\\.\\d+\\)", "?") .replaceAll("\\(\\d+\\.\\.[^)]*\\)", "?"); } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_extended_for_standardization.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_extended_for_standardization.json deleted file mode 100644 index e56186feb95..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_extended_for_standardization.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(avg_age=[$1], age_range=[$0])\n LogicalAggregate(group=[{0}], avg_age=[AVG($1)])\n LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, SEARCH($10, Sarg[[30..40]]), 'u40':VARCHAR, 'u100':VARCHAR)], age=[$10])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg_age=AVG($1)), PROJECT->[avg_age, age_range], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"age_range\":{\"terms\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"{\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"CASE\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"CASE\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"SPECIAL\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"<\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"LESS_THAN\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 0,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n },\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 1,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n }\\\\n ]\\\\n },\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 2,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"VARCHAR\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true,\\\\n \\\\\\\"precision\\\\\\\": -1\\\\n }\\\\n },\\\\n {\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"AND\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"AND\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"<=\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"LESS_THAN_OR_EQUAL\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 3,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n },\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 4,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n }\\\\n ]\\\\n },\\\\n {\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"<=\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"LESS_THAN_OR_EQUAL\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 5,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n },\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 6,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n },\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 7,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"VARCHAR\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true,\\\\n \\\\\\\"precision\\\\\\\": -1\\\\n }\\\\n },\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 8,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"VARCHAR\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true,\\\\n \\\\\\\"precision\\\\\\\": -1\\\\n }\\\\n }\\\\n ]\\\\n}\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*,\"SOURCES\":[0,2,2,2,0,0,2,2,2],\"DIGESTS\":[\"age\",30,\"u30\",30,\"age\",\"age\",40,\"u40\",\"u100\"]}},\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_extended_for_standardization.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_extended_for_standardization.yaml new file mode 100644 index 00000000000..73586df5e66 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_extended_for_standardization.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg_age=[$1], age_range=[$0]) + LogicalAggregate(group=[{0}], avg_age=[AVG($1)]) + LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, SEARCH($10, Sarg[[30..40]]), 'u40':VARCHAR, 'u100':VARCHAR)], age=[$10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg_age=AVG($1)), PROJECT->[avg_age, age_range], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age_range":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"{\\n \\\"op\\\": {\\n \\\"name\\\": \\\"CASE\\\",\\n \\\"kind\\\": \\\"CASE\\\",\\n \\\"syntax\\\": \\\"SPECIAL\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"op\\\": {\\n \\\"name\\\": \\\"<\\\",\\n \\\"kind\\\": \\\"LESS_THAN\\\",\\n \\\"syntax\\\": \\\"BINARY\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"dynamicParam\\\": 0,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n },\\n {\\n \\\"dynamicParam\\\": 1,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n }\\n ]\\n },\\n {\\n \\\"dynamicParam\\\": 2,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"VARCHAR\\\",\\n \\\"nullable\\\": true,\\n \\\"precision\\\": -1\\n }\\n },\\n {\\n \\\"op\\\": {\\n \\\"name\\\": \\\"AND\\\",\\n \\\"kind\\\": \\\"AND\\\",\\n \\\"syntax\\\": \\\"BINARY\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"op\\\": {\\n \\\"name\\\": \\\"<=\\\",\\n \\\"kind\\\": \\\"LESS_THAN_OR_EQUAL\\\",\\n \\\"syntax\\\": \\\"BINARY\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"dynamicParam\\\": 3,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n },\\n {\\n \\\"dynamicParam\\\": 4,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n }\\n ]\\n },\\n {\\n \\\"op\\\": {\\n \\\"name\\\": \\\"<=\\\",\\n \\\"kind\\\": \\\"LESS_THAN_OR_EQUAL\\\",\\n \\\"syntax\\\": \\\"BINARY\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"dynamicParam\\\": 5,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n },\\n {\\n \\\"dynamicParam\\\": 6,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n }\\n ]\\n }\\n ]\\n },\\n {\\n \\\"dynamicParam\\\": 7,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"VARCHAR\\\",\\n \\\"nullable\\\": true,\\n \\\"precision\\\": -1\\n }\\n },\\n {\\n \\\"dynamicParam\\\": 8,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"VARCHAR\\\",\\n \\\"nullable\\\": true,\\n \\\"precision\\\": -1\\n }\\n }\\n ]\\n}\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2,2,0,0,2,2,2],"DIGESTS":["age",30,"u30",30,"age","age",40,"u40","u100"]}},"missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_output_cost.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_cost.json new file mode 100644 index 00000000000..6235593dc6f --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_cost.json @@ -0,0 +1,6 @@ +{ + "calcite": { + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]): rowcount = 56.25, cumulative cost = {165681.25 rows, 105156.5471810663 cpu, 0.0 io}, id = 9401\n LogicalProject(age2=[$2]): rowcount = 56.25, cumulative cost = {165625.0 rows, 104256.5471810663 cpu, 0.0 io}, id = 9400\n LogicalFilter(condition=[<=($3, 1)]): rowcount = 56.25, cumulative cost = {165568.75 rows, 104200.2971810663 cpu, 0.0 io}, id = 9398\n LogicalProject(avg_age=[$0], state=[$1], age2=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $2)]): rowcount = 225.0, cumulative cost = {165512.5 rows, 103975.2971810663 cpu, 0.0 io}, id = 9397\n LogicalFilter(condition=[IS NOT NULL($2)]): rowcount = 225.0, cumulative cost = {165287.5 rows, 103075.2971810663 cpu, 0.0 io}, id = 9396\n LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)]): rowcount = 500.0, cumulative cost = {165062.5 rows, 102575.2971810663 cpu, 0.0 io}, id = 9395\n LogicalSort(sort0=[$1], dir0=[ASC-nulls-first]): rowcount = 500.0, cumulative cost = {164562.5 rows, 101075.2971810663 cpu, 0.0 io}, id = 9393\n LogicalProject(avg_age=[$2], state=[$0], city=[$1]): rowcount = 500.0, cumulative cost = {164062.5 rows, 26500.0 cpu, 0.0 io}, id = 9392\n LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]): rowcount = 500.0, cumulative cost = {163562.5 rows, 25000.0 cpu, 0.0 io}, id = 9391\n LogicalProject(state=[$7], city=[$5], age=[$8]): rowcount = 5000.0, cumulative cost = {163000.0 rows, 25000.0 cpu, 0.0 io}, id = 9390\n LogicalFilter(condition=[>($8, 30)]): rowcount = 5000.0, cumulative cost = {158000.0 rows, 10000.0 cpu, 0.0 io}, id = 9389\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]): rowcount = 10000.0, cumulative cost = {153000.0 rows, 0.0 cpu, 0.0 io}, id = 9388\n", + "physical": "EnumerableCalc(expr#0..1=[{inputs}], age2=[$t0]): rowcount = 225.0, cumulative cost = {2981.25 rows, 7950.0 cpu, 0.0 io}, id = 11254\n EnumerableLimit(fetch=[10000]): rowcount = 225.0, cumulative cost = {2756.25 rows, 7275.0 cpu, 0.0 io}, id = 11246\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=[<=($t1, $t2)], proj#0..1=[{exprs}], $condition=[$t3]): rowcount = 225.0, cumulative cost = {2531.25 rows, 7050.0 cpu, 0.0 io}, id = 11250\n EnumerableWindow(window#0=[window(partition {0} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]): rowcount = 450.0, cumulative cost = {2306.25 rows, 3900.0 cpu, 0.0 io}, id = 11242\n EnumerableCalc(expr#0=[{inputs}], expr#1=[2], expr#2=[+($t0, $t1)], expr#3=[IS NOT NULL($t0)], $0=[$t2], $condition=[$t3]): rowcount = 450.0, cumulative cost = {1856.25 rows, 3000.0 cpu, 0.0 io}, id = 11258\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[avg_age, state], SORT->[1 ASC FIRST], PROJECT->[avg_age]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]): rowcount = 500.0, cumulative cost = {1406.25 rows, 0.0 cpu, 0.0 io}, id = 10962\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_output_cost.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_cost.yaml new file mode 100644 index 00000000000..63b402833fc --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_cost.yaml @@ -0,0 +1,21 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]): rowcount = 56.25, cumulative cost = {165681.25 rows, 105156.5471810663 cpu, 0.0 io}, id = 7529 + LogicalProject(age2=[$2]): rowcount = 56.25, cumulative cost = {165625.0 rows, 104256.5471810663 cpu, 0.0 io}, id = 7528 + LogicalFilter(condition=[<=($3, 1)]): rowcount = 56.25, cumulative cost = {165568.75 rows, 104200.2971810663 cpu, 0.0 io}, id = 7526 + LogicalProject(avg_age=[$0], state=[$1], age2=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $2)]): rowcount = 225.0, cumulative cost = {165512.5 rows, 103975.2971810663 cpu, 0.0 io}, id = 7525 + LogicalFilter(condition=[IS NOT NULL($2)]): rowcount = 225.0, cumulative cost = {165287.5 rows, 103075.2971810663 cpu, 0.0 io}, id = 7524 + LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)]): rowcount = 500.0, cumulative cost = {165062.5 rows, 102575.2971810663 cpu, 0.0 io}, id = 7523 + LogicalSort(sort0=[$1], dir0=[ASC-nulls-first]): rowcount = 500.0, cumulative cost = {164562.5 rows, 101075.2971810663 cpu, 0.0 io}, id = 7521 + LogicalProject(avg_age=[$2], state=[$0], city=[$1]): rowcount = 500.0, cumulative cost = {164062.5 rows, 26500.0 cpu, 0.0 io}, id = 7520 + LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]): rowcount = 500.0, cumulative cost = {163562.5 rows, 25000.0 cpu, 0.0 io}, id = 7519 + LogicalProject(state=[$7], city=[$5], age=[$8]): rowcount = 5000.0, cumulative cost = {163000.0 rows, 25000.0 cpu, 0.0 io}, id = 7518 + LogicalFilter(condition=[>($8, 30)]): rowcount = 5000.0, cumulative cost = {158000.0 rows, 10000.0 cpu, 0.0 io}, id = 7517 + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]): rowcount = 10000.0, cumulative cost = {153000.0 rows, 0.0 cpu, 0.0 io}, id = 7516 + physical: | + EnumerableCalc(expr#0..1=[{inputs}], age2=[$t0]): rowcount = 225.0, cumulative cost = {2981.25 rows, 7950.0 cpu, 0.0 io}, id = 9382 + EnumerableLimit(fetch=[10000]): rowcount = 225.0, cumulative cost = {2756.25 rows, 7275.0 cpu, 0.0 io}, id = 9374 + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=[<=($t1, $t2)], proj#0..1=[{exprs}], $condition=[$t3]): rowcount = 225.0, cumulative cost = {2531.25 rows, 7050.0 cpu, 0.0 io}, id = 9378 + EnumerableWindow(window#0=[window(partition {0} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]): rowcount = 450.0, cumulative cost = {2306.25 rows, 3900.0 cpu, 0.0 io}, id = 9370 + EnumerableCalc(expr#0=[{inputs}], expr#1=[2], expr#2=[+($t0, $t1)], expr#3=[IS NOT NULL($t0)], $0=[$t2], $condition=[$t3]): rowcount = 450.0, cumulative cost = {1856.25 rows, 3000.0 cpu, 0.0 io}, id = 9386 + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[avg_age, state], SORT->[1 ASC FIRST], PROJECT->[avg_age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"city":{"terms":{"field":"city.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]): rowcount = 500.0, cumulative cost = {1406.25 rows, 0.0 cpu, 0.0 io}, id = 9090 diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_output_extended.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_extended.json new file mode 100644 index 00000000000..5ce93804ca4 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_extended.json @@ -0,0 +1,7 @@ +{ + "calcite": { + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age2=[$2])\n LogicalFilter(condition=[<=($3, 1)])\n LogicalProject(avg_age=[$0], state=[$1], age2=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $2)])\n LogicalFilter(condition=[IS NOT NULL($2)])\n LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)])\n LogicalSort(sort0=[$1], dir0=[ASC-nulls-first])\n LogicalProject(avg_age=[$2], state=[$0], city=[$1])\n LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)])\n LogicalProject(state=[$7], city=[$5], age=[$8])\n LogicalFilter(condition=[>($8, 30)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableCalc(expr#0..1=[{inputs}], age2=[$t0])\n EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=[<=($t1, $t2)], proj#0..1=[{exprs}], $condition=[$t3])\n EnumerableWindow(window#0=[window(partition {0} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])])\n EnumerableCalc(expr#0=[{inputs}], expr#1=[2], expr#2=[+($t0, $t1)], expr#3=[IS NOT NULL($t0)], $0=[$t2], $condition=[$t3])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[avg_age, state], SORT->[1 ASC FIRST], PROJECT->[avg_age]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n", + "extended": "public org.apache.calcite.linq4j.Enumerable bind(final org.apache.calcite.DataContext root) {\n final org.opensearch.sql.opensearch.storage.scan.CalciteEnumerableIndexScan v1stashed = (org.opensearch.sql.opensearch.storage.scan.CalciteEnumerableIndexScan) root.get(\"v1stashed\");\n final org.apache.calcite.linq4j.Enumerable _inputEnumerable = v1stashed.scan();\n final org.apache.calcite.linq4j.AbstractEnumerable source = new org.apache.calcite.linq4j.AbstractEnumerable(){\n public org.apache.calcite.linq4j.Enumerator enumerator() {\n return new org.apache.calcite.linq4j.Enumerator(){\n public final org.apache.calcite.linq4j.Enumerator inputEnumerator = _inputEnumerable.enumerator();\n public void reset() {\n inputEnumerator.reset();\n }\n\n public boolean moveNext() {\n while (inputEnumerator.moveNext()) {\n if ((Double) inputEnumerator.current() != null) {\n return true;\n }\n }\n return false;\n }\n\n public void close() {\n inputEnumerator.close();\n }\n\n public Object current() {\n return (Double) inputEnumerator.current() == null ? null : Double.valueOf(((Double) inputEnumerator.current()).doubleValue() + (double) 2);\n }\n\n };\n }\n\n };\n int prevStart;\n int prevEnd;\n final java.util.Comparator comparator = new java.util.Comparator(){\n public int compare(Double v0, Double v1) {\n int c;\n return 0;\n }\n\n public int compare(Object o0, Object o1) {\n return this.compare((Double) o0, (Double) o1);\n }\n\n };\n final org.apache.calcite.runtime.SortedMultiMap multiMap = new org.apache.calcite.runtime.SortedMultiMap();\n source.foreach(new org.apache.calcite.linq4j.function.Function1() {\n public Object apply(Double v) {\n Double key = v;\n multiMap.putMulti(key, v);\n return null;\n }\n public Object apply(Object v) {\n return apply(\n (Double) v);\n }\n }\n );\n final java.util.Iterator iterator = multiMap.arrays(comparator);\n final java.util.ArrayList _list = new java.util.ArrayList(\n multiMap.size());\n Long a0w0 = (Long) null;\n while (iterator.hasNext()) {\n final Object[] _rows = (Object[]) iterator.next();\n prevStart = -1;\n prevEnd = 2147483647;\n for (int i = 0; i < _rows.length; (++i)) {\n if (i != prevEnd) {\n int actualStart = i < prevEnd ? 0 : prevEnd + 1;\n prevEnd = i;\n a0w0 = Long.valueOf(((Number)org.apache.calcite.linq4j.tree.Primitive.of(long.class).numberValueRoundDown((i - 0 + 1))).longValue());\n }\n _list.add(new Object[] {\n (Double) _rows[i],\n a0w0});\n }\n }\n multiMap.clear();\n final org.apache.calcite.linq4j.Enumerable _inputEnumerable0 = org.apache.calcite.linq4j.Linq4j.asEnumerable(_list);\n final org.apache.calcite.linq4j.AbstractEnumerable child = new org.apache.calcite.linq4j.AbstractEnumerable(){\n public org.apache.calcite.linq4j.Enumerator enumerator() {\n return new org.apache.calcite.linq4j.Enumerator(){\n public final org.apache.calcite.linq4j.Enumerator inputEnumerator = _inputEnumerable0.enumerator();\n public void reset() {\n inputEnumerator.reset();\n }\n\n public boolean moveNext() {\n while (inputEnumerator.moveNext()) {\n if (org.apache.calcite.runtime.SqlFunctions.toLong(((Object[]) inputEnumerator.current())[1]) <= $L4J$C$_Number_org_apache_calcite_linq4j_tree_Primitive_of_long_class_358aa52b) {\n return true;\n }\n }\n return false;\n }\n\n public void close() {\n inputEnumerator.close();\n }\n\n public Object current() {\n final Object[] current = (Object[]) inputEnumerator.current();\n final Object input_value = current[0];\n final Object input_value0 = current[1];\n return new Object[] {\n input_value,\n input_value0};\n }\n\n static final long $L4J$C$_Number_org_apache_calcite_linq4j_tree_Primitive_of_long_class_358aa52b = ((Number)org.apache.calcite.linq4j.tree.Primitive.of(long.class).numberValueRoundDown(1)).longValue();\n };\n }\n\n };\n final org.apache.calcite.linq4j.Enumerable _inputEnumerable1 = child.take(10000);\n return new org.apache.calcite.linq4j.AbstractEnumerable(){\n public org.apache.calcite.linq4j.Enumerator enumerator() {\n return new org.apache.calcite.linq4j.Enumerator(){\n public final org.apache.calcite.linq4j.Enumerator inputEnumerator = _inputEnumerable1.enumerator();\n public void reset() {\n inputEnumerator.reset();\n }\n\n public boolean moveNext() {\n return inputEnumerator.moveNext();\n }\n\n public void close() {\n inputEnumerator.close();\n }\n\n public Object current() {\n return (Double) ((Object[]) inputEnumerator.current())[0];\n }\n\n };\n }\n\n };\n}\n\n\npublic Class getElementType() {\n return java.lang.Double.class;\n}\n\n\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_output_extended.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_extended.yaml new file mode 100644 index 00000000000..89137cfc835 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_extended.yaml @@ -0,0 +1,167 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age2=[$2]) + LogicalFilter(condition=[<=($3, 1)]) + LogicalProject(avg_age=[$0], state=[$1], age2=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $2)]) + LogicalFilter(condition=[IS NOT NULL($2)]) + LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)]) + LogicalSort(sort0=[$1], dir0=[ASC-nulls-first]) + LogicalProject(avg_age=[$2], state=[$0], city=[$1]) + LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]) + LogicalProject(state=[$7], city=[$5], age=[$8]) + LogicalFilter(condition=[>($8, 30)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0..1=[{inputs}], age2=[$t0]) + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=[<=($t1, $t2)], proj#0..1=[{exprs}], $condition=[$t3]) + EnumerableWindow(window#0=[window(partition {0} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[2], expr#2=[+($t0, $t1)], expr#3=[IS NOT NULL($t0)], $0=[$t2], $condition=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[avg_age, state], SORT->[1 ASC FIRST], PROJECT->[avg_age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"city":{"terms":{"field":"city.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + extended: |+ + public org.apache.calcite.linq4j.Enumerable bind(final org.apache.calcite.DataContext root) { + final org.opensearch.sql.opensearch.storage.scan.CalciteEnumerableIndexScan v1stashed = (org.opensearch.sql.opensearch.storage.scan.CalciteEnumerableIndexScan) root.get("v1stashed"); + final org.apache.calcite.linq4j.Enumerable _inputEnumerable = v1stashed.scan(); + final org.apache.calcite.linq4j.AbstractEnumerable source = new org.apache.calcite.linq4j.AbstractEnumerable(){ + public org.apache.calcite.linq4j.Enumerator enumerator() { + return new org.apache.calcite.linq4j.Enumerator(){ + public final org.apache.calcite.linq4j.Enumerator inputEnumerator = _inputEnumerable.enumerator(); + public void reset() { + inputEnumerator.reset(); + } + + public boolean moveNext() { + while (inputEnumerator.moveNext()) { + if ((Double) inputEnumerator.current() != null) { + return true; + } + } + return false; + } + + public void close() { + inputEnumerator.close(); + } + + public Object current() { + return (Double) inputEnumerator.current() == null ? null : Double.valueOf(((Double) inputEnumerator.current()).doubleValue() + (double) 2); + } + + }; + } + + }; + int prevStart; + int prevEnd; + final java.util.Comparator comparator = new java.util.Comparator(){ + public int compare(Double v0, Double v1) { + int c; + return 0; + } + + public int compare(Object o0, Object o1) { + return this.compare((Double) o0, (Double) o1); + } + + }; + final org.apache.calcite.runtime.SortedMultiMap multiMap = new org.apache.calcite.runtime.SortedMultiMap(); + source.foreach(new org.apache.calcite.linq4j.function.Function1() { + public Object apply(Double v) { + Double key = v; + multiMap.putMulti(key, v); + return null; + } + public Object apply(Object v) { + return apply( + (Double) v); + } + } + ); + final java.util.Iterator iterator = multiMap.arrays(comparator); + final java.util.ArrayList _list = new java.util.ArrayList( + multiMap.size()); + Long a0w0 = (Long) null; + while (iterator.hasNext()) { + final Object[] _rows = (Object[]) iterator.next(); + prevStart = -1; + prevEnd = 2147483647; + for (int i = 0; i < _rows.length; (++i)) { + if (i != prevEnd) { + int actualStart = i < prevEnd ? 0 : prevEnd + 1; + prevEnd = i; + a0w0 = Long.valueOf(((Number)org.apache.calcite.linq4j.tree.Primitive.of(long.class).numberValueRoundDown((i - 0 + 1))).longValue()); + } + _list.add(new Object[] { + (Double) _rows[i], + a0w0}); + } + } + multiMap.clear(); + final org.apache.calcite.linq4j.Enumerable _inputEnumerable0 = org.apache.calcite.linq4j.Linq4j.asEnumerable(_list); + final org.apache.calcite.linq4j.AbstractEnumerable child = new org.apache.calcite.linq4j.AbstractEnumerable(){ + public org.apache.calcite.linq4j.Enumerator enumerator() { + return new org.apache.calcite.linq4j.Enumerator(){ + public final org.apache.calcite.linq4j.Enumerator inputEnumerator = _inputEnumerable0.enumerator(); + public void reset() { + inputEnumerator.reset(); + } + + public boolean moveNext() { + while (inputEnumerator.moveNext()) { + if (org.apache.calcite.runtime.SqlFunctions.toLong(((Object[]) inputEnumerator.current())[1]) <= $L4J$C$_Number_org_apache_calcite_linq4j_tree_Primitive_of_long_class_358aa52b) { + return true; + } + } + return false; + } + + public void close() { + inputEnumerator.close(); + } + + public Object current() { + final Object[] current = (Object[]) inputEnumerator.current(); + final Object input_value = current[0]; + final Object input_value0 = current[1]; + return new Object[] { + input_value, + input_value0}; + } + + static final long $L4J$C$_Number_org_apache_calcite_linq4j_tree_Primitive_of_long_class_358aa52b = ((Number)org.apache.calcite.linq4j.tree.Primitive.of(long.class).numberValueRoundDown(1)).longValue(); + }; + } + + }; + final org.apache.calcite.linq4j.Enumerable _inputEnumerable1 = child.take(10000); + return new org.apache.calcite.linq4j.AbstractEnumerable(){ + public org.apache.calcite.linq4j.Enumerator enumerator() { + return new org.apache.calcite.linq4j.Enumerator(){ + public final org.apache.calcite.linq4j.Enumerator inputEnumerator = _inputEnumerable1.enumerator(); + public void reset() { + inputEnumerator.reset(); + } + + public boolean moveNext() { + return inputEnumerator.moveNext(); + } + + public void close() { + inputEnumerator.close(); + } + + public Object current() { + return (Double) ((Object[]) inputEnumerator.current())[0]; + } + + }; + } + + }; + } + + + public Class getElementType() { + return java.lang.Double.class; + } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_output_simple.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_simple.json new file mode 100644 index 00000000000..96e070954ec --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_simple.json @@ -0,0 +1,5 @@ +{ + "calcite": { + "logical": "LogicalSystemLimit\n LogicalProject\n LogicalFilter\n LogicalProject\n LogicalFilter\n LogicalProject\n LogicalSort\n LogicalProject\n LogicalAggregate\n LogicalProject\n LogicalFilter\n CalciteLogicalIndexScan\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_output_simple.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_simple.yaml new file mode 100644 index 00000000000..f58ffa560a7 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_simple.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit + LogicalProject + LogicalFilter + LogicalProject + LogicalFilter + LogicalProject + LogicalSort + LogicalProject + LogicalAggregate + LogicalProject + LogicalFilter + CalciteLogicalIndexScan diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_output_standard.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_standard.json new file mode 100644 index 00000000000..a50b0baa104 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_standard.json @@ -0,0 +1,6 @@ +{ + "calcite": { + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age2=[$2])\n LogicalFilter(condition=[<=($3, 1)])\n LogicalProject(avg_age=[$0], state=[$1], age2=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $2)])\n LogicalFilter(condition=[IS NOT NULL($2)])\n LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)])\n LogicalSort(sort0=[$1], dir0=[ASC-nulls-first])\n LogicalProject(avg_age=[$2], state=[$0], city=[$1])\n LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)])\n LogicalProject(state=[$7], city=[$5], age=[$8])\n LogicalFilter(condition=[>($8, 30)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableCalc(expr#0..1=[{inputs}], age2=[$t0])\n EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=[<=($t1, $t2)], proj#0..1=[{exprs}], $condition=[$t3])\n EnumerableWindow(window#0=[window(partition {0} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])])\n EnumerableCalc(expr#0=[{inputs}], expr#1=[2], expr#2=[+($t0, $t1)], expr#3=[IS NOT NULL($t0)], $0=[$t2], $condition=[$t3])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[avg_age, state], SORT->[1 ASC FIRST], PROJECT->[avg_age]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_output_standard.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_standard.yaml new file mode 100644 index 00000000000..862a45dc617 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_standard.yaml @@ -0,0 +1,21 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age2=[$2]) + LogicalFilter(condition=[<=($3, 1)]) + LogicalProject(avg_age=[$0], state=[$1], age2=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $2)]) + LogicalFilter(condition=[IS NOT NULL($2)]) + LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)]) + LogicalSort(sort0=[$1], dir0=[ASC-nulls-first]) + LogicalProject(avg_age=[$2], state=[$0], city=[$1]) + LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]) + LogicalProject(state=[$7], city=[$5], age=[$8]) + LogicalFilter(condition=[>($8, 30)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0..1=[{inputs}], age2=[$t0]) + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=[<=($t1, $t2)], proj#0..1=[{exprs}], $condition=[$t3]) + EnumerableWindow(window#0=[window(partition {0} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[2], expr#2=[+($t0, $t1)], expr#3=[IS NOT NULL($t0)], $0=[$t2], $condition=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[avg_age, state], SORT->[1 ASC FIRST], PROJECT->[avg_age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"city":{"terms":{"field":"city.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.json deleted file mode 100644 index 978d6cf1bcc..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.json +++ /dev/null @@ -1 +0,0 @@ -{"calcite":{"logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(firstname=[$1], age=[$8], address=[$2])\n LogicalFilter(condition=[AND(=($2, '671 Bristol Street'), =(-($8, 2), 30))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n","physical":"CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, address, age], SCRIPT->AND(=($1, '671 Bristol Street'), =(-($2, 2), 30)), PROJECT->[firstname, age, address], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\":{\"must\":[{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"{\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"=\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"EQUALS\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 0,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"VARCHAR\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true,\\\\n \\\\\\\"precision\\\\\\\": -1\\\\n }\\\\n },\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 1,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"VARCHAR\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true,\\\\n \\\\\\\"precision\\\\\\\": -1\\\\n }\\\\n }\\\\n ]\\\\n}\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*,\"SOURCES\":[1,2],\"DIGESTS\":[\"address\",\"671 Bristol Street\"]}},\"boost\":1.0}},{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"{\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"=\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"EQUALS\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"-\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"MINUS\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 0,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n },\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 1,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n }\\\\n ],\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n },\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 2,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n }\\\\n ]\\\\n}\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*,\"SOURCES\":[0,2,2],\"DIGESTS\":[\"age\",2,30]}},\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"firstname\",\"age\",\"address\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n"}} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.yaml new file mode 100644 index 00000000000..7938c9ea602 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(firstname=[$1], age=[$8], address=[$2]) + LogicalFilter(condition=[AND(=($2, '671 Bristol Street'), =(-($8, 2), 30))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, address, age], SCRIPT->AND(=($1, '671 Bristol Street'), =(-($2, 2), 30)), PROJECT->[firstname, age, address], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"{\\n \\\"op\\\": {\\n \\\"name\\\": \\\"=\\\",\\n \\\"kind\\\": \\\"EQUALS\\\",\\n \\\"syntax\\\": \\\"BINARY\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"dynamicParam\\\": 0,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"VARCHAR\\\",\\n \\\"nullable\\\": true,\\n \\\"precision\\\": -1\\n }\\n },\\n {\\n \\\"dynamicParam\\\": 1,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"VARCHAR\\\",\\n \\\"nullable\\\": true,\\n \\\"precision\\\": -1\\n }\\n }\\n ]\\n}\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1,2],"DIGESTS":["address","671 Bristol Street"]}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"{\\n \\\"op\\\": {\\n \\\"name\\\": \\\"=\\\",\\n \\\"kind\\\": \\\"EQUALS\\\",\\n \\\"syntax\\\": \\\"BINARY\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"op\\\": {\\n \\\"name\\\": \\\"-\\\",\\n \\\"kind\\\": \\\"MINUS\\\",\\n \\\"syntax\\\": \\\"BINARY\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"dynamicParam\\\": 0,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n },\\n {\\n \\\"dynamicParam\\\": 1,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n }\\n ],\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n },\\n {\\n \\\"dynamicParam\\\": 2,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n }\\n ]\\n}\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["age",2,30]}},"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["firstname","age","address"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/Format.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/Format.java index 52d3f0fbfe8..227bf02426e 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/Format.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/Format.java @@ -17,17 +17,11 @@ public enum Format { JSON("json"), CSV("csv"), RAW("raw"), - TABLE("table"), - // format of explain response - SIMPLE("simple"), - STANDARD("standard"), - EXTENDED("extended"), - COST("cost"); + TABLE("table"); @Getter private final String formatName; public static final Map RESPONSE_FORMATS; - public static final Map EXPLAIN_FORMATS; static { ImmutableMap.Builder builder; @@ -38,20 +32,9 @@ public enum Format { builder.put(TABLE.formatName, TABLE); builder.put(JSON.formatName, JSON); RESPONSE_FORMATS = builder.build(); - - builder = new ImmutableMap.Builder<>(); - builder.put(SIMPLE.formatName, SIMPLE); - builder.put(STANDARD.formatName, STANDARD); - builder.put(EXTENDED.formatName, EXTENDED); - builder.put(COST.formatName, COST); - EXPLAIN_FORMATS = builder.build(); } public static Optional of(String formatName) { return Optional.ofNullable(RESPONSE_FORMATS.getOrDefault(formatName, null)); } - - public static Optional ofExplain(String formatName) { - return Optional.ofNullable(EXPLAIN_FORMATS.getOrDefault(formatName, null)); - } } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/request/SqlRequestParam.java b/legacy/src/main/java/org/opensearch/sql/legacy/request/SqlRequestParam.java index 5d2bb1971b0..33ca46486a8 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/request/SqlRequestParam.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/request/SqlRequestParam.java @@ -16,7 +16,6 @@ public class SqlRequestParam { public static final String QUERY_PARAMS_ESCAPE = "escape"; private static final String DEFAULT_RESPONSE_FORMAT = "jdbc"; - private static final String DEFAULT_EXPLAIN_FORMAT = "standard"; /** * Parse the pretty params to decide whether the response should be pretty formatted. @@ -44,9 +43,8 @@ public static Format getFormat(Map requestParams, String path) { String formatName = requestParams.containsKey(QUERY_PARAMS_FORMAT) ? requestParams.get(QUERY_PARAMS_FORMAT).toLowerCase() - : isExplainRequest(path) ? DEFAULT_EXPLAIN_FORMAT : DEFAULT_RESPONSE_FORMAT; - Optional optionalFormat = - isExplainRequest(path) ? Format.ofExplain(formatName) : Format.of(formatName); + : DEFAULT_RESPONSE_FORMAT; + Optional optionalFormat = Format.of(formatName); if (optionalFormat.isPresent()) { return optionalFormat.get(); } else { @@ -55,10 +53,6 @@ public static Format getFormat(Map requestParams, String path) { } } - private static boolean isExplainRequest(String path) { - return path != null && path.endsWith("/_explain"); - } - public static boolean getEscapeOption(Map requestParams) { if (requestParams.containsKey(QUERY_PARAMS_ESCAPE)) { return Boolean.parseBoolean(requestParams.get(QUERY_PARAMS_ESCAPE)); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java index f3bef13b795..b55af891e7d 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java @@ -40,7 +40,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.locationtech.jts.geom.Point; -import org.opensearch.sql.ast.statement.Explain.ExplainFormat; +import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.calcite.CalcitePlanContext; import org.opensearch.sql.calcite.utils.CalciteToolsHelper.OpenSearchRelRunners; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; @@ -178,26 +178,26 @@ private Hook.Closeable getCodegenInHook(AtomicReference codegen) { @Override public void explain( RelNode rel, - ExplainFormat format, + ExplainMode mode, CalcitePlanContext context, ResponseListener listener) { client.schedule( () -> { try { - if (format == ExplainFormat.SIMPLE) { + if (mode == ExplainMode.SIMPLE) { String logical = RelOptUtil.toString(rel, SqlExplainLevel.NO_ATTRIBUTES); listener.onResponse( new ExplainResponse(new ExplainResponseNodeV2(logical, null, null))); } else { SqlExplainLevel level = - format == ExplainFormat.COST + mode == ExplainMode.COST ? SqlExplainLevel.ALL_ATTRIBUTES : SqlExplainLevel.EXPPLAN_ATTRIBUTES; String logical = RelOptUtil.toString(rel, level); AtomicReference physical = new AtomicReference<>(); AtomicReference javaCode = new AtomicReference<>(); try (Hook.Closeable closeable = getPhysicalPlanInHook(physical, level)) { - if (format == ExplainFormat.EXTENDED) { + if (mode == ExplainMode.EXTENDED) { getCodegenInHook(javaCode); CalcitePlanContext.skipEncoding.set(true); } diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/request/PPLQueryRequestFactory.java b/plugin/src/main/java/org/opensearch/sql/plugin/request/PPLQueryRequestFactory.java index 0908d27b491..0dd8e1a9651 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/request/PPLQueryRequestFactory.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/request/PPLQueryRequestFactory.java @@ -7,6 +7,8 @@ import java.util.Map; import java.util.Optional; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.json.JSONException; import org.json.JSONObject; import org.opensearch.rest.RestRequest; @@ -16,12 +18,16 @@ /** Factory of {@link PPLQueryRequest}. */ public class PPLQueryRequestFactory { + private static final Logger LOG = LogManager.getLogger(PPLQueryRequestFactory.class); + private static final String PPL_URL_PARAM_KEY = "ppl"; private static final String PPL_FIELD_NAME = "query"; private static final String QUERY_PARAMS_FORMAT = "format"; + private static final String QUERY_PARAMS_EXPLAIN_MODE = "mode"; private static final String QUERY_PARAMS_SANITIZE = "sanitize"; private static final String DEFAULT_RESPONSE_FORMAT = "jdbc"; - private static final String DEFAULT_EXPLAIN_FORMAT = "standard"; + private static final String DEFAULT_EXPLAIN_FORMAT = "json"; + private static final String DEFAULT_EXPLAIN_MODE = "standard"; private static final String QUERY_PARAMS_PRETTY = "pretty"; private static final String QUERY_PARAMS_PROFILE = "profile"; @@ -57,6 +63,20 @@ private static PPLQueryRequest parsePPLRequestFromPayload(RestRequest restReques String content = restRequest.content().utf8ToString(); JSONObject jsonContent; Format format = getFormat(restRequest.params(), restRequest.rawPath()); + String explainMode; + // For backward compatible consideration, if the format=[simple, standard, extended, cost], we + // accept it as well and view it as mode and use json format. + // TODO: deprecated after 4.x + if (Format.isExplainMode(format)) { + // Log deprecation warning for legacy format-based explain mode usage + LOG.warn( + "Using 'format' parameter for explain mode is deprecated. Please use 'mode' parameter" + + " instead. This will be removed in 4.x."); + explainMode = format.getFormatName(); + format = Format.JSON; + } else { + explainMode = getExplainMode(restRequest.params(), restRequest.rawPath()); + } boolean pretty = getPrettyOption(restRequest.params()); try { jsonContent = new JSONObject(content); @@ -70,6 +90,7 @@ private static PPLQueryRequest parsePPLRequestFromPayload(RestRequest restReques jsonContent, restRequest.path(), format.getFormatName(), + explainMode, enableProfile); // set sanitize option if csv format if (format.equals(Format.CSV)) { @@ -129,4 +150,13 @@ private static boolean isProfileSupported(String path, Format format, String que format != null && DEFAULT_RESPONSE_FORMAT.equalsIgnoreCase(format.getFormatName()); return !explainPath && !explainQuery && isJdbcFormat; } + + private static String getExplainMode(Map requestParams, String path) { + if (!isExplainRequest(path)) { + return null; + } + return requestParams + .getOrDefault(QUERY_PARAMS_EXPLAIN_MODE, DEFAULT_EXPLAIN_MODE) + .toLowerCase(); + } } diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java index 34cc70cdee3..8b91475b858 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java @@ -88,7 +88,7 @@ public String getName() { @Override protected Set responseParams() { Set responseParams = new HashSet<>(super.responseParams()); - responseParams.addAll(Arrays.asList("format", "sanitize")); + responseParams.addAll(Arrays.asList("format", "mode", "sanitize")); return responseParams; } diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryRequest.java b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryRequest.java index 65663bbec00..e342d9a90f0 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryRequest.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryRequest.java @@ -34,6 +34,7 @@ public class TransportPPLQueryRequest extends ActionRequest { @Getter private final String path; @Getter private String format = ""; + @Getter private String explainMode; @Setter @Getter @@ -59,6 +60,7 @@ public TransportPPLQueryRequest(PPLQueryRequest pplQueryRequest) { sanitize = pplQueryRequest.sanitize(); style = pplQueryRequest.style(); profile = pplQueryRequest.profile(); + explainMode = pplQueryRequest.mode().getModeName(); } /** Constructor of TransportPPLQueryRequest from StreamInput. */ @@ -66,6 +68,7 @@ public TransportPPLQueryRequest(StreamInput in) throws IOException { super(in); pplQuery = in.readOptionalString(); format = in.readOptionalString(); + explainMode = in.readOptionalString(); String jsonContentString = in.readOptionalString(); jsonContent = jsonContentString != null ? new JSONObject(jsonContentString) : null; path = in.readOptionalString(); @@ -98,6 +101,7 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeOptionalString(pplQuery); out.writeOptionalString(format); + out.writeOptionalString(explainMode); out.writeOptionalString(jsonContent != null ? jsonContent.toString() : null); out.writeOptionalString(path); out.writeBoolean(sanitize); @@ -137,7 +141,7 @@ public ActionRequestValidationException validate() { /** Convert to PPLQueryRequest. */ public PPLQueryRequest toPPLQueryRequest() { PPLQueryRequest pplQueryRequest = - new PPLQueryRequest(pplQuery, jsonContent, path, format, profile); + new PPLQueryRequest(pplQuery, jsonContent, path, format, explainMode, profile); pplQueryRequest.sanitize(sanitize); pplQueryRequest.style(style); return pplQueryRequest; diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java b/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java index ccd4d49dd4a..ecae97283ed 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java @@ -98,6 +98,7 @@ private AbstractPlan plan( AstStatementBuilder.StatementBuilderContext.builder() .isExplain(request.isExplainRequest()) .format(request.getFormat()) + .explainMode(request.getExplainMode()) .build())); log.info( diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/domain/PPLQueryRequest.java b/ppl/src/main/java/org/opensearch/sql/ppl/domain/PPLQueryRequest.java index 321e1d410c4..ef21e0f2803 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/domain/PPLQueryRequest.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/domain/PPLQueryRequest.java @@ -11,6 +11,7 @@ import lombok.Setter; import lombok.experimental.Accessors; import org.json.JSONObject; +import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.protocol.response.format.Format; import org.opensearch.sql.protocol.response.format.JsonResponseFormatter; @@ -24,6 +25,7 @@ public class PPLQueryRequest { @Getter private final JSONObject jsonContent; @Getter private final String path; @Getter private String format = ""; + @Getter private String explainMode; @Setter @Getter @@ -45,16 +47,22 @@ public PPLQueryRequest(String pplQuery, JSONObject jsonContent, String path) { } public PPLQueryRequest(String pplQuery, JSONObject jsonContent, String path, String format) { - this(pplQuery, jsonContent, path, format, false); + this(pplQuery, jsonContent, path, format, ExplainMode.STANDARD.getModeName(), false); } /** Constructor of PPLQueryRequest. */ public PPLQueryRequest( - String pplQuery, JSONObject jsonContent, String path, String format, boolean profile) { + String pplQuery, + JSONObject jsonContent, + String path, + String format, + String explainMode, + boolean profile) { this.pplQuery = pplQuery; this.jsonContent = jsonContent; this.path = Optional.ofNullable(path).orElse(DEFAULT_PPL_PATH); this.format = format; + this.explainMode = explainMode; this.profile = profile; } @@ -81,4 +89,8 @@ public Format format() { String.format(Locale.ROOT, "response in %s format is not supported.", format)); } } + + public ExplainMode mode() { + return ExplainMode.of(explainMode); + } } diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstStatementBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstStatementBuilder.java index 1a5b198b42c..88ca7533cf5 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstStatementBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstStatementBuilder.java @@ -41,7 +41,7 @@ public Statement visitPplStatement(OpenSearchPPLParser.PplStatementContext ctx) return new Explain(query, PPL, ctx.explainStatement().explainMode().getText()); } } else { - return context.isExplain ? new Explain(query, PPL, context.format) : query; + return context.isExplain ? new Explain(query, PPL, context.explainMode) : query; } } @@ -56,6 +56,7 @@ public static class StatementBuilderContext { private final boolean isExplain; private final int fetchSize; private final String format; + private final String explainMode; } private UnresolvedPlan addSelectAll(UnresolvedPlan plan) { diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java index 24b99d48b9c..e3137182580 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -153,7 +153,7 @@ public String visitQuery(Query node, String context) { public String visitExplain(Explain node, String context) { return StringUtils.format( "explain %s %s", - node.getFormat().name().toLowerCase(Locale.ROOT), node.getStatement().accept(this, null)); + node.getMode().name().toLowerCase(Locale.ROOT), node.getStatement().accept(this, null)); } @Override diff --git a/protocol/src/main/java/org/opensearch/sql/protocol/response/format/Format.java b/protocol/src/main/java/org/opensearch/sql/protocol/response/format/Format.java index 14204554ff3..6db28e4d4ab 100644 --- a/protocol/src/main/java/org/opensearch/sql/protocol/response/format/Format.java +++ b/protocol/src/main/java/org/opensearch/sql/protocol/response/format/Format.java @@ -9,6 +9,7 @@ import com.google.common.collect.ImmutableMap; import java.util.Map; import java.util.Optional; +import java.util.Set; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -18,13 +19,18 @@ public enum Format { CSV("csv"), RAW("raw"), VIZ("viz"), - // format of explain response + + /*----- explain output format ------*/ + /** Returns explain output json format */ + JSON("json"), + /** Returns explain output in yaml format */ + YAML("yaml"), + + /*---- backward compatible format of explain response -----*/ SIMPLE("simple"), STANDARD("standard"), EXTENDED("extended"), - COST("cost"), - /** Returns explain output in yaml format */ - YAML("yaml"); + COST("cost"); @Getter private final String formatName; @@ -32,6 +38,8 @@ public enum Format { public static final Map EXPLAIN_FORMATS; + public static final Set EXPLAIN_MODES; + static { ImmutableMap.Builder builder; builder = new ImmutableMap.Builder<>(); @@ -42,12 +50,15 @@ public enum Format { RESPONSE_FORMATS = builder.build(); builder = new ImmutableMap.Builder<>(); + builder.put(JSON.formatName, JSON); + builder.put(YAML.formatName, YAML); builder.put(SIMPLE.formatName, SIMPLE); builder.put(STANDARD.formatName, STANDARD); builder.put(EXTENDED.formatName, EXTENDED); builder.put(COST.formatName, COST); - builder.put(YAML.formatName, YAML); EXPLAIN_FORMATS = builder.build(); + + EXPLAIN_MODES = Set.of(SIMPLE, STANDARD, EXTENDED, COST); } public static Optional of(String formatName) { @@ -56,7 +67,11 @@ public static Optional of(String formatName) { } public static Optional ofExplain(String formatName) { - String format = Strings.isNullOrEmpty(formatName) ? "standard" : formatName.toLowerCase(); + String format = Strings.isNullOrEmpty(formatName) ? "json" : formatName.toLowerCase(); return Optional.ofNullable(EXPLAIN_FORMATS.getOrDefault(format, null)); } + + public static boolean isExplainMode(Format format) { + return EXPLAIN_MODES.contains(format); + } } diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/FormatTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/FormatTest.java index fbe12ef44b3..2b1b9dde851 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/FormatTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/FormatTest.java @@ -48,6 +48,7 @@ void extended() { Optional format = Format.ofExplain("extended"); assertTrue(format.isPresent()); assertEquals(Format.EXTENDED, format.get()); + assertTrue(Format.isExplainMode(format.get())); } @Test @@ -61,7 +62,7 @@ void yaml() { void defaultExplainFormat() { Optional format = Format.ofExplain(""); assertTrue(format.isPresent()); - assertEquals(Format.STANDARD, format.get()); + assertEquals(Format.JSON, format.get()); } @Test