diff --git a/core/src/main/java/org/opensearch/sql/ast/statement/Explain.java b/core/src/main/java/org/opensearch/sql/ast/statement/Explain.java index dd918a886a4..b2fcb2b8179 100644 --- a/core/src/main/java/org/opensearch/sql/ast/statement/Explain.java +++ b/core/src/main/java/org/opensearch/sql/ast/statement/Explain.java @@ -5,7 +5,6 @@ package org.opensearch.sql.ast.statement; -import java.util.Locale; import lombok.EqualsAndHashCode; import lombok.Getter; import org.opensearch.sql.ast.AbstractNodeVisitor; @@ -18,37 +17,20 @@ public class Explain extends Statement { private final Statement statement; private final QueryType queryType; - private final ExplainFormat format; + private final ExplainMode mode; public Explain(Statement statement, QueryType queryType) { this(statement, queryType, null); } - public Explain(Statement statement, QueryType queryType, String format) { + public Explain(Statement statement, QueryType queryType, String mode) { this.statement = statement; this.queryType = queryType; - this.format = Explain.format(format); + this.mode = ExplainMode.of(mode); } @Override public R accept(AbstractNodeVisitor visitor, C context) { return visitor.visitExplain(this, context); } - - public enum ExplainFormat { - SIMPLE, - STANDARD, - EXTENDED, - COST, - /** Formats explain output in yaml format. */ - YAML - } - - public static ExplainFormat format(String format) { - try { - return ExplainFormat.valueOf(format.toUpperCase(Locale.ROOT)); - } catch (Exception e) { - return ExplainFormat.STANDARD; - } - } } diff --git a/core/src/main/java/org/opensearch/sql/ast/statement/ExplainMode.java b/core/src/main/java/org/opensearch/sql/ast/statement/ExplainMode.java new file mode 100644 index 00000000000..9043f05929b --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/statement/ExplainMode.java @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.statement; + +import java.util.Locale; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public enum ExplainMode { + SIMPLE("simple"), + STANDARD("standard"), + EXTENDED("extended"), + COST("cost"); + + @Getter private final String modeName; + + public static ExplainMode of(String mode) { + if (mode == null || mode.isEmpty()) return STANDARD; + try { + return ExplainMode.valueOf(mode.toUpperCase(Locale.ROOT)); + } catch (Exception e) { + return ExplainMode.STANDARD; + } + } +} diff --git a/core/src/main/java/org/opensearch/sql/executor/ExecutionEngine.java b/core/src/main/java/org/opensearch/sql/executor/ExecutionEngine.java index ec1e427e365..e65db7b4065 100644 --- a/core/src/main/java/org/opensearch/sql/executor/ExecutionEngine.java +++ b/core/src/main/java/org/opensearch/sql/executor/ExecutionEngine.java @@ -13,7 +13,7 @@ import lombok.RequiredArgsConstructor; import lombok.ToString; import org.apache.calcite.rel.RelNode; -import org.opensearch.sql.ast.statement.Explain; +import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.calcite.CalcitePlanContext; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.data.model.ExprValue; @@ -53,7 +53,7 @@ default void execute( default void explain( RelNode plan, - Explain.ExplainFormat format, + ExplainMode mode, CalcitePlanContext context, ResponseListener listener) {} diff --git a/core/src/main/java/org/opensearch/sql/executor/QueryService.java b/core/src/main/java/org/opensearch/sql/executor/QueryService.java index d40d8dcaa71..8edc3ad3f2c 100644 --- a/core/src/main/java/org/opensearch/sql/executor/QueryService.java +++ b/core/src/main/java/org/opensearch/sql/executor/QueryService.java @@ -26,7 +26,7 @@ import org.apache.calcite.tools.Programs; import org.opensearch.sql.analysis.AnalysisContext; import org.opensearch.sql.analysis.Analyzer; -import org.opensearch.sql.ast.statement.Explain; +import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.calcite.CalcitePlanContext; import org.opensearch.sql.calcite.CalciteRelNodeVisitor; @@ -81,11 +81,11 @@ public void explain( UnresolvedPlan plan, QueryType queryType, ResponseListener listener, - Explain.ExplainFormat format) { + ExplainMode mode) { if (shouldUseCalcite(queryType)) { - explainWithCalcite(plan, queryType, listener, format); + explainWithCalcite(plan, queryType, listener, mode); } else { - explainWithLegacy(plan, queryType, listener, format, Optional.empty()); + explainWithLegacy(plan, queryType, listener, mode, Optional.empty()); } } @@ -134,7 +134,7 @@ public void explainWithCalcite( UnresolvedPlan plan, QueryType queryType, ResponseListener listener, - Explain.ExplainFormat format) { + ExplainMode mode) { CalcitePlanContext.run( () -> { try { @@ -146,13 +146,13 @@ public void explainWithCalcite( () -> { RelNode relNode = analyze(plan, context); RelNode calcitePlan = convertToCalcitePlan(relNode, context); - executionEngine.explain(calcitePlan, format, context, listener); + executionEngine.explain(calcitePlan, mode, context, listener); }, settings); } catch (Throwable t) { if (isCalciteFallbackAllowed(t)) { log.warn("Fallback to V2 query engine since got exception", t); - explainWithLegacy(plan, queryType, listener, format, Optional.of(t)); + explainWithLegacy(plan, queryType, listener, mode, Optional.of(t)); } else { if (t instanceof Error) { // Calcite may throw AssertError during query execution. @@ -198,13 +198,12 @@ public void explainWithLegacy( UnresolvedPlan plan, QueryType queryType, ResponseListener listener, - Explain.ExplainFormat format, + ExplainMode mode, Optional calciteFailure) { try { - if (format != null - && (format != Explain.ExplainFormat.STANDARD && format != Explain.ExplainFormat.YAML)) { + if (mode != null && (mode != ExplainMode.STANDARD)) { throw new UnsupportedOperationException( - "Explain mode " + format.name() + " is not supported in v2 engine"); + "Explain mode " + mode.name() + " is not supported in v2 engine"); } executionEngine.explain(plan(analyze(plan, queryType)), listener); } catch (Exception e) { diff --git a/core/src/main/java/org/opensearch/sql/executor/execution/AbstractPlan.java b/core/src/main/java/org/opensearch/sql/executor/execution/AbstractPlan.java index 0a81d71e5b0..e470d12507e 100644 --- a/core/src/main/java/org/opensearch/sql/executor/execution/AbstractPlan.java +++ b/core/src/main/java/org/opensearch/sql/executor/execution/AbstractPlan.java @@ -7,7 +7,7 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; -import org.opensearch.sql.ast.statement.Explain; +import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.QueryId; @@ -31,5 +31,5 @@ public abstract class AbstractPlan { * @param listener query explain response listener. */ public abstract void explain( - ResponseListener listener, Explain.ExplainFormat format); + ResponseListener listener, ExplainMode mode); } diff --git a/core/src/main/java/org/opensearch/sql/executor/execution/CommandPlan.java b/core/src/main/java/org/opensearch/sql/executor/execution/CommandPlan.java index cdd5380df5b..0d6750dfe28 100644 --- a/core/src/main/java/org/opensearch/sql/executor/execution/CommandPlan.java +++ b/core/src/main/java/org/opensearch/sql/executor/execution/CommandPlan.java @@ -5,7 +5,7 @@ package org.opensearch.sql.executor.execution; -import org.opensearch.sql.ast.statement.Explain; +import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.executor.ExecutionEngine; @@ -47,7 +47,7 @@ public void execute() { @Override public void explain( - ResponseListener listener, Explain.ExplainFormat format) { + ResponseListener listener, ExplainMode mode) { throw new UnsupportedOperationException("CommandPlan does not support explain"); } } diff --git a/core/src/main/java/org/opensearch/sql/executor/execution/ExplainPlan.java b/core/src/main/java/org/opensearch/sql/executor/execution/ExplainPlan.java index 67c2f65f5d1..27f7a47e504 100644 --- a/core/src/main/java/org/opensearch/sql/executor/execution/ExplainPlan.java +++ b/core/src/main/java/org/opensearch/sql/executor/execution/ExplainPlan.java @@ -5,7 +5,7 @@ package org.opensearch.sql.executor.execution; -import org.opensearch.sql.ast.statement.Explain; +import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.QueryId; @@ -15,7 +15,7 @@ public class ExplainPlan extends AbstractPlan { private final AbstractPlan plan; - private final Explain.ExplainFormat format; + private final ExplainMode mode; private final ResponseListener explainListener; @@ -24,22 +24,22 @@ public ExplainPlan( QueryId queryId, QueryType queryType, AbstractPlan plan, - Explain.ExplainFormat format, + ExplainMode mode, ResponseListener explainListener) { super(queryId, queryType); this.plan = plan; - this.format = format; + this.mode = mode; this.explainListener = explainListener; } @Override public void execute() { - plan.explain(explainListener, format); + plan.explain(explainListener, mode); } @Override public void explain( - ResponseListener listener, Explain.ExplainFormat format) { + ResponseListener listener, ExplainMode mode) { throw new UnsupportedOperationException("explain query can not been explained."); } } diff --git a/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlan.java b/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlan.java index c37327f3df4..5896871f5d0 100644 --- a/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlan.java +++ b/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlan.java @@ -7,7 +7,7 @@ import java.util.Optional; import org.apache.commons.lang3.NotImplementedException; -import org.opensearch.sql.ast.statement.Explain; +import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.ast.tree.Paginate; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.response.ResponseListener; @@ -69,13 +69,13 @@ public void execute() { @Override public void explain( - ResponseListener listener, Explain.ExplainFormat format) { + ResponseListener listener, ExplainMode mode) { if (pageSize.isPresent()) { listener.onFailure( new NotImplementedException( "`explain` feature for paginated requests is not implemented yet.")); } else { - queryService.explain(plan, getQueryType(), listener, format); + queryService.explain(plan, getQueryType(), listener, mode); } } } diff --git a/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java b/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java index 5f10df92894..7335cf574b2 100644 --- a/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java +++ b/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java @@ -11,6 +11,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.opensearch.sql.ast.AbstractNodeVisitor; import org.opensearch.sql.ast.statement.Explain; +import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.ast.statement.Query; import org.opensearch.sql.ast.statement.Statement; import org.opensearch.sql.ast.tree.CloseCursor; @@ -77,7 +78,7 @@ public AbstractPlan create( new QueryPlan( queryId, queryType, new FetchCursor(cursor), queryService, queryResponseListener); return isExplain - ? new ExplainPlan(queryId, queryType, plan, Explain.format(format), explainListener) + ? new ExplainPlan(queryId, queryType, plan, ExplainMode.of(format), explainListener) : plan; } @@ -137,7 +138,7 @@ public AbstractPlan visitExplain( QueryId.queryId(), node.getQueryType(), create(node.getStatement(), NO_CONSUMER_RESPONSE_LISTENER, context.getRight()), - node.getFormat(), + node.getMode(), context.getRight()); } } diff --git a/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java b/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java index 77a5081bb5b..0ee97754c66 100644 --- a/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java @@ -20,7 +20,7 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.sql.analysis.Analyzer; -import org.opensearch.sql.ast.statement.Explain; +import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.common.setting.Settings; @@ -59,7 +59,7 @@ class QueryServiceTest { @Mock private Split split; - private final Explain.ExplainFormat format = Explain.ExplainFormat.STANDARD; + private final ExplainMode mode = ExplainMode.STANDARD; @Test public void executeWithoutContext() { @@ -222,7 +222,7 @@ public void onFailure(Exception e) { fail(); } }, - format); + mode); } void handledByExplainOnFailure() { @@ -240,7 +240,7 @@ public void onFailure(Exception e) { assertTrue(e instanceof IllegalStateException); } }, - format); + mode); } } } diff --git a/core/src/test/java/org/opensearch/sql/executor/execution/CommandPlanTest.java b/core/src/test/java/org/opensearch/sql/executor/execution/CommandPlanTest.java index e074c1ebdaf..433c0336735 100644 --- a/core/src/test/java/org/opensearch/sql/executor/execution/CommandPlanTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/execution/CommandPlanTest.java @@ -19,7 +19,7 @@ import org.junit.jupiter.api.DisplayNameGeneration; import org.junit.jupiter.api.DisplayNameGenerator; import org.junit.jupiter.api.Test; -import org.opensearch.sql.ast.statement.Explain; +import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.executor.QueryId; @@ -63,7 +63,7 @@ public void explain_not_supported() { QueryService qs = mock(QueryService.class); ResponseListener listener = mock(ResponseListener.class); ResponseListener explainListener = mock(ResponseListener.class); - Explain.ExplainFormat format = mock(Explain.ExplainFormat.class); + ExplainMode mode = mock(ExplainMode.class); var exception = assertThrows( @@ -75,7 +75,7 @@ public void explain_not_supported() { mock(UnresolvedPlan.class), qs, listener) - .explain(explainListener, format)); + .explain(explainListener, mode)); assertEquals("CommandPlan does not support explain", exception.getMessage()); verify(listener, never()).onResponse(any()); diff --git a/core/src/test/java/org/opensearch/sql/executor/execution/ExplainPlanTest.java b/core/src/test/java/org/opensearch/sql/executor/execution/ExplainPlanTest.java index 2c24dbf820e..4cb5c755d14 100644 --- a/core/src/test/java/org/opensearch/sql/executor/execution/ExplainPlanTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/execution/ExplainPlanTest.java @@ -16,7 +16,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.sql.ast.statement.Explain; +import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.QueryId; @@ -32,29 +32,27 @@ public class ExplainPlanTest { @Mock private ResponseListener explainListener; - @Mock private Explain.ExplainFormat format; + @Mock private ExplainMode mode; @Test public void execute() { doNothing().when(queryPlan).explain(any(), any()); - ExplainPlan explainPlan = - new ExplainPlan(queryId, queryType, queryPlan, format, explainListener); + ExplainPlan explainPlan = new ExplainPlan(queryId, queryType, queryPlan, mode, explainListener); explainPlan.execute(); - verify(queryPlan, times(1)).explain(explainListener, format); + verify(queryPlan, times(1)).explain(explainListener, mode); } @Test public void explainThrowException() { - ExplainPlan explainPlan = - new ExplainPlan(queryId, queryType, queryPlan, format, explainListener); + ExplainPlan explainPlan = new ExplainPlan(queryId, queryType, queryPlan, mode, explainListener); UnsupportedOperationException unsupportedExplainException = assertThrows( UnsupportedOperationException.class, () -> { - explainPlan.explain(explainListener, format); + explainPlan.explain(explainListener, mode); }); assertEquals("explain query can not been explained.", unsupportedExplainException.getMessage()); } diff --git a/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanTest.java b/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanTest.java index 809cbee9483..77deb9b6a48 100644 --- a/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanTest.java @@ -20,7 +20,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.sql.ast.statement.Explain; +import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.executor.DefaultExecutionEngine; @@ -45,7 +45,7 @@ class QueryPlanTest { @Mock private ResponseListener queryListener; - @Mock private Explain.ExplainFormat format; + @Mock private ExplainMode mode; @Test public void execute_no_page_size() { @@ -58,9 +58,9 @@ public void execute_no_page_size() { @Test public void explain_no_page_size() { QueryPlan query = new QueryPlan(queryId, queryType, plan, queryService, queryListener); - query.explain(explainListener, format); + query.explain(explainListener, mode); - verify(queryService, times(1)).explain(plan, queryType, explainListener, format); + verify(queryService, times(1)).explain(plan, queryType, explainListener, mode); } @Test @@ -124,6 +124,6 @@ public void onFailure(Exception e) { assertTrue(e instanceof NotImplementedException); } }, - format); + mode); } } diff --git a/docs/user/ppl/interfaces/endpoint.md b/docs/user/ppl/interfaces/endpoint.md index 981d20e6580..c109381791d 100644 --- a/docs/user/ppl/interfaces/endpoint.md +++ b/docs/user/ppl/interfaces/endpoint.md @@ -61,7 +61,8 @@ You can send HTTP explain request to endpoint **/_plugins/_ppl/_explain** with y ### Description To translate your query, send it to explain endpoint. The explain output is OpenSearch domain specific language (DSL) in JSON format. You can just copy and paste it to your console to run it against OpenSearch directly. -Explain output could be set different formats: `standard` (the default format), `simple`, `extended`, `dsl`. +Explain API supports various mode: `standard` (the default format), `simple`, `extended`, `cost`. +And the explain output could be shown in different formats: `json` (the default format), `yaml` ### Example 1 default (standard) format Explain query @@ -84,13 +85,13 @@ Expected output: ``` -### Example 2 simple format +### Example 2 simple mode Explain query ```bash ppl curl -sS -H 'Content-Type: application/json' \ --X POST localhost:9200/_plugins/_ppl/_explain?format=simple \ +-X POST localhost:9200/_plugins/_ppl/_explain?mode=simple \ -d '{"query" : "source=state_country | where age>30"}' ``` @@ -104,13 +105,13 @@ Expected output: } ``` -### Example 3 extended format +### Example 3 extended mode Explain query ```bash ppl curl -sS -H 'Content-Type: application/json' \ --X POST localhost:9200/_plugins/_ppl/_explain?format=extended \ +-X POST localhost:9200/_plugins/_ppl/_explain?mode=extended \ -d '{"query" : "source=state_country | head 10 | where age>30"}' ``` @@ -130,7 +131,7 @@ Expected output: YAML explain output is an experimental feature and not intended for production use. The interface and output may change without notice. -Return Explain response format in In `yaml` format. +Return Explain response in `yaml` format. Explain query ```bash ppl diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index f00fbfec75b..b6cd327989a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -25,8 +25,10 @@ import java.util.Locale; import org.junit.Ignore; import org.junit.Test; -import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.ast.statement.ExplainMode; +import org.opensearch.sql.common.setting.Settings.Key; import org.opensearch.sql.ppl.ExplainIT; +import org.opensearch.sql.protocol.response.format.Format; public class CalciteExplainIT extends ExplainIT { @Override @@ -382,9 +384,9 @@ public void testSkipScriptEncodingOnExtendedFormat() throws IOException { String query = "source=opensearch-sql_test_index_account | where address = '671 Bristol Street' and age -" + " 2 = 30 | fields firstname, age, address"; - var result = explainQueryToString(query, true); - String expected = loadFromFile("expectedOutput/calcite/explain_skip_script_encoding.json"); - assertJsonEqualsIgnoreId(expected, result); + var result = explainQueryYaml(query, ExplainMode.EXTENDED); + String expected = loadFromFile("expectedOutput/calcite/explain_skip_script_encoding.yaml"); + assertYamlEqualsIgnoreId(expected, result); } // Only for Calcite, as v2 gets unstable serialized string for function @@ -1720,7 +1722,7 @@ public void testExplainRareCommandUseNull() throws IOException { explainQueryYaml( String.format("source=%s | rare 2 usenull=true state by gender", TEST_INDEX_ACCOUNT))); withSettings( - Settings.Key.PPL_SYNTAX_LEGACY_PREFERRED, + Key.PPL_SYNTAX_LEGACY_PREFERRED, "false", () -> { try { @@ -1747,7 +1749,7 @@ public void testExplainTopCommandUseNull() throws IOException { explainQueryYaml( String.format("source=%s | top 2 usenull=true state by gender", TEST_INDEX_ACCOUNT))); withSettings( - Settings.Key.PPL_SYNTAX_LEGACY_PREFERRED, + Key.PPL_SYNTAX_LEGACY_PREFERRED, "false", () -> { try { @@ -2207,14 +2209,14 @@ public void testAliasTypeField() throws IOException { @Test public void testRexStandardizationForScript() throws IOException { enabledOnlyWhenPushdownIsEnabled(); - assertJsonEqualsIgnoreId( - loadExpectedPlan("explain_extended_for_standardization.json"), - explainQueryToString( + assertYamlEqualsIgnoreId( + loadExpectedPlan("explain_extended_for_standardization.yaml"), + explainQueryYaml( String.format( "source=%s | eval age_range = case(age < 30, 'u30', age >= 30 and age <= 40, 'u40'" + " else 'u100') | stats avg(age) as avg_age by age_range", TEST_INDEX_BANK), - true)); + ExplainMode.EXTENDED)); } @Test @@ -2318,4 +2320,52 @@ public void testNotBetweenPushDownExplain() throws Exception { explainQueryYaml( "source=opensearch-sql_test_index_bank | where age not between 30 and 39")); } + + @Test + public void testExplainInVariousModeAndFormat() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + String query = + "source=opensearch-sql_test_index_account" + + "| where age > 30 " + + "| stats avg(age) AS avg_age by state, city " + + "| sort state " + + "| fields - city " + + "| eval age2 = avg_age + 2 " + + "| dedup age2 " + + "| fields age2"; + ExplainMode[] explainModes = + new ExplainMode[] { + ExplainMode.SIMPLE, ExplainMode.STANDARD, ExplainMode.EXTENDED, ExplainMode.COST + }; + for (ExplainMode explainMode : explainModes) { + String modeName = explainMode.getModeName().toLowerCase(Locale.ROOT); + assertYamlEqualsIgnoreId( + loadExpectedPlan(String.format("explain_output_%s.yaml", modeName)), + explainQueryYaml(query, explainMode)); + assertJsonEqualsIgnoreId( + loadExpectedPlan(String.format("explain_output_%s.json", modeName)), + explainQueryToString(query, explainMode)); + } + } + + @Test + public void testExplainBWC() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + String query = + "source=opensearch-sql_test_index_account" + + "| where age > 30 " + + "| stats avg(age) AS avg_age by state, city " + + "| sort state " + + "| fields - city " + + "| eval age2 = avg_age + 2 " + + "| dedup age2 " + + "| fields age2"; + Format[] formats = new Format[] {Format.SIMPLE, Format.STANDARD, Format.EXTENDED, Format.COST}; + for (Format format : formats) { + String formatName = format.getFormatName().toLowerCase(Locale.ROOT); + assertJsonEqualsIgnoreId( + loadExpectedPlan(String.format("explain_output_%s.json", formatName)), + explainQueryToStringBWC(query, format)); + } + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java index 68a9c2afa69..6135d74b2fd 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java @@ -6,7 +6,6 @@ package org.opensearch.sql.ppl; import static org.opensearch.sql.legacy.TestUtils.getResponseBody; -import static org.opensearch.sql.plugin.rest.RestPPLQueryAction.EXPLAIN_API_ENDPOINT; import static org.opensearch.sql.plugin.rest.RestPPLQueryAction.QUERY_API_ENDPOINT; import com.google.common.io.Resources; @@ -27,16 +26,17 @@ import org.opensearch.client.Response; import org.opensearch.client.ResponseException; import org.opensearch.common.collect.MapBuilder; +import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.common.setting.Settings.Key; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.protocol.response.format.Format; import org.opensearch.sql.util.RetryProcessor; /** OpenSearch Rest integration test base for PPL testing. */ public abstract class PPLIntegTestCase extends SQLIntegTestCase { - private static final String EXTENDED_EXPLAIN_API_ENDPOINT = - "/_plugins/_ppl/_explain?format=extended"; - private static final String YAML_EXPLAIN_API_ENDPOINT = "/_plugins/_ppl/_explain?format=yaml"; + private static final String BWC_EXPLAIN_API_ENDPOINT = "/_plugins/_ppl/_explain?format=%s"; + private static final String EXPLAIN_API_ENDPOINT = "/_plugins/_ppl/_explain?format=%s&mode=%s"; private static final Logger LOG = LogManager.getLogger(); @Rule public final RetryProcessor retryProcessor = new RetryProcessor(); public static final Integer DEFAULT_SUBSEARCH_MAXOUT = 10000; @@ -62,25 +62,35 @@ protected String executeQueryToString(String query) throws IOException { /** Deprecated, use {@link #explainQueryYaml(String)} */ @Deprecated protected String explainQueryToString(String query) throws IOException { - return explainQueryToString(query, false); + return explainQueryToString(query, ExplainMode.STANDARD); } protected String explainQueryYaml(String query) throws IOException { - Response response = client().performRequest(buildRequest(query, YAML_EXPLAIN_API_ENDPOINT)); + return explainQueryYaml(query, ExplainMode.STANDARD); + } + + protected String explainQueryYaml(String query, ExplainMode mode) throws IOException { + return explainQuery(query, Format.YAML, mode); + } + + protected String explainQueryToString(String query, ExplainMode mode) throws IOException { + return explainQuery(query, Format.JSON, mode).replace("\\r\\n", "\\n"); + } + + private String explainQuery(String query, Format format, ExplainMode mode) throws IOException { + Response response = + client() + .performRequest(buildRequest(query, String.format(EXPLAIN_API_ENDPOINT, format, mode))); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); - String responseBody = getResponseBody(response, true); - return responseBody; + return getResponseBody(response, true); } - protected String explainQueryToString(String query, boolean extended) throws IOException { + protected String explainQueryToStringBWC(String query, Format format) throws IOException { Response response = client() - .performRequest( - buildRequest( - query, extended ? EXTENDED_EXPLAIN_API_ENDPOINT : EXPLAIN_API_ENDPOINT)); + .performRequest(buildRequest(query, String.format(BWC_EXPLAIN_API_ENDPOINT, format))); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); - String responseBody = getResponseBody(response, true); - return responseBody.replace("\\r\\n", "\\n"); + return getResponseBody(response, true).replace("\\r\\n", "\\n"); } protected String executeCsvQuery(String query, boolean sanitize) throws IOException { diff --git a/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java b/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java index a70e27c1f54..e41c85a8e80 100644 --- a/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java @@ -431,7 +431,8 @@ private static String eliminateTimeStamp(String s) { private static String eliminateRelId(String s) { return s.replaceAll("rel#\\d+", "rel#") .replaceAll("RelSubset#\\d+", "RelSubset#") - .replaceAll("LogicalProject#\\d+", "LogicalProject#"); + .replaceAll("LogicalProject#\\d+", "LogicalProject#") + .replaceAll("id = \\d+", "id = *"); } private static String eliminateRequestOptions(String s) { @@ -484,7 +485,8 @@ private static String cleanUpYaml(String s) { .replaceAll("LogicalProject#\\d+", "LogicalProject#") .replaceAll("pitId=[^,]+,", "pitId=*,") .replaceAll(" needClean=true,", "") - .replaceAll(" searchDone=false,", ""); + .replaceAll(" searchDone=false,", "") + .replaceAll("id = \\d+", "id = *"); } private static String jsonToYaml(String json) { diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_extended_for_standardization.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_extended_for_standardization.json deleted file mode 100644 index e56186feb95..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_extended_for_standardization.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(avg_age=[$1], age_range=[$0])\n LogicalAggregate(group=[{0}], avg_age=[AVG($1)])\n LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, SEARCH($10, Sarg[[30..40]]), 'u40':VARCHAR, 'u100':VARCHAR)], age=[$10])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg_age=AVG($1)), PROJECT->[avg_age, age_range], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"age_range\":{\"terms\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"{\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"CASE\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"CASE\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"SPECIAL\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"<\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"LESS_THAN\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 0,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n },\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 1,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n }\\\\n ]\\\\n },\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 2,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"VARCHAR\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true,\\\\n \\\\\\\"precision\\\\\\\": -1\\\\n }\\\\n },\\\\n {\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"AND\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"AND\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"<=\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"LESS_THAN_OR_EQUAL\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 3,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n },\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 4,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n }\\\\n ]\\\\n },\\\\n {\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"<=\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"LESS_THAN_OR_EQUAL\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 5,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n },\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 6,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n },\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 7,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"VARCHAR\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true,\\\\n \\\\\\\"precision\\\\\\\": -1\\\\n }\\\\n },\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 8,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"VARCHAR\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true,\\\\n \\\\\\\"precision\\\\\\\": -1\\\\n }\\\\n }\\\\n ]\\\\n}\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*,\"SOURCES\":[0,2,2,2,0,0,2,2,2],\"DIGESTS\":[\"age\",30,\"u30\",30,\"age\",\"age\",40,\"u40\",\"u100\"]}},\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_extended_for_standardization.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_extended_for_standardization.yaml new file mode 100644 index 00000000000..73586df5e66 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_extended_for_standardization.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg_age=[$1], age_range=[$0]) + LogicalAggregate(group=[{0}], avg_age=[AVG($1)]) + LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, SEARCH($10, Sarg[[30..40]]), 'u40':VARCHAR, 'u100':VARCHAR)], age=[$10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg_age=AVG($1)), PROJECT->[avg_age, age_range], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age_range":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"{\\n \\\"op\\\": {\\n \\\"name\\\": \\\"CASE\\\",\\n \\\"kind\\\": \\\"CASE\\\",\\n \\\"syntax\\\": \\\"SPECIAL\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"op\\\": {\\n \\\"name\\\": \\\"<\\\",\\n \\\"kind\\\": \\\"LESS_THAN\\\",\\n \\\"syntax\\\": \\\"BINARY\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"dynamicParam\\\": 0,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n },\\n {\\n \\\"dynamicParam\\\": 1,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n }\\n ]\\n },\\n {\\n \\\"dynamicParam\\\": 2,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"VARCHAR\\\",\\n \\\"nullable\\\": true,\\n \\\"precision\\\": -1\\n }\\n },\\n {\\n \\\"op\\\": {\\n \\\"name\\\": \\\"AND\\\",\\n \\\"kind\\\": \\\"AND\\\",\\n \\\"syntax\\\": \\\"BINARY\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"op\\\": {\\n \\\"name\\\": \\\"<=\\\",\\n \\\"kind\\\": \\\"LESS_THAN_OR_EQUAL\\\",\\n \\\"syntax\\\": \\\"BINARY\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"dynamicParam\\\": 3,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n },\\n {\\n \\\"dynamicParam\\\": 4,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n }\\n ]\\n },\\n {\\n \\\"op\\\": {\\n \\\"name\\\": \\\"<=\\\",\\n \\\"kind\\\": \\\"LESS_THAN_OR_EQUAL\\\",\\n \\\"syntax\\\": \\\"BINARY\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"dynamicParam\\\": 5,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n },\\n {\\n \\\"dynamicParam\\\": 6,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n }\\n ]\\n }\\n ]\\n },\\n {\\n \\\"dynamicParam\\\": 7,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"VARCHAR\\\",\\n \\\"nullable\\\": true,\\n \\\"precision\\\": -1\\n }\\n },\\n {\\n \\\"dynamicParam\\\": 8,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"VARCHAR\\\",\\n \\\"nullable\\\": true,\\n \\\"precision\\\": -1\\n }\\n }\\n ]\\n}\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2,2,0,0,2,2,2],"DIGESTS":["age",30,"u30",30,"age","age",40,"u40","u100"]}},"missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_output_cost.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_cost.json new file mode 100644 index 00000000000..6235593dc6f --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_cost.json @@ -0,0 +1,6 @@ +{ + "calcite": { + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]): rowcount = 56.25, cumulative cost = {165681.25 rows, 105156.5471810663 cpu, 0.0 io}, id = 9401\n LogicalProject(age2=[$2]): rowcount = 56.25, cumulative cost = {165625.0 rows, 104256.5471810663 cpu, 0.0 io}, id = 9400\n LogicalFilter(condition=[<=($3, 1)]): rowcount = 56.25, cumulative cost = {165568.75 rows, 104200.2971810663 cpu, 0.0 io}, id = 9398\n LogicalProject(avg_age=[$0], state=[$1], age2=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $2)]): rowcount = 225.0, cumulative cost = {165512.5 rows, 103975.2971810663 cpu, 0.0 io}, id = 9397\n LogicalFilter(condition=[IS NOT NULL($2)]): rowcount = 225.0, cumulative cost = {165287.5 rows, 103075.2971810663 cpu, 0.0 io}, id = 9396\n LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)]): rowcount = 500.0, cumulative cost = {165062.5 rows, 102575.2971810663 cpu, 0.0 io}, id = 9395\n LogicalSort(sort0=[$1], dir0=[ASC-nulls-first]): rowcount = 500.0, cumulative cost = {164562.5 rows, 101075.2971810663 cpu, 0.0 io}, id = 9393\n LogicalProject(avg_age=[$2], state=[$0], city=[$1]): rowcount = 500.0, cumulative cost = {164062.5 rows, 26500.0 cpu, 0.0 io}, id = 9392\n LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]): rowcount = 500.0, cumulative cost = {163562.5 rows, 25000.0 cpu, 0.0 io}, id = 9391\n LogicalProject(state=[$7], city=[$5], age=[$8]): rowcount = 5000.0, cumulative cost = {163000.0 rows, 25000.0 cpu, 0.0 io}, id = 9390\n LogicalFilter(condition=[>($8, 30)]): rowcount = 5000.0, cumulative cost = {158000.0 rows, 10000.0 cpu, 0.0 io}, id = 9389\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]): rowcount = 10000.0, cumulative cost = {153000.0 rows, 0.0 cpu, 0.0 io}, id = 9388\n", + "physical": "EnumerableCalc(expr#0..1=[{inputs}], age2=[$t0]): rowcount = 225.0, cumulative cost = {2981.25 rows, 7950.0 cpu, 0.0 io}, id = 11254\n EnumerableLimit(fetch=[10000]): rowcount = 225.0, cumulative cost = {2756.25 rows, 7275.0 cpu, 0.0 io}, id = 11246\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=[<=($t1, $t2)], proj#0..1=[{exprs}], $condition=[$t3]): rowcount = 225.0, cumulative cost = {2531.25 rows, 7050.0 cpu, 0.0 io}, id = 11250\n EnumerableWindow(window#0=[window(partition {0} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]): rowcount = 450.0, cumulative cost = {2306.25 rows, 3900.0 cpu, 0.0 io}, id = 11242\n EnumerableCalc(expr#0=[{inputs}], expr#1=[2], expr#2=[+($t0, $t1)], expr#3=[IS NOT NULL($t0)], $0=[$t2], $condition=[$t3]): rowcount = 450.0, cumulative cost = {1856.25 rows, 3000.0 cpu, 0.0 io}, id = 11258\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[avg_age, state], SORT->[1 ASC FIRST], PROJECT->[avg_age]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]): rowcount = 500.0, cumulative cost = {1406.25 rows, 0.0 cpu, 0.0 io}, id = 10962\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_output_cost.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_cost.yaml new file mode 100644 index 00000000000..63b402833fc --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_cost.yaml @@ -0,0 +1,21 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]): rowcount = 56.25, cumulative cost = {165681.25 rows, 105156.5471810663 cpu, 0.0 io}, id = 7529 + LogicalProject(age2=[$2]): rowcount = 56.25, cumulative cost = {165625.0 rows, 104256.5471810663 cpu, 0.0 io}, id = 7528 + LogicalFilter(condition=[<=($3, 1)]): rowcount = 56.25, cumulative cost = {165568.75 rows, 104200.2971810663 cpu, 0.0 io}, id = 7526 + LogicalProject(avg_age=[$0], state=[$1], age2=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $2)]): rowcount = 225.0, cumulative cost = {165512.5 rows, 103975.2971810663 cpu, 0.0 io}, id = 7525 + LogicalFilter(condition=[IS NOT NULL($2)]): rowcount = 225.0, cumulative cost = {165287.5 rows, 103075.2971810663 cpu, 0.0 io}, id = 7524 + LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)]): rowcount = 500.0, cumulative cost = {165062.5 rows, 102575.2971810663 cpu, 0.0 io}, id = 7523 + LogicalSort(sort0=[$1], dir0=[ASC-nulls-first]): rowcount = 500.0, cumulative cost = {164562.5 rows, 101075.2971810663 cpu, 0.0 io}, id = 7521 + LogicalProject(avg_age=[$2], state=[$0], city=[$1]): rowcount = 500.0, cumulative cost = {164062.5 rows, 26500.0 cpu, 0.0 io}, id = 7520 + LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]): rowcount = 500.0, cumulative cost = {163562.5 rows, 25000.0 cpu, 0.0 io}, id = 7519 + LogicalProject(state=[$7], city=[$5], age=[$8]): rowcount = 5000.0, cumulative cost = {163000.0 rows, 25000.0 cpu, 0.0 io}, id = 7518 + LogicalFilter(condition=[>($8, 30)]): rowcount = 5000.0, cumulative cost = {158000.0 rows, 10000.0 cpu, 0.0 io}, id = 7517 + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]): rowcount = 10000.0, cumulative cost = {153000.0 rows, 0.0 cpu, 0.0 io}, id = 7516 + physical: | + EnumerableCalc(expr#0..1=[{inputs}], age2=[$t0]): rowcount = 225.0, cumulative cost = {2981.25 rows, 7950.0 cpu, 0.0 io}, id = 9382 + EnumerableLimit(fetch=[10000]): rowcount = 225.0, cumulative cost = {2756.25 rows, 7275.0 cpu, 0.0 io}, id = 9374 + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=[<=($t1, $t2)], proj#0..1=[{exprs}], $condition=[$t3]): rowcount = 225.0, cumulative cost = {2531.25 rows, 7050.0 cpu, 0.0 io}, id = 9378 + EnumerableWindow(window#0=[window(partition {0} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]): rowcount = 450.0, cumulative cost = {2306.25 rows, 3900.0 cpu, 0.0 io}, id = 9370 + EnumerableCalc(expr#0=[{inputs}], expr#1=[2], expr#2=[+($t0, $t1)], expr#3=[IS NOT NULL($t0)], $0=[$t2], $condition=[$t3]): rowcount = 450.0, cumulative cost = {1856.25 rows, 3000.0 cpu, 0.0 io}, id = 9386 + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[avg_age, state], SORT->[1 ASC FIRST], PROJECT->[avg_age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"city":{"terms":{"field":"city.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]): rowcount = 500.0, cumulative cost = {1406.25 rows, 0.0 cpu, 0.0 io}, id = 9090 diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_output_extended.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_extended.json new file mode 100644 index 00000000000..5ce93804ca4 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_extended.json @@ -0,0 +1,7 @@ +{ + "calcite": { + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age2=[$2])\n LogicalFilter(condition=[<=($3, 1)])\n LogicalProject(avg_age=[$0], state=[$1], age2=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $2)])\n LogicalFilter(condition=[IS NOT NULL($2)])\n LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)])\n LogicalSort(sort0=[$1], dir0=[ASC-nulls-first])\n LogicalProject(avg_age=[$2], state=[$0], city=[$1])\n LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)])\n LogicalProject(state=[$7], city=[$5], age=[$8])\n LogicalFilter(condition=[>($8, 30)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableCalc(expr#0..1=[{inputs}], age2=[$t0])\n EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=[<=($t1, $t2)], proj#0..1=[{exprs}], $condition=[$t3])\n EnumerableWindow(window#0=[window(partition {0} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])])\n EnumerableCalc(expr#0=[{inputs}], expr#1=[2], expr#2=[+($t0, $t1)], expr#3=[IS NOT NULL($t0)], $0=[$t2], $condition=[$t3])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[avg_age, state], SORT->[1 ASC FIRST], PROJECT->[avg_age]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n", + "extended": "public org.apache.calcite.linq4j.Enumerable bind(final org.apache.calcite.DataContext root) {\n final org.opensearch.sql.opensearch.storage.scan.CalciteEnumerableIndexScan v1stashed = (org.opensearch.sql.opensearch.storage.scan.CalciteEnumerableIndexScan) root.get(\"v1stashed\");\n final org.apache.calcite.linq4j.Enumerable _inputEnumerable = v1stashed.scan();\n final org.apache.calcite.linq4j.AbstractEnumerable source = new org.apache.calcite.linq4j.AbstractEnumerable(){\n public org.apache.calcite.linq4j.Enumerator enumerator() {\n return new org.apache.calcite.linq4j.Enumerator(){\n public final org.apache.calcite.linq4j.Enumerator inputEnumerator = _inputEnumerable.enumerator();\n public void reset() {\n inputEnumerator.reset();\n }\n\n public boolean moveNext() {\n while (inputEnumerator.moveNext()) {\n if ((Double) inputEnumerator.current() != null) {\n return true;\n }\n }\n return false;\n }\n\n public void close() {\n inputEnumerator.close();\n }\n\n public Object current() {\n return (Double) inputEnumerator.current() == null ? null : Double.valueOf(((Double) inputEnumerator.current()).doubleValue() + (double) 2);\n }\n\n };\n }\n\n };\n int prevStart;\n int prevEnd;\n final java.util.Comparator comparator = new java.util.Comparator(){\n public int compare(Double v0, Double v1) {\n int c;\n return 0;\n }\n\n public int compare(Object o0, Object o1) {\n return this.compare((Double) o0, (Double) o1);\n }\n\n };\n final org.apache.calcite.runtime.SortedMultiMap multiMap = new org.apache.calcite.runtime.SortedMultiMap();\n source.foreach(new org.apache.calcite.linq4j.function.Function1() {\n public Object apply(Double v) {\n Double key = v;\n multiMap.putMulti(key, v);\n return null;\n }\n public Object apply(Object v) {\n return apply(\n (Double) v);\n }\n }\n );\n final java.util.Iterator iterator = multiMap.arrays(comparator);\n final java.util.ArrayList _list = new java.util.ArrayList(\n multiMap.size());\n Long a0w0 = (Long) null;\n while (iterator.hasNext()) {\n final Object[] _rows = (Object[]) iterator.next();\n prevStart = -1;\n prevEnd = 2147483647;\n for (int i = 0; i < _rows.length; (++i)) {\n if (i != prevEnd) {\n int actualStart = i < prevEnd ? 0 : prevEnd + 1;\n prevEnd = i;\n a0w0 = Long.valueOf(((Number)org.apache.calcite.linq4j.tree.Primitive.of(long.class).numberValueRoundDown((i - 0 + 1))).longValue());\n }\n _list.add(new Object[] {\n (Double) _rows[i],\n a0w0});\n }\n }\n multiMap.clear();\n final org.apache.calcite.linq4j.Enumerable _inputEnumerable0 = org.apache.calcite.linq4j.Linq4j.asEnumerable(_list);\n final org.apache.calcite.linq4j.AbstractEnumerable child = new org.apache.calcite.linq4j.AbstractEnumerable(){\n public org.apache.calcite.linq4j.Enumerator enumerator() {\n return new org.apache.calcite.linq4j.Enumerator(){\n public final org.apache.calcite.linq4j.Enumerator inputEnumerator = _inputEnumerable0.enumerator();\n public void reset() {\n inputEnumerator.reset();\n }\n\n public boolean moveNext() {\n while (inputEnumerator.moveNext()) {\n if (org.apache.calcite.runtime.SqlFunctions.toLong(((Object[]) inputEnumerator.current())[1]) <= $L4J$C$_Number_org_apache_calcite_linq4j_tree_Primitive_of_long_class_358aa52b) {\n return true;\n }\n }\n return false;\n }\n\n public void close() {\n inputEnumerator.close();\n }\n\n public Object current() {\n final Object[] current = (Object[]) inputEnumerator.current();\n final Object input_value = current[0];\n final Object input_value0 = current[1];\n return new Object[] {\n input_value,\n input_value0};\n }\n\n static final long $L4J$C$_Number_org_apache_calcite_linq4j_tree_Primitive_of_long_class_358aa52b = ((Number)org.apache.calcite.linq4j.tree.Primitive.of(long.class).numberValueRoundDown(1)).longValue();\n };\n }\n\n };\n final org.apache.calcite.linq4j.Enumerable _inputEnumerable1 = child.take(10000);\n return new org.apache.calcite.linq4j.AbstractEnumerable(){\n public org.apache.calcite.linq4j.Enumerator enumerator() {\n return new org.apache.calcite.linq4j.Enumerator(){\n public final org.apache.calcite.linq4j.Enumerator inputEnumerator = _inputEnumerable1.enumerator();\n public void reset() {\n inputEnumerator.reset();\n }\n\n public boolean moveNext() {\n return inputEnumerator.moveNext();\n }\n\n public void close() {\n inputEnumerator.close();\n }\n\n public Object current() {\n return (Double) ((Object[]) inputEnumerator.current())[0];\n }\n\n };\n }\n\n };\n}\n\n\npublic Class getElementType() {\n return java.lang.Double.class;\n}\n\n\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_output_extended.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_extended.yaml new file mode 100644 index 00000000000..89137cfc835 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_extended.yaml @@ -0,0 +1,167 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age2=[$2]) + LogicalFilter(condition=[<=($3, 1)]) + LogicalProject(avg_age=[$0], state=[$1], age2=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $2)]) + LogicalFilter(condition=[IS NOT NULL($2)]) + LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)]) + LogicalSort(sort0=[$1], dir0=[ASC-nulls-first]) + LogicalProject(avg_age=[$2], state=[$0], city=[$1]) + LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]) + LogicalProject(state=[$7], city=[$5], age=[$8]) + LogicalFilter(condition=[>($8, 30)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0..1=[{inputs}], age2=[$t0]) + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=[<=($t1, $t2)], proj#0..1=[{exprs}], $condition=[$t3]) + EnumerableWindow(window#0=[window(partition {0} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[2], expr#2=[+($t0, $t1)], expr#3=[IS NOT NULL($t0)], $0=[$t2], $condition=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[avg_age, state], SORT->[1 ASC FIRST], PROJECT->[avg_age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"city":{"terms":{"field":"city.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + extended: |+ + public org.apache.calcite.linq4j.Enumerable bind(final org.apache.calcite.DataContext root) { + final org.opensearch.sql.opensearch.storage.scan.CalciteEnumerableIndexScan v1stashed = (org.opensearch.sql.opensearch.storage.scan.CalciteEnumerableIndexScan) root.get("v1stashed"); + final org.apache.calcite.linq4j.Enumerable _inputEnumerable = v1stashed.scan(); + final org.apache.calcite.linq4j.AbstractEnumerable source = new org.apache.calcite.linq4j.AbstractEnumerable(){ + public org.apache.calcite.linq4j.Enumerator enumerator() { + return new org.apache.calcite.linq4j.Enumerator(){ + public final org.apache.calcite.linq4j.Enumerator inputEnumerator = _inputEnumerable.enumerator(); + public void reset() { + inputEnumerator.reset(); + } + + public boolean moveNext() { + while (inputEnumerator.moveNext()) { + if ((Double) inputEnumerator.current() != null) { + return true; + } + } + return false; + } + + public void close() { + inputEnumerator.close(); + } + + public Object current() { + return (Double) inputEnumerator.current() == null ? null : Double.valueOf(((Double) inputEnumerator.current()).doubleValue() + (double) 2); + } + + }; + } + + }; + int prevStart; + int prevEnd; + final java.util.Comparator comparator = new java.util.Comparator(){ + public int compare(Double v0, Double v1) { + int c; + return 0; + } + + public int compare(Object o0, Object o1) { + return this.compare((Double) o0, (Double) o1); + } + + }; + final org.apache.calcite.runtime.SortedMultiMap multiMap = new org.apache.calcite.runtime.SortedMultiMap(); + source.foreach(new org.apache.calcite.linq4j.function.Function1() { + public Object apply(Double v) { + Double key = v; + multiMap.putMulti(key, v); + return null; + } + public Object apply(Object v) { + return apply( + (Double) v); + } + } + ); + final java.util.Iterator iterator = multiMap.arrays(comparator); + final java.util.ArrayList _list = new java.util.ArrayList( + multiMap.size()); + Long a0w0 = (Long) null; + while (iterator.hasNext()) { + final Object[] _rows = (Object[]) iterator.next(); + prevStart = -1; + prevEnd = 2147483647; + for (int i = 0; i < _rows.length; (++i)) { + if (i != prevEnd) { + int actualStart = i < prevEnd ? 0 : prevEnd + 1; + prevEnd = i; + a0w0 = Long.valueOf(((Number)org.apache.calcite.linq4j.tree.Primitive.of(long.class).numberValueRoundDown((i - 0 + 1))).longValue()); + } + _list.add(new Object[] { + (Double) _rows[i], + a0w0}); + } + } + multiMap.clear(); + final org.apache.calcite.linq4j.Enumerable _inputEnumerable0 = org.apache.calcite.linq4j.Linq4j.asEnumerable(_list); + final org.apache.calcite.linq4j.AbstractEnumerable child = new org.apache.calcite.linq4j.AbstractEnumerable(){ + public org.apache.calcite.linq4j.Enumerator enumerator() { + return new org.apache.calcite.linq4j.Enumerator(){ + public final org.apache.calcite.linq4j.Enumerator inputEnumerator = _inputEnumerable0.enumerator(); + public void reset() { + inputEnumerator.reset(); + } + + public boolean moveNext() { + while (inputEnumerator.moveNext()) { + if (org.apache.calcite.runtime.SqlFunctions.toLong(((Object[]) inputEnumerator.current())[1]) <= $L4J$C$_Number_org_apache_calcite_linq4j_tree_Primitive_of_long_class_358aa52b) { + return true; + } + } + return false; + } + + public void close() { + inputEnumerator.close(); + } + + public Object current() { + final Object[] current = (Object[]) inputEnumerator.current(); + final Object input_value = current[0]; + final Object input_value0 = current[1]; + return new Object[] { + input_value, + input_value0}; + } + + static final long $L4J$C$_Number_org_apache_calcite_linq4j_tree_Primitive_of_long_class_358aa52b = ((Number)org.apache.calcite.linq4j.tree.Primitive.of(long.class).numberValueRoundDown(1)).longValue(); + }; + } + + }; + final org.apache.calcite.linq4j.Enumerable _inputEnumerable1 = child.take(10000); + return new org.apache.calcite.linq4j.AbstractEnumerable(){ + public org.apache.calcite.linq4j.Enumerator enumerator() { + return new org.apache.calcite.linq4j.Enumerator(){ + public final org.apache.calcite.linq4j.Enumerator inputEnumerator = _inputEnumerable1.enumerator(); + public void reset() { + inputEnumerator.reset(); + } + + public boolean moveNext() { + return inputEnumerator.moveNext(); + } + + public void close() { + inputEnumerator.close(); + } + + public Object current() { + return (Double) ((Object[]) inputEnumerator.current())[0]; + } + + }; + } + + }; + } + + + public Class getElementType() { + return java.lang.Double.class; + } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_output_simple.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_simple.json new file mode 100644 index 00000000000..96e070954ec --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_simple.json @@ -0,0 +1,5 @@ +{ + "calcite": { + "logical": "LogicalSystemLimit\n LogicalProject\n LogicalFilter\n LogicalProject\n LogicalFilter\n LogicalProject\n LogicalSort\n LogicalProject\n LogicalAggregate\n LogicalProject\n LogicalFilter\n CalciteLogicalIndexScan\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_output_simple.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_simple.yaml new file mode 100644 index 00000000000..f58ffa560a7 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_simple.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit + LogicalProject + LogicalFilter + LogicalProject + LogicalFilter + LogicalProject + LogicalSort + LogicalProject + LogicalAggregate + LogicalProject + LogicalFilter + CalciteLogicalIndexScan diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_output_standard.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_standard.json new file mode 100644 index 00000000000..a50b0baa104 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_standard.json @@ -0,0 +1,6 @@ +{ + "calcite": { + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age2=[$2])\n LogicalFilter(condition=[<=($3, 1)])\n LogicalProject(avg_age=[$0], state=[$1], age2=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $2)])\n LogicalFilter(condition=[IS NOT NULL($2)])\n LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)])\n LogicalSort(sort0=[$1], dir0=[ASC-nulls-first])\n LogicalProject(avg_age=[$2], state=[$0], city=[$1])\n LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)])\n LogicalProject(state=[$7], city=[$5], age=[$8])\n LogicalFilter(condition=[>($8, 30)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableCalc(expr#0..1=[{inputs}], age2=[$t0])\n EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=[<=($t1, $t2)], proj#0..1=[{exprs}], $condition=[$t3])\n EnumerableWindow(window#0=[window(partition {0} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])])\n EnumerableCalc(expr#0=[{inputs}], expr#1=[2], expr#2=[+($t0, $t1)], expr#3=[IS NOT NULL($t0)], $0=[$t2], $condition=[$t3])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[avg_age, state], SORT->[1 ASC FIRST], PROJECT->[avg_age]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_output_standard.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_standard.yaml new file mode 100644 index 00000000000..862a45dc617 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_standard.yaml @@ -0,0 +1,21 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age2=[$2]) + LogicalFilter(condition=[<=($3, 1)]) + LogicalProject(avg_age=[$0], state=[$1], age2=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $2)]) + LogicalFilter(condition=[IS NOT NULL($2)]) + LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)]) + LogicalSort(sort0=[$1], dir0=[ASC-nulls-first]) + LogicalProject(avg_age=[$2], state=[$0], city=[$1]) + LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]) + LogicalProject(state=[$7], city=[$5], age=[$8]) + LogicalFilter(condition=[>($8, 30)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0..1=[{inputs}], age2=[$t0]) + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=[<=($t1, $t2)], proj#0..1=[{exprs}], $condition=[$t3]) + EnumerableWindow(window#0=[window(partition {0} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[2], expr#2=[+($t0, $t1)], expr#3=[IS NOT NULL($t0)], $0=[$t2], $condition=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[avg_age, state], SORT->[1 ASC FIRST], PROJECT->[avg_age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"city":{"terms":{"field":"city.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.json deleted file mode 100644 index 978d6cf1bcc..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.json +++ /dev/null @@ -1 +0,0 @@ -{"calcite":{"logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(firstname=[$1], age=[$8], address=[$2])\n LogicalFilter(condition=[AND(=($2, '671 Bristol Street'), =(-($8, 2), 30))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n","physical":"CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, address, age], SCRIPT->AND(=($1, '671 Bristol Street'), =(-($2, 2), 30)), PROJECT->[firstname, age, address], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\":{\"must\":[{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"{\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"=\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"EQUALS\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 0,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"VARCHAR\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true,\\\\n \\\\\\\"precision\\\\\\\": -1\\\\n }\\\\n },\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 1,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"VARCHAR\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true,\\\\n \\\\\\\"precision\\\\\\\": -1\\\\n }\\\\n }\\\\n ]\\\\n}\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*,\"SOURCES\":[1,2],\"DIGESTS\":[\"address\",\"671 Bristol Street\"]}},\"boost\":1.0}},{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"{\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"=\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"EQUALS\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"-\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"MINUS\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 0,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n },\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 1,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n }\\\\n ],\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n },\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 2,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n }\\\\n ]\\\\n}\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*,\"SOURCES\":[0,2,2],\"DIGESTS\":[\"age\",2,30]}},\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"firstname\",\"age\",\"address\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n"}} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.yaml new file mode 100644 index 00000000000..7938c9ea602 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(firstname=[$1], age=[$8], address=[$2]) + LogicalFilter(condition=[AND(=($2, '671 Bristol Street'), =(-($8, 2), 30))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, address, age], SCRIPT->AND(=($1, '671 Bristol Street'), =(-($2, 2), 30)), PROJECT->[firstname, age, address], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"{\\n \\\"op\\\": {\\n \\\"name\\\": \\\"=\\\",\\n \\\"kind\\\": \\\"EQUALS\\\",\\n \\\"syntax\\\": \\\"BINARY\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"dynamicParam\\\": 0,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"VARCHAR\\\",\\n \\\"nullable\\\": true,\\n \\\"precision\\\": -1\\n }\\n },\\n {\\n \\\"dynamicParam\\\": 1,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"VARCHAR\\\",\\n \\\"nullable\\\": true,\\n \\\"precision\\\": -1\\n }\\n }\\n ]\\n}\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1,2],"DIGESTS":["address","671 Bristol Street"]}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"{\\n \\\"op\\\": {\\n \\\"name\\\": \\\"=\\\",\\n \\\"kind\\\": \\\"EQUALS\\\",\\n \\\"syntax\\\": \\\"BINARY\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"op\\\": {\\n \\\"name\\\": \\\"-\\\",\\n \\\"kind\\\": \\\"MINUS\\\",\\n \\\"syntax\\\": \\\"BINARY\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"dynamicParam\\\": 0,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n },\\n {\\n \\\"dynamicParam\\\": 1,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n }\\n ],\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n },\\n {\\n \\\"dynamicParam\\\": 2,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n }\\n ]\\n}\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["age",2,30]}},"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["firstname","age","address"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/Format.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/Format.java index e0f33395194..7bf120c0157 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/Format.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/Format.java @@ -16,17 +16,11 @@ public enum Format { JDBC("jdbc"), CSV("csv"), RAW("raw"), - TABLE("table"), - // format of explain response - SIMPLE("simple"), - STANDARD("standard"), - EXTENDED("extended"), - COST("cost"); + TABLE("table"); @Getter private final String formatName; public static final Map RESPONSE_FORMATS; - public static final Map EXPLAIN_FORMATS; static { ImmutableMap.Builder builder; @@ -36,20 +30,9 @@ public enum Format { builder.put(RAW.formatName, RAW); builder.put(TABLE.formatName, TABLE); RESPONSE_FORMATS = builder.build(); - - builder = new ImmutableMap.Builder<>(); - builder.put(SIMPLE.formatName, SIMPLE); - builder.put(STANDARD.formatName, STANDARD); - builder.put(EXTENDED.formatName, EXTENDED); - builder.put(COST.formatName, COST); - EXPLAIN_FORMATS = builder.build(); } public static Optional of(String formatName) { return Optional.ofNullable(RESPONSE_FORMATS.getOrDefault(formatName, null)); } - - public static Optional ofExplain(String formatName) { - return Optional.ofNullable(EXPLAIN_FORMATS.getOrDefault(formatName, null)); - } } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/request/SqlRequestParam.java b/legacy/src/main/java/org/opensearch/sql/legacy/request/SqlRequestParam.java index 5d2bb1971b0..33ca46486a8 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/request/SqlRequestParam.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/request/SqlRequestParam.java @@ -16,7 +16,6 @@ public class SqlRequestParam { public static final String QUERY_PARAMS_ESCAPE = "escape"; private static final String DEFAULT_RESPONSE_FORMAT = "jdbc"; - private static final String DEFAULT_EXPLAIN_FORMAT = "standard"; /** * Parse the pretty params to decide whether the response should be pretty formatted. @@ -44,9 +43,8 @@ public static Format getFormat(Map requestParams, String path) { String formatName = requestParams.containsKey(QUERY_PARAMS_FORMAT) ? requestParams.get(QUERY_PARAMS_FORMAT).toLowerCase() - : isExplainRequest(path) ? DEFAULT_EXPLAIN_FORMAT : DEFAULT_RESPONSE_FORMAT; - Optional optionalFormat = - isExplainRequest(path) ? Format.ofExplain(formatName) : Format.of(formatName); + : DEFAULT_RESPONSE_FORMAT; + Optional optionalFormat = Format.of(formatName); if (optionalFormat.isPresent()) { return optionalFormat.get(); } else { @@ -55,10 +53,6 @@ public static Format getFormat(Map requestParams, String path) { } } - private static boolean isExplainRequest(String path) { - return path != null && path.endsWith("/_explain"); - } - public static boolean getEscapeOption(Map requestParams) { if (requestParams.containsKey(QUERY_PARAMS_ESCAPE)) { return Boolean.parseBoolean(requestParams.get(QUERY_PARAMS_ESCAPE)); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java index fd852d7f894..2aa0641be83 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java @@ -36,7 +36,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.locationtech.jts.geom.Point; -import org.opensearch.sql.ast.statement.Explain.ExplainFormat; +import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.calcite.CalcitePlanContext; import org.opensearch.sql.calcite.utils.CalciteToolsHelper.OpenSearchRelRunners; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; @@ -166,26 +166,26 @@ private Hook.Closeable getCodegenInHook(AtomicReference codegen) { @Override public void explain( RelNode rel, - ExplainFormat format, + ExplainMode mode, CalcitePlanContext context, ResponseListener listener) { client.schedule( () -> { try { - if (format == ExplainFormat.SIMPLE) { + if (mode == ExplainMode.SIMPLE) { String logical = RelOptUtil.toString(rel, SqlExplainLevel.NO_ATTRIBUTES); listener.onResponse( new ExplainResponse(new ExplainResponseNodeV2(logical, null, null))); } else { SqlExplainLevel level = - format == ExplainFormat.COST + mode == ExplainMode.COST ? SqlExplainLevel.ALL_ATTRIBUTES : SqlExplainLevel.EXPPLAN_ATTRIBUTES; String logical = RelOptUtil.toString(rel, level); AtomicReference physical = new AtomicReference<>(); AtomicReference javaCode = new AtomicReference<>(); try (Hook.Closeable closeable = getPhysicalPlanInHook(physical, level)) { - if (format == ExplainFormat.EXTENDED) { + if (mode == ExplainMode.EXTENDED) { getCodegenInHook(javaCode); CalcitePlanContext.skipEncoding.set(true); } diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/request/PPLQueryRequestFactory.java b/plugin/src/main/java/org/opensearch/sql/plugin/request/PPLQueryRequestFactory.java index 0908d27b491..0dd8e1a9651 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/request/PPLQueryRequestFactory.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/request/PPLQueryRequestFactory.java @@ -7,6 +7,8 @@ import java.util.Map; import java.util.Optional; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.json.JSONException; import org.json.JSONObject; import org.opensearch.rest.RestRequest; @@ -16,12 +18,16 @@ /** Factory of {@link PPLQueryRequest}. */ public class PPLQueryRequestFactory { + private static final Logger LOG = LogManager.getLogger(PPLQueryRequestFactory.class); + private static final String PPL_URL_PARAM_KEY = "ppl"; private static final String PPL_FIELD_NAME = "query"; private static final String QUERY_PARAMS_FORMAT = "format"; + private static final String QUERY_PARAMS_EXPLAIN_MODE = "mode"; private static final String QUERY_PARAMS_SANITIZE = "sanitize"; private static final String DEFAULT_RESPONSE_FORMAT = "jdbc"; - private static final String DEFAULT_EXPLAIN_FORMAT = "standard"; + private static final String DEFAULT_EXPLAIN_FORMAT = "json"; + private static final String DEFAULT_EXPLAIN_MODE = "standard"; private static final String QUERY_PARAMS_PRETTY = "pretty"; private static final String QUERY_PARAMS_PROFILE = "profile"; @@ -57,6 +63,20 @@ private static PPLQueryRequest parsePPLRequestFromPayload(RestRequest restReques String content = restRequest.content().utf8ToString(); JSONObject jsonContent; Format format = getFormat(restRequest.params(), restRequest.rawPath()); + String explainMode; + // For backward compatible consideration, if the format=[simple, standard, extended, cost], we + // accept it as well and view it as mode and use json format. + // TODO: deprecated after 4.x + if (Format.isExplainMode(format)) { + // Log deprecation warning for legacy format-based explain mode usage + LOG.warn( + "Using 'format' parameter for explain mode is deprecated. Please use 'mode' parameter" + + " instead. This will be removed in 4.x."); + explainMode = format.getFormatName(); + format = Format.JSON; + } else { + explainMode = getExplainMode(restRequest.params(), restRequest.rawPath()); + } boolean pretty = getPrettyOption(restRequest.params()); try { jsonContent = new JSONObject(content); @@ -70,6 +90,7 @@ private static PPLQueryRequest parsePPLRequestFromPayload(RestRequest restReques jsonContent, restRequest.path(), format.getFormatName(), + explainMode, enableProfile); // set sanitize option if csv format if (format.equals(Format.CSV)) { @@ -129,4 +150,13 @@ private static boolean isProfileSupported(String path, Format format, String que format != null && DEFAULT_RESPONSE_FORMAT.equalsIgnoreCase(format.getFormatName()); return !explainPath && !explainQuery && isJdbcFormat; } + + private static String getExplainMode(Map requestParams, String path) { + if (!isExplainRequest(path)) { + return null; + } + return requestParams + .getOrDefault(QUERY_PARAMS_EXPLAIN_MODE, DEFAULT_EXPLAIN_MODE) + .toLowerCase(); + } } diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java index 81b2b5d26dc..54efb861cf5 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java @@ -77,7 +77,7 @@ public String getName() { @Override protected Set responseParams() { Set responseParams = new HashSet<>(super.responseParams()); - responseParams.addAll(Arrays.asList("format", "sanitize")); + responseParams.addAll(Arrays.asList("format", "mode", "sanitize")); return responseParams; } diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryRequest.java b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryRequest.java index 65663bbec00..e342d9a90f0 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryRequest.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryRequest.java @@ -34,6 +34,7 @@ public class TransportPPLQueryRequest extends ActionRequest { @Getter private final String path; @Getter private String format = ""; + @Getter private String explainMode; @Setter @Getter @@ -59,6 +60,7 @@ public TransportPPLQueryRequest(PPLQueryRequest pplQueryRequest) { sanitize = pplQueryRequest.sanitize(); style = pplQueryRequest.style(); profile = pplQueryRequest.profile(); + explainMode = pplQueryRequest.mode().getModeName(); } /** Constructor of TransportPPLQueryRequest from StreamInput. */ @@ -66,6 +68,7 @@ public TransportPPLQueryRequest(StreamInput in) throws IOException { super(in); pplQuery = in.readOptionalString(); format = in.readOptionalString(); + explainMode = in.readOptionalString(); String jsonContentString = in.readOptionalString(); jsonContent = jsonContentString != null ? new JSONObject(jsonContentString) : null; path = in.readOptionalString(); @@ -98,6 +101,7 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeOptionalString(pplQuery); out.writeOptionalString(format); + out.writeOptionalString(explainMode); out.writeOptionalString(jsonContent != null ? jsonContent.toString() : null); out.writeOptionalString(path); out.writeBoolean(sanitize); @@ -137,7 +141,7 @@ public ActionRequestValidationException validate() { /** Convert to PPLQueryRequest. */ public PPLQueryRequest toPPLQueryRequest() { PPLQueryRequest pplQueryRequest = - new PPLQueryRequest(pplQuery, jsonContent, path, format, profile); + new PPLQueryRequest(pplQuery, jsonContent, path, format, explainMode, profile); pplQueryRequest.sanitize(sanitize); pplQueryRequest.style(style); return pplQueryRequest; diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java b/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java index ccd4d49dd4a..ecae97283ed 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java @@ -98,6 +98,7 @@ private AbstractPlan plan( AstStatementBuilder.StatementBuilderContext.builder() .isExplain(request.isExplainRequest()) .format(request.getFormat()) + .explainMode(request.getExplainMode()) .build())); log.info( diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/domain/PPLQueryRequest.java b/ppl/src/main/java/org/opensearch/sql/ppl/domain/PPLQueryRequest.java index 321e1d410c4..ef21e0f2803 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/domain/PPLQueryRequest.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/domain/PPLQueryRequest.java @@ -11,6 +11,7 @@ import lombok.Setter; import lombok.experimental.Accessors; import org.json.JSONObject; +import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.protocol.response.format.Format; import org.opensearch.sql.protocol.response.format.JsonResponseFormatter; @@ -24,6 +25,7 @@ public class PPLQueryRequest { @Getter private final JSONObject jsonContent; @Getter private final String path; @Getter private String format = ""; + @Getter private String explainMode; @Setter @Getter @@ -45,16 +47,22 @@ public PPLQueryRequest(String pplQuery, JSONObject jsonContent, String path) { } public PPLQueryRequest(String pplQuery, JSONObject jsonContent, String path, String format) { - this(pplQuery, jsonContent, path, format, false); + this(pplQuery, jsonContent, path, format, ExplainMode.STANDARD.getModeName(), false); } /** Constructor of PPLQueryRequest. */ public PPLQueryRequest( - String pplQuery, JSONObject jsonContent, String path, String format, boolean profile) { + String pplQuery, + JSONObject jsonContent, + String path, + String format, + String explainMode, + boolean profile) { this.pplQuery = pplQuery; this.jsonContent = jsonContent; this.path = Optional.ofNullable(path).orElse(DEFAULT_PPL_PATH); this.format = format; + this.explainMode = explainMode; this.profile = profile; } @@ -81,4 +89,8 @@ public Format format() { String.format(Locale.ROOT, "response in %s format is not supported.", format)); } } + + public ExplainMode mode() { + return ExplainMode.of(explainMode); + } } diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstStatementBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstStatementBuilder.java index a18da36c487..9d8338ecea7 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstStatementBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstStatementBuilder.java @@ -38,7 +38,7 @@ public Statement visitPplStatement(OpenSearchPPLParser.PplStatementContext ctx) return new Explain(query, PPL, ctx.explainStatement().explainMode().getText()); } } else { - return context.isExplain ? new Explain(query, PPL, context.format) : query; + return context.isExplain ? new Explain(query, PPL, context.explainMode) : query; } } @@ -53,6 +53,7 @@ public static class StatementBuilderContext { private final boolean isExplain; private final int fetchSize; private final String format; + private final String explainMode; } private UnresolvedPlan addSelectAll(UnresolvedPlan plan) { diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java index 7d04ad8e6ad..a1e31e896dc 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -153,7 +153,7 @@ public String visitQuery(Query node, String context) { public String visitExplain(Explain node, String context) { return StringUtils.format( "explain %s %s", - node.getFormat().name().toLowerCase(Locale.ROOT), node.getStatement().accept(this, null)); + node.getMode().name().toLowerCase(Locale.ROOT), node.getStatement().accept(this, null)); } @Override diff --git a/protocol/src/main/java/org/opensearch/sql/protocol/response/format/Format.java b/protocol/src/main/java/org/opensearch/sql/protocol/response/format/Format.java index 14204554ff3..6db28e4d4ab 100644 --- a/protocol/src/main/java/org/opensearch/sql/protocol/response/format/Format.java +++ b/protocol/src/main/java/org/opensearch/sql/protocol/response/format/Format.java @@ -9,6 +9,7 @@ import com.google.common.collect.ImmutableMap; import java.util.Map; import java.util.Optional; +import java.util.Set; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -18,13 +19,18 @@ public enum Format { CSV("csv"), RAW("raw"), VIZ("viz"), - // format of explain response + + /*----- explain output format ------*/ + /** Returns explain output json format */ + JSON("json"), + /** Returns explain output in yaml format */ + YAML("yaml"), + + /*---- backward compatible format of explain response -----*/ SIMPLE("simple"), STANDARD("standard"), EXTENDED("extended"), - COST("cost"), - /** Returns explain output in yaml format */ - YAML("yaml"); + COST("cost"); @Getter private final String formatName; @@ -32,6 +38,8 @@ public enum Format { public static final Map EXPLAIN_FORMATS; + public static final Set EXPLAIN_MODES; + static { ImmutableMap.Builder builder; builder = new ImmutableMap.Builder<>(); @@ -42,12 +50,15 @@ public enum Format { RESPONSE_FORMATS = builder.build(); builder = new ImmutableMap.Builder<>(); + builder.put(JSON.formatName, JSON); + builder.put(YAML.formatName, YAML); builder.put(SIMPLE.formatName, SIMPLE); builder.put(STANDARD.formatName, STANDARD); builder.put(EXTENDED.formatName, EXTENDED); builder.put(COST.formatName, COST); - builder.put(YAML.formatName, YAML); EXPLAIN_FORMATS = builder.build(); + + EXPLAIN_MODES = Set.of(SIMPLE, STANDARD, EXTENDED, COST); } public static Optional of(String formatName) { @@ -56,7 +67,11 @@ public static Optional of(String formatName) { } public static Optional ofExplain(String formatName) { - String format = Strings.isNullOrEmpty(formatName) ? "standard" : formatName.toLowerCase(); + String format = Strings.isNullOrEmpty(formatName) ? "json" : formatName.toLowerCase(); return Optional.ofNullable(EXPLAIN_FORMATS.getOrDefault(format, null)); } + + public static boolean isExplainMode(Format format) { + return EXPLAIN_MODES.contains(format); + } } diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/FormatTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/FormatTest.java index fbe12ef44b3..2b1b9dde851 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/FormatTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/FormatTest.java @@ -48,6 +48,7 @@ void extended() { Optional format = Format.ofExplain("extended"); assertTrue(format.isPresent()); assertEquals(Format.EXTENDED, format.get()); + assertTrue(Format.isExplainMode(format.get())); } @Test @@ -61,7 +62,7 @@ void yaml() { void defaultExplainFormat() { Optional format = Format.ofExplain(""); assertTrue(format.isPresent()); - assertEquals(Format.STANDARD, format.get()); + assertEquals(Format.JSON, format.get()); } @Test