From 9a41cb4ca9a0113cb9ab00d48c80e430101a6609 Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Wed, 15 Oct 2025 16:55:38 +0800 Subject: [PATCH 01/35] WIP: Make poc implementation for chart command Signed-off-by: Yuanchun Shen --- .../sql/ast/AbstractNodeVisitor.java | 5 + .../org/opensearch/sql/ast/tree/Chart.java | 46 +++++++++ .../sql/calcite/CalciteRelNodeVisitor.java | 93 ++++++++++++++++++- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 3 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 19 ++++ .../opensearch/sql/ppl/parser/AstBuilder.java | 85 ++++++++++------- .../sql/ppl/parser/AstExpressionBuilder.java | 54 +++++++++++ .../sql/ppl/utils/ArgumentFactory.java | 16 ++++ 8 files changed, 284 insertions(+), 37 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/ast/tree/Chart.java diff --git a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java index f5d2a1623b3..28d39ca39bc 100644 --- a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java @@ -49,6 +49,7 @@ import org.opensearch.sql.ast.tree.Append; import org.opensearch.sql.ast.tree.AppendCol; import org.opensearch.sql.ast.tree.Bin; +import org.opensearch.sql.ast.tree.Chart; import org.opensearch.sql.ast.tree.CloseCursor; import org.opensearch.sql.ast.tree.Dedupe; import org.opensearch.sql.ast.tree.Eval; @@ -274,6 +275,10 @@ public T visitReverse(Reverse node, C context) { return visitChildren(node, context); } + public T visitChart(Chart node, C context) { + return visitChildren(node, context); + } + public T visitTimechart(Timechart node, C context) { return visitChildren(node, context); } diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/Chart.java b/core/src/main/java/org/opensearch/sql/ast/tree/Chart.java new file mode 100644 index 00000000000..5d4a036bc70 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/tree/Chart.java @@ -0,0 +1,46 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.tree; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.expression.Argument; +import org.opensearch.sql.ast.expression.UnresolvedExpression; + +/** AST node represent chart command. */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) +@AllArgsConstructor +@lombok.Builder(toBuilder = true) +public class Chart extends UnresolvedPlan { + private UnresolvedPlan child; + private UnresolvedExpression rowSplit; + private UnresolvedExpression columnSplit; + private List aggregationFunctions; + private List arguments; + + @Override + public UnresolvedPlan attach(UnresolvedPlan child) { + this.child = child; + return this; + } + + @Override + public List getChild() { + return this.child == null ? ImmutableList.of() : ImmutableList.of(this.child); + } + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitChart(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 2c90f059986..9adb628514b 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -98,6 +98,7 @@ import org.opensearch.sql.ast.tree.Append; import org.opensearch.sql.ast.tree.AppendCol; import org.opensearch.sql.ast.tree.Bin; +import org.opensearch.sql.ast.tree.Chart; import org.opensearch.sql.ast.tree.CloseCursor; import org.opensearch.sql.ast.tree.Dedupe; import org.opensearch.sql.ast.tree.Eval; @@ -1085,6 +1086,11 @@ private Pair, List> resolveAttributesForAggregation( @Override public RelNode visitAggregation(Aggregation node, CalcitePlanContext context) { + return visitAggregationAndReturnProjection(node, context).getLeft(); + } + + private Pair> visitAggregationAndReturnProjection( + Aggregation node, CalcitePlanContext context) { visitChildren(node, context); List aggExprList = node.getAggExprList(); @@ -1169,7 +1175,7 @@ public RelNode visitAggregation(Aggregation node, CalcitePlanContext context) { reordered.addAll(aliasedGroupByList); context.relBuilder.project(reordered); - return context.relBuilder.peek(); + return Pair.of(context.relBuilder.peek(), reordered); } private Optional getTimeSpanField(UnresolvedExpression expr) { @@ -2019,6 +2025,90 @@ private String getValueFunctionName(UnresolvedExpression aggregateFunction) { return sb.toString(); } + @Override + public RelNode visitChart(Chart node, CalcitePlanContext context) { + visitChildren(node, context); + ArgumentMap argMap = ArgumentMap.of(node.getArguments()); + List groupExprList = new ArrayList<>(); + UnresolvedExpression span; + if (node.getColumnSplit() instanceof Span && node.getRowSplit() instanceof Span) { + throw new UnsupportedOperationException("It is not supported to have two span splits"); + } else if (node.getRowSplit() instanceof Span) { + if (node.getColumnSplit() != null) { + groupExprList.add(node.getColumnSplit()); + } + span = node.getRowSplit(); + } else if (node.getColumnSplit() instanceof Span) { + if (node.getRowSplit() != null) { + groupExprList.add(node.getRowSplit()); + } + span = node.getColumnSplit(); + } else { + groupExprList.addAll( + Stream.of(node.getRowSplit(), node.getColumnSplit()).filter(Objects::nonNull).toList()); + span = null; + } + Aggregation aggregation = + new Aggregation(node.getAggregationFunctions(), List.of(), groupExprList, span, List.of()); + Pair> aggregated = + visitAggregationAndReturnProjection(aggregation, context); + // If row or column split does not present or limit equals 0, this is the same as `stats agg + // [group by col]` + + Integer limit = + Optional.ofNullable(argMap.get("limit")).map(l -> (Integer) l.getValue()).orElse(10); + Boolean top = + Optional.ofNullable(argMap.get("top")).map(t -> (Boolean) t.getValue()).orElse(true); + if (node.getRowSplit() == null || node.getColumnSplit() == null || Objects.equals(limit, 0)) { + return aggregated.getLeft(); + } + List projected = aggregated.getRight(); + String columSplitName = aggregated.getLeft().getRowType().getFieldNames().getLast(); + RelBuilder relBuilder = context.relBuilder; + // 0: agg; 2: column-split + relBuilder.project(relBuilder.field(0), relBuilder.field(2)); + relBuilder.filter(relBuilder.isNotNull(relBuilder.field(1))); + // 1: column split; 0: agg + relBuilder.aggregate( + relBuilder.groupKey(relBuilder.field(1)), + relBuilder.sum(relBuilder.field(0)).as("__grand_total__")); // results: group key, agg calls + RexNode grandTotal = relBuilder.field("__grand_total__"); + if (top) { + grandTotal = relBuilder.desc(grandTotal); + } + RexNode rowNum = + PlanUtils.makeOver( + context, + BuiltinFunctionName.ROW_NUMBER, + relBuilder.literal(1), + List.of(), + List.of(), + List.of(grandTotal), + WindowFrame.toCurrentRow()); + relBuilder.projectPlus(relBuilder.alias(rowNum, "__row_number__")); + RelNode ranked = relBuilder.build(); + + relBuilder.push(aggregated.getLeft()); + relBuilder.push(ranked); + + // on column-split = group key + relBuilder.join( + JoinRelType.INNER, relBuilder.equals(relBuilder.field(2, 0, 2), relBuilder.field(2, 1, 0))); + RexNode caseExpr = + relBuilder.alias( + relBuilder.call( + SqlStdOperatorTable.CASE, + relBuilder.call( + SqlStdOperatorTable.LESS_THAN_OR_EQUAL, + relBuilder.field("__row_number__"), + relBuilder.literal(limit)), + relBuilder.field(2), + relBuilder.literal("OTHER")), + columSplitName); + relBuilder.project(relBuilder.field(0), relBuilder.field(1), caseExpr); + return relBuilder.peek(); + } + /** Transforms timechart command into SQL-based operations. */ @Override public RelNode visitTimechart( @@ -2136,7 +2226,6 @@ private RelNode buildTopCategoriesQuery( if (limit > 0) { context.relBuilder.limit(0, limit); } - return context.relBuilder.build(); } diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index bb3ff245e49..ce5e5a06e5a 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -45,6 +45,7 @@ ML: 'ML'; FILLNULL: 'FILLNULL'; FLATTEN: 'FLATTEN'; TRENDLINE: 'TRENDLINE'; +CHART: 'CHART'; TIMECHART: 'TIMECHART'; APPENDCOL: 'APPENDCOL'; EXPAND: 'EXPAND'; @@ -76,6 +77,7 @@ RIGHT_HINT: 'HINT.RIGHT'; // COMMAND ASSIST KEYWORDS AS: 'AS'; BY: 'BY'; +OVER: 'OVER'; SOURCE: 'SOURCE'; INDEX: 'INDEX'; A: 'A'; @@ -92,6 +94,7 @@ COST: 'COST'; EXTENDED: 'EXTENDED'; OVERRIDE: 'OVERRIDE'; OVERWRITE: 'OVERWRITE'; +BOTTOM: 'BOTTOM'; // SORT FIELD KEYWORDS // TODO #3180: Fix broken sort functionality diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 95171cfa763..a2451adabe0 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -77,6 +77,7 @@ commands | flattenCommand | reverseCommand | regexCommand + | chartCommand | timechartCommand | rexCommand | replaceCommand @@ -258,6 +259,24 @@ reverseCommand : REVERSE ; +chartCommand + : CHART chartOptions* statsAggTerm (COMMA statsAggTerm)* (OVER rowSplit)? (BY columnSplit)? + | CHART chartOptions* statsAggTerm (COMMA statsAggTerm)* BY rowSplit (COMMA)? columnSplit + ; + +chartOptions + : LIMIT EQUAL (TOP | BOTTOM)? integerLiteral + | USEOTHER EQUAL booleanLiteral + ; + +rowSplit + : fieldExpression binOption* + ; + +columnSplit + : fieldExpression binOption* + ; + timechartCommand : TIMECHART timechartParameter* statsFunction (BY fieldExpression)? ; diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index b7e33246027..149ecbf9330 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -42,6 +42,7 @@ import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.tree.ParseTree; +import org.antlr.v4.runtime.tree.TerminalNode; import org.apache.commons.lang3.tuple.Pair; import org.opensearch.sql.ast.EmptySourcePropagateVisitor; import org.opensearch.sql.ast.dsl.AstDSL; @@ -50,6 +51,7 @@ import org.opensearch.sql.ast.tree.Aggregation; import org.opensearch.sql.ast.tree.Append; import org.opensearch.sql.ast.tree.AppendCol; +import org.opensearch.sql.ast.tree.Chart; import org.opensearch.sql.ast.tree.CountBin; import org.opensearch.sql.ast.tree.Dedupe; import org.opensearch.sql.ast.tree.DefaultBin; @@ -499,60 +501,39 @@ public UnresolvedPlan visitBinCommand(BinCommandContext ctx) { UnresolvedExpression aligntime = null; UnresolvedExpression start = null; UnresolvedExpression end = null; - + String errorFormat = "Duplicate %s parameter in bin command"; // Process each bin option: detect duplicates and assign values in one shot for (OpenSearchPPLParser.BinOptionContext option : ctx.binOption()) { + UnresolvedExpression resolvedOption = internalVisitExpression(option); // SPAN parameter if (option.span != null) { - if (!seenParams.add("SPAN")) { - throw new IllegalArgumentException("Duplicate SPAN parameter in bin command"); - } - span = internalVisitExpression(option.span); + checkParamDuplication(seenParams, option.SPAN(), errorFormat); + span = resolvedOption; } - // BINS parameter if (option.bins != null) { - if (!seenParams.add("BINS")) { - throw new IllegalArgumentException("Duplicate BINS parameter in bin command"); - } - bins = Integer.parseInt(option.bins.getText()); + checkParamDuplication(seenParams, option.SPAN(), errorFormat); + bins = (Integer) ((Literal) resolvedOption).getValue(); } - // MINSPAN parameter if (option.minspan != null) { - if (!seenParams.add("MINSPAN")) { - throw new IllegalArgumentException("Duplicate MINSPAN parameter in bin command"); - } - minspan = internalVisitExpression(option.minspan); + checkParamDuplication(seenParams, option.MINSPAN(), errorFormat); + minspan = resolvedOption; } - // ALIGNTIME parameter if (option.aligntime != null) { - if (!seenParams.add("ALIGNTIME")) { - throw new IllegalArgumentException("Duplicate ALIGNTIME parameter in bin command"); - } - aligntime = - option.aligntime.EARLIEST() != null - ? org.opensearch.sql.ast.dsl.AstDSL.stringLiteral("earliest") - : option.aligntime.LATEST() != null - ? org.opensearch.sql.ast.dsl.AstDSL.stringLiteral("latest") - : internalVisitExpression(option.aligntime.literalValue()); + checkParamDuplication(seenParams, option.ALIGNTIME(), errorFormat); + aligntime = resolvedOption; } - // START parameter if (option.start != null) { - if (!seenParams.add("START")) { - throw new IllegalArgumentException("Duplicate START parameter in bin command"); - } - start = internalVisitExpression(option.start); + checkParamDuplication(seenParams, option.START(), errorFormat); + start = resolvedOption; } - // END parameter if (option.end != null) { - if (!seenParams.add("END")) { - throw new IllegalArgumentException("Duplicate END parameter in bin command"); - } - end = internalVisitExpression(option.end); + checkParamDuplication(seenParams, option.END(), errorFormat); + end = resolvedOption; } } @@ -581,6 +562,14 @@ public UnresolvedPlan visitBinCommand(BinCommandContext ctx) { } } + private void checkParamDuplication( + Set seenParams, TerminalNode terminalNode, String errorFormat) { + String paramName = terminalNode.getText(); + if (!seenParams.add(paramName)) { + throw new IllegalArgumentException(StringUtils.format(errorFormat, paramName)); + } + } + /** Sort command. */ @Override public UnresolvedPlan visitSortCommand(SortCommandContext ctx) { @@ -618,6 +607,32 @@ public UnresolvedPlan visitReverseCommand(OpenSearchPPLParser.ReverseCommandCont return new Reverse(); } + /** Chart command. */ + @Override + public UnresolvedPlan visitChartCommand(OpenSearchPPLParser.ChartCommandContext ctx) { + UnresolvedExpression rowSplit = + ctx.rowSplit() == null ? null : internalVisitExpression(ctx.rowSplit()); + UnresolvedExpression columnSplit = + ctx.columnSplit() == null ? null : internalVisitExpression(ctx.columnSplit()); + List arguments = ArgumentFactory.getArgumentList(ctx); + ImmutableList.Builder aggListBuilder = new ImmutableList.Builder<>(); + for (OpenSearchPPLParser.StatsAggTermContext aggCtx : ctx.statsAggTerm()) { + UnresolvedExpression aggExpression = internalVisitExpression(aggCtx.statsFunction()); + String name = + aggCtx.alias == null + ? getTextInQuery(aggCtx) + : StringUtils.unquoteIdentifier(aggCtx.alias.getText()); + Alias alias = new Alias(name, aggExpression); + aggListBuilder.add(alias); + } + return Chart.builder() + .rowSplit(rowSplit) + .columnSplit(columnSplit) + .aggregationFunctions(aggListBuilder.build()) + .arguments(arguments) + .build(); + } + /** Timechart command. */ @Override public UnresolvedPlan visitTimechartCommand(OpenSearchPPLParser.TimechartCommandContext ctx) { diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index 4a5230d356e..5dc1bf44d86 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -1030,4 +1030,58 @@ public UnresolvedExpression visitTimeModifierExpression( : SearchComparison.Operator.LESS_OR_EQUAL; return new SearchComparison(implicitTimestampField, operator, osDateMathLiteral); } + + @Override + public UnresolvedExpression visitBinOption(OpenSearchPPLParser.BinOptionContext ctx) { + UnresolvedExpression option; + if (ctx.span != null) { + option = visit(ctx.span); + } else if (ctx.bins != null) { + option = visit(ctx.bins); + } else if (ctx.minspan != null) { + option = visit(ctx.minspan); + } else if (ctx.aligntime != null) { + option = + ctx.aligntime.EARLIEST() != null + ? org.opensearch.sql.ast.dsl.AstDSL.stringLiteral("earliest") + : ctx.aligntime.LATEST() != null + ? org.opensearch.sql.ast.dsl.AstDSL.stringLiteral("latest") + : visit(ctx.aligntime.literalValue()); + } else if (ctx.start != null) { + option = visit(ctx.start); + } else if (ctx.end != null) { + option = visit(ctx.end); + } else { + throw new SyntaxCheckException(StringUtils.format("Unknown bin option: %s", ctx.getText())); + } + return option; + } + + @Override + public UnresolvedExpression visitRowSplit(OpenSearchPPLParser.RowSplitContext ctx) { + // TODO: options ignored for now + Field field = (Field) visit(ctx.fieldExpression()); + for (var option : ctx.binOption()) { + if (option.span != null) { + return AstDSL.alias( + field.getField().toString(), + AstDSL.spanFromSpanLengthLiteral(field, (Literal) visit(option.binSpanValue()))); + } + } + return AstDSL.alias(ctx.fieldExpression().getText(), field); + } + + @Override + public UnresolvedExpression visitColumnSplit(OpenSearchPPLParser.ColumnSplitContext ctx) { + Field field = (Field) visit(ctx.fieldExpression()); + for (var option : ctx.binOption()) { + if (option.span != null) { + return AstDSL.alias( + field.getField().toString(), + AstDSL.spanFromSpanLengthLiteral(field, (Literal) visit(option.binSpanValue()))); + } + } + // TODO: options ignored for now + return AstDSL.alias(ctx.fieldExpression().getText(), field); + } } diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java index 8f58e41f5e3..4be1cd8e116 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java @@ -10,6 +10,7 @@ import java.util.Collections; import java.util.List; import org.antlr.v4.runtime.ParserRuleContext; +import org.opensearch.sql.ast.dsl.AstDSL; import org.opensearch.sql.ast.expression.Argument; import org.opensearch.sql.ast.expression.DataType; import org.opensearch.sql.ast.expression.Literal; @@ -19,6 +20,7 @@ import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.BooleanLiteralContext; +import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.ChartCommandContext; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.DecimalLiteralContext; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.DedupCommandContext; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.DefaultSortFieldContext; @@ -198,6 +200,20 @@ public static List getArgumentList(TopCommandContext ctx) { : new Argument("showCount", new Literal(true, DataType.BOOLEAN))); } + public static List getArgumentList(ChartCommandContext ctx) { + List arguments = new ArrayList<>(); + for (var optionCtx : ctx.chartOptions()) { + if (optionCtx.LIMIT() != null) { + arguments.add(new Argument("limit", getArgumentValue(optionCtx.integerLiteral()))); + // not specified | top presents -> true; bottom presents -> false + arguments.add(new Argument("top", AstDSL.booleanLiteral(optionCtx.BOTTOM() == null))); + } else if (optionCtx.USEOTHER() != null) { + arguments.add(new Argument("useother", getArgumentValue(optionCtx.booleanLiteral()))); + } + } + return arguments; + } + /** * Get list of {@link Argument}. * From 04084240e8191a3dbda044300d08fd6b331e9283 Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Thu, 16 Oct 2025 11:30:09 +0800 Subject: [PATCH 02/35] Support param useother and otherstr Signed-off-by: Yuanchun Shen --- .../sql/calcite/CalciteRelNodeVisitor.java | 39 +++++++++++++------ ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 1 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 1 + .../opensearch/sql/ppl/parser/AstBuilder.java | 2 +- .../sql/ppl/utils/ArgumentFactory.java | 2 + 5 files changed, 32 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 9adb628514b..a66ba09d446 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -2059,6 +2059,10 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { Optional.ofNullable(argMap.get("limit")).map(l -> (Integer) l.getValue()).orElse(10); Boolean top = Optional.ofNullable(argMap.get("top")).map(t -> (Boolean) t.getValue()).orElse(true); + Boolean useOther = + Optional.ofNullable(argMap.get("useother")).map(u -> (Boolean) u.getValue()).orElse(true); + String otherStr = + Optional.ofNullable(argMap.get("otherstr")).map(o -> (String) o.getValue()).orElse("OTHER"); if (node.getRowSplit() == null || node.getColumnSplit() == null || Objects.equals(limit, 0)) { return aggregated.getLeft(); } @@ -2094,18 +2098,29 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { // on column-split = group key relBuilder.join( JoinRelType.INNER, relBuilder.equals(relBuilder.field(2, 0, 2), relBuilder.field(2, 1, 0))); - RexNode caseExpr = - relBuilder.alias( - relBuilder.call( - SqlStdOperatorTable.CASE, - relBuilder.call( - SqlStdOperatorTable.LESS_THAN_OR_EQUAL, - relBuilder.field("__row_number__"), - relBuilder.literal(limit)), - relBuilder.field(2), - relBuilder.literal("OTHER")), - columSplitName); - relBuilder.project(relBuilder.field(0), relBuilder.field(1), caseExpr); + + RexNode condition = + relBuilder.call( + SqlStdOperatorTable.LESS_THAN_OR_EQUAL, + relBuilder.field("__row_number__"), + relBuilder.literal(limit)); + RexNode columnSplitExpr; + if (useOther) { + columnSplitExpr = + relBuilder.call( + SqlStdOperatorTable.CASE, + condition, + relBuilder.field(2), + relBuilder.literal(otherStr)); + } else { + relBuilder.filter(condition); + columnSplitExpr = relBuilder.field(2); + } + + relBuilder.project( + relBuilder.field(0), + relBuilder.field(1), + relBuilder.alias(columnSplitExpr, columSplitName)); return relBuilder.peek(); } diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index ce5e5a06e5a..85b2d267454 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -134,6 +134,7 @@ COUNTFIELD: 'COUNTFIELD'; SHOWCOUNT: 'SHOWCOUNT'; LIMIT: 'LIMIT'; USEOTHER: 'USEOTHER'; +OTHERSTR: 'OTHERSTR'; INPUT: 'INPUT'; OUTPUT: 'OUTPUT'; PATH: 'PATH'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index a2451adabe0..272243651c2 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -267,6 +267,7 @@ chartCommand chartOptions : LIMIT EQUAL (TOP | BOTTOM)? integerLiteral | USEOTHER EQUAL booleanLiteral + | OTHERSTR EQUAL stringLiteral ; rowSplit diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index 149ecbf9330..00a4e10c0e6 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -512,7 +512,7 @@ public UnresolvedPlan visitBinCommand(BinCommandContext ctx) { } // BINS parameter if (option.bins != null) { - checkParamDuplication(seenParams, option.SPAN(), errorFormat); + checkParamDuplication(seenParams, option.BINS(), errorFormat); bins = (Integer) ((Literal) resolvedOption).getValue(); } // MINSPAN parameter diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java index 4be1cd8e116..a2be9b3e0d9 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java @@ -209,6 +209,8 @@ public static List getArgumentList(ChartCommandContext ctx) { arguments.add(new Argument("top", AstDSL.booleanLiteral(optionCtx.BOTTOM() == null))); } else if (optionCtx.USEOTHER() != null) { arguments.add(new Argument("useother", getArgumentValue(optionCtx.booleanLiteral()))); + } else if (optionCtx.OTHERSTR() != null) { + arguments.add(new Argument("otherstr", getArgumentValue(optionCtx.stringLiteral()))); } } return arguments; From 851f536c8c64092633af53697c77537bd7fdf8da Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Fri, 17 Oct 2025 18:03:34 +0800 Subject: [PATCH 03/35] Support usenull and nullstr (when both row split and col split present) Signed-off-by: Yuanchun Shen --- .../org/opensearch/sql/analysis/Analyzer.java | 6 +++ .../org/opensearch/sql/ast/tree/Chart.java | 9 ++++ .../sql/calcite/CalciteRelNodeVisitor.java | 53 +++++++++++-------- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 2 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 2 + .../sql/ppl/utils/ArgumentFactory.java | 4 ++ 6 files changed, 53 insertions(+), 23 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index 9f78b245942..297998d028c 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -61,6 +61,7 @@ import org.opensearch.sql.ast.tree.Append; import org.opensearch.sql.ast.tree.AppendCol; import org.opensearch.sql.ast.tree.Bin; +import org.opensearch.sql.ast.tree.Chart; import org.opensearch.sql.ast.tree.CloseCursor; import org.opensearch.sql.ast.tree.Dedupe; import org.opensearch.sql.ast.tree.Eval; @@ -764,6 +765,11 @@ public LogicalPlan visitSpath(SPath node, AnalysisContext context) { throw getOnlyForCalciteException("Spath"); } + @Override + public LogicalPlan visitChart(Chart node, AnalysisContext context) { + throw getOnlyForCalciteException("Chart"); + } + @Override public LogicalPlan visitTimechart(Timechart node, AnalysisContext context) { throw getOnlyForCalciteException("Timechart"); diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/Chart.java b/core/src/main/java/org/opensearch/sql/ast/tree/Chart.java index 5d4a036bc70..02e0878e12d 100644 --- a/core/src/main/java/org/opensearch/sql/ast/tree/Chart.java +++ b/core/src/main/java/org/opensearch/sql/ast/tree/Chart.java @@ -12,7 +12,9 @@ import lombok.Getter; import lombok.ToString; import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.dsl.AstDSL; import org.opensearch.sql.ast.expression.Argument; +import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.expression.UnresolvedExpression; /** AST node represent chart command. */ @@ -22,6 +24,13 @@ @AllArgsConstructor @lombok.Builder(toBuilder = true) public class Chart extends UnresolvedPlan { + public static final Literal DEFAULT_USE_OTHER = Literal.TRUE; + public static final Literal DEFAULT_OTHER_STR = AstDSL.stringLiteral("OTHER"); + public static final Literal DEFAULT_LIMIT = AstDSL.intLiteral(10); + public static final Literal DEFAULT_USE_NULL = Literal.TRUE; + public static final Literal DEFAULT_NULL_STR = AstDSL.stringLiteral("NULL"); + public static final Literal DEFAULT_TOP = Literal.TRUE; + private UnresolvedPlan child; private UnresolvedExpression rowSplit; private UnresolvedExpression columnSplit; diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index a66ba09d446..aa3f7dcea9f 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -2055,23 +2055,25 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { // If row or column split does not present or limit equals 0, this is the same as `stats agg // [group by col]` - Integer limit = - Optional.ofNullable(argMap.get("limit")).map(l -> (Integer) l.getValue()).orElse(10); - Boolean top = - Optional.ofNullable(argMap.get("top")).map(t -> (Boolean) t.getValue()).orElse(true); - Boolean useOther = - Optional.ofNullable(argMap.get("useother")).map(u -> (Boolean) u.getValue()).orElse(true); - String otherStr = - Optional.ofNullable(argMap.get("otherstr")).map(o -> (String) o.getValue()).orElse("OTHER"); + Integer limit = (Integer) argMap.getOrDefault("limit", Chart.DEFAULT_LIMIT).getValue(); if (node.getRowSplit() == null || node.getColumnSplit() == null || Objects.equals(limit, 0)) { return aggregated.getLeft(); } - List projected = aggregated.getRight(); + + Boolean top = (Boolean) argMap.getOrDefault("top", Chart.DEFAULT_TOP).getValue(); + Boolean useOther = + (Boolean) argMap.getOrDefault("useother", Chart.DEFAULT_USE_OTHER).getValue(); + Boolean useNull = (Boolean) argMap.getOrDefault("usenull", Chart.DEFAULT_USE_NULL).getValue(); + String otherStr = (String) argMap.getOrDefault("otherstr", Chart.DEFAULT_OTHER_STR).getValue(); + String nullStr = (String) argMap.getOrDefault("nullstr", Chart.DEFAULT_NULL_STR).getValue(); + String columSplitName = aggregated.getLeft().getRowType().getFieldNames().getLast(); RelBuilder relBuilder = context.relBuilder; // 0: agg; 2: column-split relBuilder.project(relBuilder.field(0), relBuilder.field(2)); - relBuilder.filter(relBuilder.isNotNull(relBuilder.field(1))); + if (!useNull) { + relBuilder.filter(relBuilder.isNotNull(relBuilder.field(1))); + } // 1: column split; 0: agg relBuilder.aggregate( relBuilder.groupKey(relBuilder.field(1)), @@ -2080,11 +2082,13 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { if (top) { grandTotal = relBuilder.desc(grandTotal); } + // Always set it to null last so that it does not interfere with top / bottom calculation + grandTotal = relBuilder.nullsLast(grandTotal); RexNode rowNum = PlanUtils.makeOver( context, BuiltinFunctionName.ROW_NUMBER, - relBuilder.literal(1), + relBuilder.literal(1), // dummy expression for row number calculation List.of(), List.of(), List.of(grandTotal), @@ -2097,26 +2101,29 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { // on column-split = group key relBuilder.join( - JoinRelType.INNER, relBuilder.equals(relBuilder.field(2, 0, 2), relBuilder.field(2, 1, 0))); + JoinRelType.LEFT, relBuilder.equals(relBuilder.field(2, 0, 2), relBuilder.field(2, 1, 0))); - RexNode condition = + RexNode colSplitPostJoin = relBuilder.field(2); + RexNode lteCondition = relBuilder.call( SqlStdOperatorTable.LESS_THAN_OR_EQUAL, relBuilder.field("__row_number__"), relBuilder.literal(limit)); + RexNode nullCondition = relBuilder.isNull(colSplitPostJoin); RexNode columnSplitExpr; - if (useOther) { - columnSplitExpr = - relBuilder.call( - SqlStdOperatorTable.CASE, - condition, - relBuilder.field(2), - relBuilder.literal(otherStr)); - } else { - relBuilder.filter(condition); - columnSplitExpr = relBuilder.field(2); + if (!useOther) { + relBuilder.filter(lteCondition); } + columnSplitExpr = + relBuilder.call( + SqlStdOperatorTable.CASE, + nullCondition, + relBuilder.literal(nullStr), + lteCondition, + relBuilder.field(2), + relBuilder.literal(otherStr)); + relBuilder.project( relBuilder.field(0), relBuilder.field(1), diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 85b2d267454..d92725aa631 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -135,6 +135,8 @@ SHOWCOUNT: 'SHOWCOUNT'; LIMIT: 'LIMIT'; USEOTHER: 'USEOTHER'; OTHERSTR: 'OTHERSTR'; +USENULL: 'USENULL'; +NULLSTR: 'NULLSTR'; INPUT: 'INPUT'; OUTPUT: 'OUTPUT'; PATH: 'PATH'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 272243651c2..1623e02376b 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -268,6 +268,8 @@ chartOptions : LIMIT EQUAL (TOP | BOTTOM)? integerLiteral | USEOTHER EQUAL booleanLiteral | OTHERSTR EQUAL stringLiteral + | USENULL EQUAL booleanLiteral + | NULLSTR EQUAL stringLiteral ; rowSplit diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java index a2be9b3e0d9..84b9689bd84 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java @@ -211,6 +211,10 @@ public static List getArgumentList(ChartCommandContext ctx) { arguments.add(new Argument("useother", getArgumentValue(optionCtx.booleanLiteral()))); } else if (optionCtx.OTHERSTR() != null) { arguments.add(new Argument("otherstr", getArgumentValue(optionCtx.stringLiteral()))); + } else if (optionCtx.USENULL() != null) { + arguments.add(new Argument("usenull", getArgumentValue(optionCtx.booleanLiteral()))); + } else if (optionCtx.NULLSTR() != null) { + arguments.add(new Argument("nullstr", getArgumentValue(optionCtx.stringLiteral()))); } } return arguments; From 70d4722279d940b481888fb69571fb32a73c1145 Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Mon, 20 Oct 2025 17:37:11 +0800 Subject: [PATCH 04/35] Append a final aggregation to merge OTHER categories Signed-off-by: Yuanchun Shen --- .../sql/calcite/CalciteRelNodeVisitor.java | 59 +++++++++++-------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index aa3f7dcea9f..06b4b1be22d 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -1086,11 +1086,6 @@ private Pair, List> resolveAttributesForAggregation( @Override public RelNode visitAggregation(Aggregation node, CalcitePlanContext context) { - return visitAggregationAndReturnProjection(node, context).getLeft(); - } - - private Pair> visitAggregationAndReturnProjection( - Aggregation node, CalcitePlanContext context) { visitChildren(node, context); List aggExprList = node.getAggExprList(); @@ -1175,7 +1170,7 @@ private Pair> visitAggregationAndReturnProjection( reordered.addAll(aliasedGroupByList); context.relBuilder.project(reordered); - return Pair.of(context.relBuilder.peek(), reordered); + return context.relBuilder.peek(); } private Optional getTimeSpanField(UnresolvedExpression expr) { @@ -2048,32 +2043,33 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { Stream.of(node.getRowSplit(), node.getColumnSplit()).filter(Objects::nonNull).toList()); span = null; } + Boolean useNull = (Boolean) argMap.getOrDefault("usenull", Chart.DEFAULT_USE_NULL).getValue(); Aggregation aggregation = - new Aggregation(node.getAggregationFunctions(), List.of(), groupExprList, span, List.of()); - Pair> aggregated = - visitAggregationAndReturnProjection(aggregation, context); + new Aggregation( + node.getAggregationFunctions(), + List.of(), + groupExprList, + span, + List.of(new Argument(Argument.BUCKET_NULLABLE, AstDSL.booleanLiteral(useNull)))); + RelNode aggregated = visitAggregation(aggregation, context); + // If row or column split does not present or limit equals 0, this is the same as `stats agg // [group by col]` - Integer limit = (Integer) argMap.getOrDefault("limit", Chart.DEFAULT_LIMIT).getValue(); if (node.getRowSplit() == null || node.getColumnSplit() == null || Objects.equals(limit, 0)) { - return aggregated.getLeft(); + return aggregated; } Boolean top = (Boolean) argMap.getOrDefault("top", Chart.DEFAULT_TOP).getValue(); Boolean useOther = (Boolean) argMap.getOrDefault("useother", Chart.DEFAULT_USE_OTHER).getValue(); - Boolean useNull = (Boolean) argMap.getOrDefault("usenull", Chart.DEFAULT_USE_NULL).getValue(); String otherStr = (String) argMap.getOrDefault("otherstr", Chart.DEFAULT_OTHER_STR).getValue(); String nullStr = (String) argMap.getOrDefault("nullstr", Chart.DEFAULT_NULL_STR).getValue(); - String columSplitName = aggregated.getLeft().getRowType().getFieldNames().getLast(); + String columSplitName = aggregated.getRowType().getFieldNames().getLast(); RelBuilder relBuilder = context.relBuilder; // 0: agg; 2: column-split relBuilder.project(relBuilder.field(0), relBuilder.field(2)); - if (!useNull) { - relBuilder.filter(relBuilder.isNotNull(relBuilder.field(1))); - } // 1: column split; 0: agg relBuilder.aggregate( relBuilder.groupKey(relBuilder.field(1)), @@ -2096,7 +2092,7 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { relBuilder.projectPlus(relBuilder.alias(rowNum, "__row_number__")); RelNode ranked = relBuilder.build(); - relBuilder.push(aggregated.getLeft()); + relBuilder.push(aggregated); relBuilder.push(ranked); // on column-split = group key @@ -2115,19 +2111,32 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { relBuilder.filter(lteCondition); } - columnSplitExpr = - relBuilder.call( - SqlStdOperatorTable.CASE, - nullCondition, - relBuilder.literal(nullStr), - lteCondition, - relBuilder.field(2), - relBuilder.literal(otherStr)); + if (useNull) { + columnSplitExpr = + relBuilder.call( + SqlStdOperatorTable.CASE, + nullCondition, + relBuilder.literal(nullStr), + lteCondition, + relBuilder.field(2), + relBuilder.literal(otherStr)); + } else { + columnSplitExpr = + relBuilder.call( + SqlStdOperatorTable.CASE, + lteCondition, + relBuilder.field(2), + relBuilder.literal(otherStr)); + } + String aggFieldName = relBuilder.peek().getRowType().getFieldNames().getFirst(); relBuilder.project( relBuilder.field(0), relBuilder.field(1), relBuilder.alias(columnSplitExpr, columSplitName)); + relBuilder.aggregate( + relBuilder.groupKey(relBuilder.field(1), relBuilder.field(2)), + relBuilder.sum(relBuilder.field(0)).as(aggFieldName)); return relBuilder.peek(); } From 9253e67dabe7dc58f0dccdd27052441dd9fb95dc Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Fri, 17 Oct 2025 15:30:51 +0800 Subject: [PATCH 05/35] Handle common agg functions for OTHER category for timechart Signed-off-by: Yuanchun Shen # Conflicts: # core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java # integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java --- .../sql/calcite/CalciteRelNodeVisitor.java | 107 ++++++++++++---- .../sql/calcite/remote/CalciteExplainIT.java | 5 +- .../calcite/explain_timechart.yaml | 38 +++--- .../explain_timechart_no_pushdown.yaml | 37 ------ .../explain_timechart.yaml | 39 ++++++ .../rest-api-spec/test/issues/4582.yml | 120 ++++++++++++++++++ .../ppl/calcite/CalcitePPLTimechartTest.java | 8 +- 7 files changed, 266 insertions(+), 88 deletions(-) delete mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_no_pushdown.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart.yaml create mode 100644 integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4582.yml diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 06b4b1be22d..ad78ac0034c 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -1983,7 +1983,7 @@ public RelNode visitFlatten(Flatten node, CalcitePlanContext context) { } /** Helper method to get the function name for proper column naming */ - private String getValueFunctionName(UnresolvedExpression aggregateFunction) { + private String getAggFieldAlias(UnresolvedExpression aggregateFunction) { if (aggregateFunction instanceof Alias) { return ((Alias) aggregateFunction).getName(); } @@ -2153,7 +2153,7 @@ public RelNode visitTimechart( // Handle no by field case if (node.getByField() == null) { - String valueFunctionName = getValueFunctionName(node.getAggregateFunction()); + String aggFieldAlias = getAggFieldAlias(node.getAggregateFunction()); // Create group expression list with just the timestamp span but use a different alias // to avoid @timestamp naming conflict @@ -2161,7 +2161,7 @@ public RelNode visitTimechart( simpleGroupExprList.add(new Alias("timestamp", spanExpr)); // Create agg expression list with the aggregate function List simpleAggExprList = - List.of(new Alias(valueFunctionName, node.getAggregateFunction())); + List.of(new Alias(aggFieldAlias, node.getAggregateFunction())); // Create an Aggregation object Aggregation aggregation = new Aggregation( @@ -2176,9 +2176,9 @@ public RelNode visitTimechart( context.relBuilder.push(result); // Reorder fields: timestamp first, then count context.relBuilder.project( - context.relBuilder.field("timestamp"), context.relBuilder.field(valueFunctionName)); + context.relBuilder.field("timestamp"), context.relBuilder.field(aggFieldAlias)); // Rename timestamp to @timestamp - context.relBuilder.rename(List.of("@timestamp", valueFunctionName)); + context.relBuilder.rename(List.of("@timestamp", aggFieldAlias)); context.relBuilder.sort(context.relBuilder.field(0)); return context.relBuilder.peek(); @@ -2187,7 +2187,7 @@ public RelNode visitTimechart( // Extract parameters for byField case UnresolvedExpression byField = node.getByField(); String byFieldName = ((Field) byField).getField().toString(); - String valueFunctionName = getValueFunctionName(node.getAggregateFunction()); + String aggFieldAlias = getAggFieldAlias(node.getAggregateFunction()); int limit = Optional.ofNullable(node.getLimit()).orElse(10); boolean useOther = Optional.ofNullable(node.getUseOther()).orElse(true); @@ -2214,11 +2214,11 @@ public RelNode visitTimechart( // Handle no limit case - just sort and return with proper field aliases if (limit == 0) { - // Add final projection with proper aliases: [@timestamp, byField, valueFunctionName] + // Add final projection with proper aliases: [@timestamp, byField, aggFieldAlias] context.relBuilder.project( context.relBuilder.alias(context.relBuilder.field(0), "@timestamp"), context.relBuilder.alias(context.relBuilder.field(1), byFieldName), - context.relBuilder.alias(context.relBuilder.field(2), valueFunctionName)); + context.relBuilder.alias(context.relBuilder.field(2), aggFieldAlias)); context.relBuilder.sort(context.relBuilder.field(0), context.relBuilder.field(1)); return context.relBuilder.peek(); } @@ -2228,32 +2228,61 @@ public RelNode visitTimechart( // Step 2: Find top N categories using window function approach (more efficient than separate // aggregation) - RelNode topCategories = buildTopCategoriesQuery(completeResults, limit, context); + String aggFunctionName = getAggFunctionName(node.getAggregateFunction()); + Optional aggFuncNameOptional = BuiltinFunctionName.of(aggFunctionName); + if (aggFuncNameOptional.isEmpty()) { + throw new IllegalArgumentException( + StringUtils.format("Unrecognized aggregation function: %s", aggFunctionName)); + } + BuiltinFunctionName aggFunction = aggFuncNameOptional.get(); + RelNode topCategories = buildTopCategoriesQuery(completeResults, limit, aggFunction, context); // Step 3: Apply OTHER logic with single pass return buildFinalResultWithOther( - completeResults, topCategories, byFieldName, valueFunctionName, useOther, limit, context); + completeResults, + topCategories, + byFieldName, + aggFunction, + aggFieldAlias, + useOther, + limit, + context); } catch (Exception e) { throw new RuntimeException("Error in visitTimechart: " + e.getMessage(), e); } } + private String getAggFunctionName(UnresolvedExpression aggregateFunction) { + if (aggregateFunction instanceof Alias alias) { + return getAggFunctionName(alias.getDelegated()); + } + return ((AggregateFunction) aggregateFunction).getFuncName(); + } + /** Build top categories query - simpler approach that works better with OTHER handling */ private RelNode buildTopCategoriesQuery( - RelNode completeResults, int limit, CalcitePlanContext context) { + RelNode completeResults, + int limit, + BuiltinFunctionName aggFunction, + CalcitePlanContext context) { context.relBuilder.push(completeResults); // Filter out null values when determining top categories - null should not count towards limit context.relBuilder.filter(context.relBuilder.isNotNull(context.relBuilder.field(1))); // Get totals for non-null categories - field positions: 0=@timestamp, 1=byField, 2=value + RexInputRef valueField = context.relBuilder.field(2); + AggCall call = buildAggCall(context.relBuilder, aggFunction, valueField); + context.relBuilder.aggregate( - context.relBuilder.groupKey(context.relBuilder.field(1)), - context.relBuilder.sum(context.relBuilder.field(2)).as("grand_total")); + context.relBuilder.groupKey(context.relBuilder.field(1)), call.as("grand_total")); // Apply sorting and limit to non-null categories only - context.relBuilder.sort(context.relBuilder.desc(context.relBuilder.field("grand_total"))); + RexNode sortField = context.relBuilder.field("grand_total"); + sortField = + aggFunction == BuiltinFunctionName.MIN ? sortField : context.relBuilder.desc(sortField); + context.relBuilder.sort(sortField); if (limit > 0) { context.relBuilder.limit(0, limit); } @@ -2265,18 +2294,25 @@ private RelNode buildFinalResultWithOther( RelNode completeResults, RelNode topCategories, String byFieldName, - String valueFunctionName, + BuiltinFunctionName aggFunction, + String aggFieldAlias, boolean useOther, int limit, CalcitePlanContext context) { // Use zero-filling for count aggregations, standard result for others - if (valueFunctionName.equals("count")) { + if (aggFieldAlias.equals("count")) { return buildZeroFilledResult( - completeResults, topCategories, byFieldName, valueFunctionName, useOther, limit, context); + completeResults, topCategories, byFieldName, aggFieldAlias, useOther, limit, context); } else { return buildStandardResult( - completeResults, topCategories, byFieldName, valueFunctionName, useOther, context); + completeResults, + topCategories, + byFieldName, + aggFunction, + aggFieldAlias, + useOther, + context); } } @@ -2285,7 +2321,8 @@ private RelNode buildStandardResult( RelNode completeResults, RelNode topCategories, String byFieldName, - String valueFunctionName, + BuiltinFunctionName aggFunctionName, + String aggFieldAlias, boolean useOther, CalcitePlanContext context) { @@ -2308,11 +2345,13 @@ private RelNode buildStandardResult( context.relBuilder.project( context.relBuilder.alias(context.relBuilder.field(0), "@timestamp"), context.relBuilder.alias(categoryExpr, byFieldName), - context.relBuilder.alias(context.relBuilder.field(2), valueFunctionName)); + context.relBuilder.alias(context.relBuilder.field(2), aggFieldAlias)); + RexInputRef valueField = context.relBuilder.field(2); + AggCall aggCall = buildAggCall(context.relBuilder, aggFunctionName, valueField); context.relBuilder.aggregate( context.relBuilder.groupKey(context.relBuilder.field(0), context.relBuilder.field(1)), - context.relBuilder.sum(context.relBuilder.field(2)).as(valueFunctionName)); + aggCall.as(aggFieldAlias)); applyFiltersAndSort(useOther, context); return context.relBuilder.peek(); @@ -2347,7 +2386,7 @@ private RelNode buildZeroFilledResult( RelNode completeResults, RelNode topCategories, String byFieldName, - String valueFunctionName, + String aggFieldAlias, boolean useOther, int limit, CalcitePlanContext context) { @@ -2386,7 +2425,7 @@ private RelNode buildZeroFilledResult( context.relBuilder.cast(context.relBuilder.field(0), SqlTypeName.TIMESTAMP), "@timestamp"), context.relBuilder.alias(context.relBuilder.field(1), byFieldName), - context.relBuilder.alias(context.relBuilder.literal(0), valueFunctionName)); + context.relBuilder.alias(context.relBuilder.literal(0), aggFieldAlias)); RelNode zeroFilledCombinations = context.relBuilder.build(); // Get actual results with OTHER logic applied @@ -2408,7 +2447,7 @@ private RelNode buildZeroFilledResult( context.relBuilder.cast(context.relBuilder.field(0), SqlTypeName.TIMESTAMP), "@timestamp"), context.relBuilder.alias(actualCategoryExpr, byFieldName), - context.relBuilder.alias(context.relBuilder.field(2), valueFunctionName)); + context.relBuilder.alias(context.relBuilder.field(2), aggFieldAlias)); context.relBuilder.aggregate( context.relBuilder.groupKey(context.relBuilder.field(0), context.relBuilder.field(1)), @@ -2423,12 +2462,30 @@ private RelNode buildZeroFilledResult( // Aggregate to combine actual and zero-filled data context.relBuilder.aggregate( context.relBuilder.groupKey(context.relBuilder.field(0), context.relBuilder.field(1)), - context.relBuilder.sum(context.relBuilder.field(2)).as(valueFunctionName)); + context.relBuilder.sum(context.relBuilder.field(2)).as(aggFieldAlias)); applyFiltersAndSort(useOther, context); return context.relBuilder.peek(); } + /** + * Aggregate a field based on a given built-in aggregation function name. + * + *

It is intended for secondary aggregations in timechart and chart commands. Using it + * elsewhere may lead to unintended results. It handles explicitly only MIN, MAX, AVG, COUNT, + * DISTINCT_COUNT, EARLIEST, and LATEST. It sums the results for the rest aggregation types, + * assuming them to be accumulative. + */ + private AggCall buildAggCall( + RelBuilder relBuilder, BuiltinFunctionName aggFunction, RexNode node) { + return switch (aggFunction) { + case MIN, EARLIEST -> relBuilder.min(node); + case MAX, LATEST -> relBuilder.max(node); + case AVG -> relBuilder.avg(node); + default -> relBuilder.sum(node); + }; + } + @Override public RelNode visitTrendline(Trendline node, CalcitePlanContext context) { visitChildren(node, context); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index c2bb21fff5e..6528eb4d2b9 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -422,10 +422,7 @@ public void testExplainWithReverse() throws IOException { @Test public void testExplainWithTimechartAvg() throws IOException { var result = explainQueryYaml("source=events | timechart span=1m avg(cpu_usage) by host"); - String expected = - !isPushdownDisabled() - ? loadFromFile("expectedOutput/calcite/explain_timechart.yaml") - : loadFromFile("expectedOutput/calcite/explain_timechart_no_pushdown.yaml"); + String expected = loadExpectedPlan("explain_timechart.yaml"); assertYamlEqualsIgnoreId(expected, result); } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.yaml index f212b4c8bfd..a315860aac9 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) - LogicalAggregate(group=[{0, 1}], avg(cpu_usage)=[SUM($2)]) + LogicalAggregate(group=[{0, 1}], avg(cpu_usage)=[AVG($2)]) LogicalProject(@timestamp=[$0], host=[CASE(IS NOT NULL($3), $1, CASE(IS NULL($1), null:NULL, 'OTHER'))], avg(cpu_usage)=[$2]) LogicalJoin(condition=[=($1, $3)], joinType=[left]) LogicalProject(@timestamp=[$1], host=[$0], $f2=[$2]) @@ -10,7 +10,7 @@ calcite: LogicalProject(host=[$4], cpu_usage=[$7], $f3=[SPAN($1, 1, 'm')]) CalciteLogicalIndexScan(table=[[OpenSearch, events]]) LogicalSort(sort0=[$1], dir0=[DESC], fetch=[10]) - LogicalAggregate(group=[{1}], grand_total=[SUM($2)]) + LogicalAggregate(group=[{1}], grand_total=[AVG($2)]) LogicalFilter(condition=[IS NOT NULL($1)]) LogicalProject(@timestamp=[$1], host=[$0], $f2=[$2]) LogicalAggregate(group=[{0, 2}], agg#0=[AVG($1)]) @@ -19,19 +19,21 @@ calcite: physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) - EnumerableAggregate(group=[{0, 1}], avg(cpu_usage)=[SUM($2)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NOT NULL($t3)], expr#6=[IS NULL($t1)], expr#7=[null:NULL], expr#8=['OTHER'], expr#9=[CASE($t6, $t7, $t8)], expr#10=[CASE($t5, $t1, $t9)], @timestamp=[$t0], host=[$t10], avg(cpu_usage)=[$t2]) - EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) - EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], @timestamp=[$t1], host=[$t0], $f2=[$t8]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=['m'], expr#5=[SPAN($t2, $t3, $t4)], proj#0..1=[{exprs}], $f2=[$t5]) - CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[host, cpu_usage, @timestamp]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["host","cpu_usage","@timestamp"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableLimit(fetch=[10]) - EnumerableSort(sort0=[$1], dir0=[DESC]) - EnumerableAggregate(group=[{0}], grand_total=[SUM($1)]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], host=[$t0], $f2=[$t8]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=['m'], expr#5=[SPAN($t2, $t3, $t4)], proj#0..1=[{exprs}], $f2=[$t5]) - CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[host, cpu_usage, @timestamp], FILTER->IS NOT NULL($0)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"exists":{"field":"host","boost":1.0}},"_source":{"includes":["host","cpu_usage","@timestamp"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(cpu_usage)=[$t8]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NOT NULL($t3)], expr#6=[IS NULL($t1)], expr#7=[null:NULL], expr#8=['OTHER'], expr#9=[CASE($t6, $t7, $t8)], expr#10=[CASE($t5, $t1, $t9)], @timestamp=[$t0], host=[$t10], avg(cpu_usage)=[$t2]) + EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) + EnumerableSort(sort0=[$1], dir0=[ASC]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], @timestamp=[$t1], host=[$t0], $f2=[$t8]) + EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=['m'], expr#5=[SPAN($t2, $t3, $t4)], proj#0..1=[{exprs}], $f2=[$t5]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[host, cpu_usage, @timestamp]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["host","cpu_usage","@timestamp"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableLimit(fetch=[10]) + EnumerableSort(sort0=[$1], dir0=[DESC]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:DOUBLE], expr#6=[CASE($t4, $t5, $t1)], expr#7=[/($t6, $t2)], host=[$t0], grand_total=[$t7]) + EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], host=[$t0], $f2=[$t8]) + EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=['m'], expr#5=[SPAN($t2, $t3, $t4)], proj#0..1=[{exprs}], $f2=[$t5]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[host, cpu_usage, @timestamp], FILTER->IS NOT NULL($0)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"exists":{"field":"host","boost":1.0}},"_source":{"includes":["host","cpu_usage","@timestamp"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_no_pushdown.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_no_pushdown.yaml deleted file mode 100644 index ae966d7eea7..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_no_pushdown.yaml +++ /dev/null @@ -1,37 +0,0 @@ -calcite: - logical: | - LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) - LogicalAggregate(group=[{0, 1}], avg(cpu_usage)=[SUM($2)]) - LogicalProject(@timestamp=[$0], host=[CASE(IS NOT NULL($3), $1, CASE(IS NULL($1), null:NULL, 'OTHER'))], avg(cpu_usage)=[$2]) - LogicalJoin(condition=[=($1, $3)], joinType=[left]) - LogicalProject(@timestamp=[$1], host=[$0], $f2=[$2]) - LogicalAggregate(group=[{0, 2}], agg#0=[AVG($1)]) - LogicalProject(host=[$4], cpu_usage=[$7], $f3=[SPAN($1, 1, 'm')]) - CalciteLogicalIndexScan(table=[[OpenSearch, events]]) - LogicalSort(sort0=[$1], dir0=[DESC], fetch=[10]) - LogicalAggregate(group=[{1}], grand_total=[SUM($2)]) - LogicalFilter(condition=[IS NOT NULL($1)]) - LogicalProject(@timestamp=[$1], host=[$0], $f2=[$2]) - LogicalAggregate(group=[{0, 2}], agg#0=[AVG($1)]) - LogicalProject(host=[$4], cpu_usage=[$7], $f3=[SPAN($1, 1, 'm')]) - CalciteLogicalIndexScan(table=[[OpenSearch, events]]) - physical: | - EnumerableLimit(fetch=[10000]) - EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) - EnumerableAggregate(group=[{0, 1}], avg(cpu_usage)=[SUM($2)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NOT NULL($t3)], expr#6=[IS NULL($t1)], expr#7=[null:NULL], expr#8=['OTHER'], expr#9=[CASE($t6, $t7, $t8)], expr#10=[CASE($t5, $t1, $t9)], @timestamp=[$t0], host=[$t10], avg(cpu_usage)=[$t2]) - EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) - EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], @timestamp=[$t1], host=[$t0], $f2=[$t8]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['m'], expr#18=[SPAN($t1, $t16, $t17)], host=[$t4], cpu_usage=[$t7], $f3=[$t18]) - CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) - EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableLimit(fetch=[10]) - EnumerableSort(sort0=[$1], dir0=[DESC]) - EnumerableAggregate(group=[{0}], grand_total=[SUM($2)]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], expr#9=[IS NOT NULL($t0)], proj#0..1=[{exprs}], $f2=[$t8], $condition=[$t9]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['m'], expr#18=[SPAN($t1, $t16, $t17)], host=[$t4], cpu_usage=[$t7], $f3=[$t18]) - CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart.yaml new file mode 100644 index 00000000000..5aa55ca656b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart.yaml @@ -0,0 +1,39 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + LogicalAggregate(group=[{0, 1}], avg(cpu_usage)=[AVG($2)]) + LogicalProject(@timestamp=[$0], host=[CASE(IS NOT NULL($3), $1, CASE(IS NULL($1), null:NULL, 'OTHER'))], avg(cpu_usage)=[$2]) + LogicalJoin(condition=[=($1, $3)], joinType=[left]) + LogicalProject(@timestamp=[$1], host=[$0], $f2=[$2]) + LogicalAggregate(group=[{0, 2}], agg#0=[AVG($1)]) + LogicalProject(host=[$4], cpu_usage=[$7], $f3=[SPAN($1, 1, 'm')]) + CalciteLogicalIndexScan(table=[[OpenSearch, events]]) + LogicalSort(sort0=[$1], dir0=[DESC], fetch=[10]) + LogicalAggregate(group=[{1}], grand_total=[AVG($2)]) + LogicalFilter(condition=[IS NOT NULL($1)]) + LogicalProject(@timestamp=[$1], host=[$0], $f2=[$2]) + LogicalAggregate(group=[{0, 2}], agg#0=[AVG($1)]) + LogicalProject(host=[$4], cpu_usage=[$7], $f3=[SPAN($1, 1, 'm')]) + CalciteLogicalIndexScan(table=[[OpenSearch, events]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(cpu_usage)=[$t8]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NOT NULL($t3)], expr#6=[IS NULL($t1)], expr#7=[null:NULL], expr#8=['OTHER'], expr#9=[CASE($t6, $t7, $t8)], expr#10=[CASE($t5, $t1, $t9)], @timestamp=[$t0], host=[$t10], avg(cpu_usage)=[$t2]) + EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) + EnumerableSort(sort0=[$1], dir0=[ASC]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], @timestamp=[$t1], host=[$t0], $f2=[$t8]) + EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['m'], expr#18=[SPAN($t1, $t16, $t17)], host=[$t4], cpu_usage=[$t7], $f3=[$t18]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableLimit(fetch=[10]) + EnumerableSort(sort0=[$1], dir0=[DESC]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:DOUBLE], expr#6=[CASE($t4, $t5, $t1)], expr#7=[/($t6, $t2)], host=[$t0], grand_total=[$t7]) + EnumerableAggregate(group=[{0}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], expr#9=[IS NOT NULL($t0)], proj#0..1=[{exprs}], $f2=[$t8], $condition=[$t9]) + EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['m'], expr#18=[SPAN($t1, $t16, $t17)], host=[$t4], cpu_usage=[$t7], $f3=[$t18]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4582.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4582.yml new file mode 100644 index 00000000000..d1ecb079f3a --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4582.yml @@ -0,0 +1,120 @@ +setup: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : true + - do: + indices.create: + index: test_timechart_4582 + body: + mappings: + properties: + "@timestamp": + type: date_nanos + severityNumber: + type: long + severityText: + type: keyword + body: + type: text + - do: + bulk: + index: test_timechart_4582 + refresh: true + body: + - '{"index": {}}' + - '{"@timestamp": "2024-01-15T10:30:04.567890123Z", "severityNumber": 9, "severityText": "INFO", "body": "Info message"}' + - '{"index": {}}' + - '{"@timestamp": "2024-01-15T10:30:05.567890123Z", "severityNumber": 13, "severityText": "WARN", "body": "Warning message"}' + - '{"index": {}}' + - '{"@timestamp": "2024-01-15T10:30:06.567890123Z", "severityNumber": 17, "severityText": "ERROR", "body": "Error message"}' + - '{"index": {}}' + - '{"@timestamp": "2024-01-15T10:30:07.567890123Z", "severityNumber": 21, "severityText": "FATAL", "body": "Fatal message"}' + - '{"index": {}}' + - '{"@timestamp": "2024-01-15T10:30:08.567890123Z", "severityNumber": 24, "severityText": "FATAL4", "body": "Fatal4 message"}' + - '{"index": {}}' + - '{"@timestamp": "2024-01-15T10:30:09.567890123Z", "severityNumber": 23, "severityText": "DEBUG", "body": "Debug message"}' + - '{"index": {}}' + - '{"@timestamp": "2024-01-15T10:30:10.567890123Z", "severityNumber": 20, "severityText": "TRACE", "body": "Trace message"}' + - '{"index": {}}' + - '{"@timestamp": "2024-01-15T10:30:11.567890123Z", "severityNumber": 22, "severityText": "CUSTOM", "body": "Custom message"}' + +--- +teardown: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : false + +--- +"timechart max aggregation with limit should not sum OTHER values": + - skip: + features: + - headers + - allowed_warnings + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_timechart_4582 | timechart limit=1 span=10seconds max(severityNumber) by severityText + + - match: { total: 3 } + - match: { "schema": [{"name": "@timestamp", "type": "timestamp"}, {"name": "severityText", "type": "string"}, {"name": "max(severityNumber)", "type": "bigint"}] } + - match: { "datarows": [["2024-01-15 10:30:00", "FATAL4", 24], ["2024-01-15 10:30:00", "OTHER", 23], ["2024-01-15 10:30:10", "OTHER",22]] } + +--- +"timechart min aggregation with limit should not sum OTHER values": + - skip: + features: + - headers + - allowed_warnings + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_timechart_4582 | timechart limit=2 span=1d min(severityNumber) by severityText + + - match: { total: 3 } + - match: { "schema": [{"name": "@timestamp", "type": "timestamp"}, {"name": "severityText", "type": "string"}, {"name": "min(severityNumber)", "type": "bigint"}] } + - match: { "datarows": [["2024-01-15 00:00:00", "INFO", 9], ["2024-01-15 00:00:00", "OTHER", 17], ["2024-01-15 00:00:00", "WARN", 13]] } + +--- +"timechart earliest aggregation with limit should not sum OTHER values": + - skip: + features: + - headers + - allowed_warnings + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_timechart_4582 | timechart limit=2 span=30seconds earliest(@timestamp) by severityText + + - match: { total: 3 } + - match: { "schema": [{"name": "@timestamp", "type": "timestamp"}, {"name": "severityText", "type": "string"}, {"name": "earliest(@timestamp)", "type": "timestamp"}] } + - match: { "datarows": [ + ["2024-01-15 10:30:00", "CUSTOM", "2024-01-15 10:30:11.567890123"], + ["2024-01-15 10:30:00", "OTHER", "2024-01-15 10:30:04.567890123"], + ["2024-01-15 10:30:00", "TRACE", "2024-01-15 10:30:10.567890123"]] } + +--- +"timechart count aggregation with limit should sum OTHER values": + - skip: + features: + - headers + - allowed_warnings + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_timechart_4582 | timechart limit=3 span=1min count() by severityText + + - match: { total: 4 } + - match: { "schema": [{"name": "@timestamp", "type": "timestamp"}, {"name": "severityText", "type": "string"}, {"name": "count", "type": "bigint"}] } + - match: { "datarows": [["2024-01-15 10:30:00", "CUSTOM", 1], ["2024-01-15 10:30:00", "DEBUG", 1], ["2024-01-15 10:30:00", "ERROR", 1], ["2024-01-15 10:30:00", "OTHER", 5]] } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimechartTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimechartTest.java index ee6b82f2d85..c3ed1ebceea 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimechartTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimechartTest.java @@ -260,13 +260,13 @@ public void testTimechartWithSpan1m() { RelNode root = getRelNode(ppl); String expectedSparkSql = "SELECT `t1`.`@timestamp`, CASE WHEN `t7`.`region` IS NOT NULL THEN `t1`.`region` ELSE CASE" - + " WHEN `t1`.`region` IS NULL THEN NULL ELSE 'OTHER' END END `region`, SUM(`t1`.`$f2`)" + + " WHEN `t1`.`region` IS NULL THEN NULL ELSE 'OTHER' END END `region`, AVG(`t1`.`$f2`)" + " `avg(cpu_usage)`\n" + "FROM (SELECT `SPAN`(`@timestamp`, 1, 'm') `@timestamp`, `region`, AVG(`cpu_usage`)" + " `$f2`\n" + "FROM `scott`.`events`\n" + "GROUP BY `region`, `SPAN`(`@timestamp`, 1, 'm')) `t1`\n" - + "LEFT JOIN (SELECT `region`, SUM(`$f2`) `grand_total`\n" + + "LEFT JOIN (SELECT `region`, AVG(`$f2`) `grand_total`\n" + "FROM (SELECT `SPAN`(`@timestamp`, 1, 'm') `@timestamp`, `region`, AVG(`cpu_usage`)" + " `$f2`\n" + "FROM `scott`.`events`\n" @@ -297,13 +297,13 @@ public void testTimechartWithLimitAndUseOtherFalse() { RelNode root = getRelNode(ppl); String expectedSparkSql = "SELECT `t1`.`@timestamp`, CASE WHEN `t7`.`host` IS NOT NULL THEN `t1`.`host` ELSE CASE" - + " WHEN `t1`.`host` IS NULL THEN NULL ELSE 'OTHER' END END `host`, SUM(`t1`.`$f2`)" + + " WHEN `t1`.`host` IS NULL THEN NULL ELSE 'OTHER' END END `host`, AVG(`t1`.`$f2`)" + " `avg(cpu_usage)`\n" + "FROM (SELECT `SPAN`(`@timestamp`, 1, 'h') `@timestamp`, `host`, AVG(`cpu_usage`)" + " `$f2`\n" + "FROM `scott`.`events`\n" + "GROUP BY `host`, `SPAN`(`@timestamp`, 1, 'h')) `t1`\n" - + "LEFT JOIN (SELECT `host`, SUM(`$f2`) `grand_total`\n" + + "LEFT JOIN (SELECT `host`, AVG(`$f2`) `grand_total`\n" + "FROM (SELECT `SPAN`(`@timestamp`, 1, 'h') `@timestamp`, `host`, AVG(`cpu_usage`)" + " `$f2`\n" + "FROM `scott`.`events`\n" From 19af8374d3abaddcb44641967f8dc95e8b038e60 Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Fri, 17 Oct 2025 16:52:20 +0800 Subject: [PATCH 06/35] Fix timechart IT Signed-off-by: Yuanchun Shen --- .../remote/CalciteTimechartCommandIT.java | 30 +++++-------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTimechartCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTimechartCommandIT.java index 3b4ca27dab5..4d9352e9e87 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTimechartCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTimechartCommandIT.java @@ -183,27 +183,13 @@ public void testTimechartWithLimit() throws IOException { schema("host", "string"), schema("avg(cpu_usage)", "double")); - // Verify we have rows for web-01, web-02, and OTHER - boolean foundWeb01 = false; - boolean foundWeb02 = false; - boolean foundOther = false; - - for (int i = 0; i < result.getJSONArray("datarows").length(); i++) { - Object[] row = result.getJSONArray("datarows").getJSONArray(i).toList().toArray(); - String label = (String) row[1]; - - if ("web-01".equals(label)) { - foundWeb01 = true; - } else if ("web-02".equals(label)) { - foundWeb02 = true; - } else if ("OTHER".equals(label)) { - foundOther = true; - } - } - - assertTrue("web-01 not found in results", foundWeb01); - assertTrue("web-02 not found in results", foundWeb02); - assertTrue("OTHER category not found in results", foundOther); + verifyDataRows( + result, + rows("2024-07-01 00:00:00", "web-01", 45.2), + rows("2024-07-01 00:01:00", "OTHER", 38.7), + rows("2024-07-01 00:02:00", "web-01", 55.3), + rows("2024-07-01 00:03:00", "db-01", 42.1), + rows("2024-07-01 00:04:00", "OTHER", 41.8)); } @Test @@ -383,7 +369,7 @@ public void testTimechartWithLimitAndUseOther() throws IOException { if ("OTHER".equals(host)) { foundOther = true; - assertEquals(330.4, cpuUsage, 0.1); + assertEquals(41.3, cpuUsage, 0.1); } else if ("web-03".equals(host)) { foundWeb03 = true; assertEquals(55.3, cpuUsage, 0.1); From 6c5df2e71555c4163226814ec83fe3175d01a175 Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Tue, 21 Oct 2025 12:41:30 +0800 Subject: [PATCH 07/35] Sort earliest results with asc order Signed-off-by: Yuanchun Shen --- .../org/opensearch/sql/calcite/CalciteRelNodeVisitor.java | 5 ++++- .../resources/rest-api-spec/test/issues/4582.yml | 6 +++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index ad78ac0034c..787230ec7d4 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -2280,8 +2280,11 @@ private RelNode buildTopCategoriesQuery( // Apply sorting and limit to non-null categories only RexNode sortField = context.relBuilder.field("grand_total"); + // For MIN and EARLIEST, top results should be the minimum ones sortField = - aggFunction == BuiltinFunctionName.MIN ? sortField : context.relBuilder.desc(sortField); + aggFunction == BuiltinFunctionName.MIN || aggFunction == BuiltinFunctionName.EARLIEST + ? sortField + : context.relBuilder.desc(sortField); context.relBuilder.sort(sortField); if (limit > 0) { context.relBuilder.limit(0, limit); diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4582.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4582.yml index d1ecb079f3a..27973484d6c 100644 --- a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4582.yml +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4582.yml @@ -98,9 +98,9 @@ teardown: - match: { total: 3 } - match: { "schema": [{"name": "@timestamp", "type": "timestamp"}, {"name": "severityText", "type": "string"}, {"name": "earliest(@timestamp)", "type": "timestamp"}] } - match: { "datarows": [ - ["2024-01-15 10:30:00", "CUSTOM", "2024-01-15 10:30:11.567890123"], - ["2024-01-15 10:30:00", "OTHER", "2024-01-15 10:30:04.567890123"], - ["2024-01-15 10:30:00", "TRACE", "2024-01-15 10:30:10.567890123"]] } + ["2024-01-15 10:30:00", "INFO", "2024-01-15 10:30:04.567890123"], + ["2024-01-15 10:30:00", "OTHER", "2024-01-15 10:30:06.567890123"], + ["2024-01-15 10:30:00", "WARN", "2024-01-15 10:30:05.567890123"]] } --- "timechart count aggregation with limit should sum OTHER values": From b07c3c42f1bb8fd4351936b5819243ffa641eddb Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Wed, 22 Oct 2025 16:15:52 +0800 Subject: [PATCH 08/35] Support non-string fields as column split Signed-off-by: Yuanchun Shen --- .../sql/calcite/CalciteRelNodeVisitor.java | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 787230ec7d4..3d87fcf5358 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -62,6 +62,7 @@ import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.tools.RelBuilder.AggCall; import org.apache.calcite.util.Holder; @@ -2051,7 +2052,22 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { groupExprList, span, List.of(new Argument(Argument.BUCKET_NULLABLE, AstDSL.booleanLiteral(useNull)))); - RelNode aggregated = visitAggregation(aggregation, context); + visitAggregation(aggregation, context); + + // Convert the column split to string if necessary: column split was supposed to be pivoted to + // column names. This guarantees that its type being compatible with useother and usenull + RelBuilder relBuilder = context.relBuilder; + RexNode colSplit = relBuilder.field(2); + String columSplitName = relBuilder.peek().getRowType().getFieldNames().getLast(); + if (!SqlTypeUtil.isCharacter(colSplit.getType())) { + colSplit = + relBuilder.alias( + context.rexBuilder.makeCast( + UserDefinedFunctionUtils.NULLABLE_STRING, colSplit, true, true), + columSplitName); + } + relBuilder.project(relBuilder.field(0), relBuilder.field(1), colSplit); + RelNode aggregated = relBuilder.peek(); // If row or column split does not present or limit equals 0, this is the same as `stats agg // [group by col]` @@ -2066,8 +2082,6 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { String otherStr = (String) argMap.getOrDefault("otherstr", Chart.DEFAULT_OTHER_STR).getValue(); String nullStr = (String) argMap.getOrDefault("nullstr", Chart.DEFAULT_NULL_STR).getValue(); - String columSplitName = aggregated.getRowType().getFieldNames().getLast(); - RelBuilder relBuilder = context.relBuilder; // 0: agg; 2: column-split relBuilder.project(relBuilder.field(0), relBuilder.field(2)); // 1: column split; 0: agg From 269d4e2c058bad27d9f94cc7c37050ecec41780e Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Thu, 23 Oct 2025 13:13:53 +0800 Subject: [PATCH 09/35] Fix min/earliest order & fix non-accumulative agg for chart Signed-off-by: Yuanchun Shen --- .../sql/calcite/CalciteRelNodeVisitor.java | 62 +++++++++---------- .../calcite/remote/CalciteChartCommandIT.java | 62 +++++++++++++++++++ 2 files changed, 91 insertions(+), 33 deletions(-) create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 3d87fcf5358..7d8323890be 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -2025,37 +2025,35 @@ private String getAggFieldAlias(UnresolvedExpression aggregateFunction) { public RelNode visitChart(Chart node, CalcitePlanContext context) { visitChildren(node, context); ArgumentMap argMap = ArgumentMap.of(node.getArguments()); - List groupExprList = new ArrayList<>(); - UnresolvedExpression span; - if (node.getColumnSplit() instanceof Span && node.getRowSplit() instanceof Span) { - throw new UnsupportedOperationException("It is not supported to have two span splits"); - } else if (node.getRowSplit() instanceof Span) { - if (node.getColumnSplit() != null) { - groupExprList.add(node.getColumnSplit()); - } - span = node.getRowSplit(); - } else if (node.getColumnSplit() instanceof Span) { - if (node.getRowSplit() != null) { - groupExprList.add(node.getRowSplit()); - } - span = node.getColumnSplit(); - } else { - groupExprList.addAll( - Stream.of(node.getRowSplit(), node.getColumnSplit()).filter(Objects::nonNull).toList()); - span = null; - } + List groupExprList = + Stream.of(node.getRowSplit(), node.getColumnSplit()).filter(Objects::nonNull).toList(); Boolean useNull = (Boolean) argMap.getOrDefault("usenull", Chart.DEFAULT_USE_NULL).getValue(); Aggregation aggregation = new Aggregation( node.getAggregationFunctions(), List.of(), groupExprList, - span, + null, List.of(new Argument(Argument.BUCKET_NULLABLE, AstDSL.booleanLiteral(useNull)))); - visitAggregation(aggregation, context); + RelNode aggregated = visitAggregation(aggregation, context); + + // If row or column split does not present or limit equals 0, this is the same as `stats agg + // [group by col]` + Integer limit = (Integer) argMap.getOrDefault("limit", Chart.DEFAULT_LIMIT).getValue(); + if (node.getRowSplit() == null || node.getColumnSplit() == null || Objects.equals(limit, 0)) { + return aggregated; + } + + String aggFunctionName = getAggFunctionName(node.getAggregationFunctions().getFirst()); + Optional aggFuncNameOptional = BuiltinFunctionName.of(aggFunctionName); + if (aggFuncNameOptional.isEmpty()) { + throw new IllegalArgumentException( + StringUtils.format("Unrecognized aggregation function: %s", aggFunctionName)); + } + BuiltinFunctionName aggFunction = aggFuncNameOptional.get(); // Convert the column split to string if necessary: column split was supposed to be pivoted to - // column names. This guarantees that its type being compatible with useother and usenull + // column names. This guarantees that its type compatibility with useother and usenull RelBuilder relBuilder = context.relBuilder; RexNode colSplit = relBuilder.field(2); String columSplitName = relBuilder.peek().getRowType().getFieldNames().getLast(); @@ -2067,14 +2065,7 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { columSplitName); } relBuilder.project(relBuilder.field(0), relBuilder.field(1), colSplit); - RelNode aggregated = relBuilder.peek(); - - // If row or column split does not present or limit equals 0, this is the same as `stats agg - // [group by col]` - Integer limit = (Integer) argMap.getOrDefault("limit", Chart.DEFAULT_LIMIT).getValue(); - if (node.getRowSplit() == null || node.getColumnSplit() == null || Objects.equals(limit, 0)) { - return aggregated; - } + aggregated = relBuilder.peek(); Boolean top = (Boolean) argMap.getOrDefault("top", Chart.DEFAULT_TOP).getValue(); Boolean useOther = @@ -2087,11 +2078,16 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { // 1: column split; 0: agg relBuilder.aggregate( relBuilder.groupKey(relBuilder.field(1)), - relBuilder.sum(relBuilder.field(0)).as("__grand_total__")); // results: group key, agg calls + buildAggCall(context.relBuilder, aggFunction, relBuilder.field(0)) + .as("__grand_total__")); // results: group key, agg calls RexNode grandTotal = relBuilder.field("__grand_total__"); - if (top) { + // Apply sorting: for MIN/EARLIEST, reverse the top/bottom logic + boolean smallestFirst = + aggFunction == BuiltinFunctionName.MIN || aggFunction == BuiltinFunctionName.EARLIEST; + if (top != smallestFirst) { grandTotal = relBuilder.desc(grandTotal); } + // Always set it to null last so that it does not interfere with top / bottom calculation grandTotal = relBuilder.nullsLast(grandTotal); RexNode rowNum = @@ -2150,7 +2146,7 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { relBuilder.alias(columnSplitExpr, columSplitName)); relBuilder.aggregate( relBuilder.groupKey(relBuilder.field(1), relBuilder.field(2)), - relBuilder.sum(relBuilder.field(0)).as(aggFieldName)); + buildAggCall(context.relBuilder, aggFunction, relBuilder.field(0)).as(aggFieldName)); return relBuilder.peek(); } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java new file mode 100644 index 00000000000..b326f4ea073 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java @@ -0,0 +1,62 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +import java.io.IOException; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static org.opensearch.sql.util.MatcherUtils.assertJsonEquals; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; + +public class CalciteChartCommandIT extends PPLIntegTestCase { + @Override + protected void init() throws Exception { + super.init(); + enableCalcite(); + loadIndex(Index.BANK); + loadIndex(Index.BANK_WITH_NULL_VALUES); + loadIndex(Index.OTELLOGS); + } + + @Test + public void testChartWithSingleGroupKey() throws IOException { + JSONObject result1 = executeQuery(String.format("source=%s | chart avg(balance) by gender", TEST_INDEX_BANK)); + verifySchema( + result1, + schema("avg(balance)", "double"), + schema("gender", "string")); + verifyDataRows(result1, rows(40488, "F"), rows(16377.25, "M")); + JSONObject result2 = executeQuery(String.format("source=%s | chart avg(balance) over gender", TEST_INDEX_BANK)); + assertJsonEquals(result1.toString(), result2.toString()); + } + + @Test + public void testChartWithMultipleGroupKeys() throws IOException { + JSONObject result1 = executeQuery(String.format("source=%s | chart avg(balance) by gender, age", TEST_INDEX_BANK)); + verifySchema( + result1, + schema("avg(balance)", "double"), + schema("gender", "string"), + schema("age", "string")); + verifyDataRows(result1, rows(40488, "F", "36"), rows(16377.25, "M", 36)); + JSONObject result2 = executeQuery(String.format("source=%s | chart avg(balance) over gender, age", TEST_INDEX_BANK)); + assertJsonEquals(result1.toString(), result2.toString()); + } + + // TODOs: + // Param nullstr: source=opensearch-sql_test_index_bank_with_null_values | eval age = cast(age as string) | chart nullstr='nil' max(account_number) over gender by age + // Param usenull: source=opensearch-sql_test_index_bank_with_null_values | eval age = cast(age as string) | chart usenull=false nullstr='nil' max(account_number) over gender by age + // Param limit = 0: source=bank | chart limit=0 avg(balance) over state by gender + // SPAN: + +} From cf4c9de03955a7180e96a1363bd27ce2ef35bddb Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Thu, 23 Oct 2025 16:25:24 +0800 Subject: [PATCH 10/35] Hint non-null in aggregateWithTrimming Signed-off-by: Yuanchun Shen --- .../sql/calcite/CalciteRelNodeVisitor.java | 47 ++++++++++--------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 7d8323890be..60138475fe9 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -910,12 +910,14 @@ private boolean isCountField(RexCall call) { * @param groupExprList group by expression list * @param aggExprList aggregate expression list * @param context CalcitePlanContext + * @param hintBucketNonNull adda bucket nullable hint on LogicalAggregate if set * @return Pair of (group-by list, field list, aggregate list) */ private Pair, List> aggregateWithTrimming( List groupExprList, List aggExprList, - CalcitePlanContext context) { + CalcitePlanContext context, + boolean hintBucketNonNull) { Pair, List> resolved = resolveAttributesForAggregation(groupExprList, aggExprList, context); List resolvedGroupByList = resolved.getLeft(); @@ -1019,6 +1021,7 @@ private Pair, List> aggregateWithTrimming( List intendedGroupKeyAliases = getGroupKeyNamesAfterAggregation(reResolved.getLeft()); context.relBuilder.aggregate( context.relBuilder.groupKey(reResolved.getLeft()), reResolved.getRight()); + if (hintBucketNonNull) hintBucketNonNullOnAggregate(context.relBuilder); // During aggregation, Calcite projects both input dependencies and output group-by fields. // When names conflict, Calcite adds numeric suffixes (e.g., "value0"). // Apply explicit renaming to restore the intended aliases. @@ -1027,6 +1030,24 @@ private Pair, List> aggregateWithTrimming( return Pair.of(reResolved.getLeft(), reResolved.getRight()); } + private void hintBucketNonNullOnAggregate(RelBuilder relBuilder) { + final RelHint statHits = + RelHint.builder("stats_args").hintOption(Argument.BUCKET_NULLABLE, "false").build(); + assert relBuilder.peek() instanceof LogicalAggregate + : "Stats hits should be added to LogicalAggregate"; + relBuilder.hints(statHits); + relBuilder + .getCluster() + .setHintStrategies( + HintStrategyTable.builder() + .hintStrategy( + "stats_args", + (hint, rel) -> { + return rel instanceof LogicalAggregate; + }) + .build()); + } + /** * Imitates {@code Registrar.registerExpression} of {@link RelBuilder} to derive the output order * of group-by keys after aggregation. @@ -1128,25 +1149,7 @@ public RelNode visitAggregation(Aggregation node, CalcitePlanContext context) { } Pair, List> aggregationAttributes = - aggregateWithTrimming(groupExprList, aggExprList, context); - if (toAddHintsOnAggregate) { - final RelHint statHits = - RelHint.builder("stats_args").hintOption(Argument.BUCKET_NULLABLE, "false").build(); - assert context.relBuilder.peek() instanceof LogicalAggregate - : "Stats hits should be added to LogicalAggregate"; - context.relBuilder.hints(statHits); - context - .relBuilder - .getCluster() - .setHintStrategies( - HintStrategyTable.builder() - .hintStrategy( - "stats_args", - (hint, rel) -> { - return rel instanceof LogicalAggregate; - }) - .build()); - } + aggregateWithTrimming(groupExprList, aggExprList, context, toAddHintsOnAggregate); // schema reordering // As an example, in command `stats count() by colA, colB`, @@ -1881,7 +1884,7 @@ public RelNode visitRareTopN(RareTopN node, CalcitePlanContext context) { groupExprList.addAll(fieldList); List aggExprList = List.of(AstDSL.alias(countFieldName, AstDSL.aggregate("count", null))); - aggregateWithTrimming(groupExprList, aggExprList, context); + aggregateWithTrimming(groupExprList, aggExprList, context, false); // 2. add a window column List partitionKeys = rexVisitor.analyze(node.getGroupExprList(), context); @@ -2205,7 +2208,7 @@ public RelNode visitTimechart( try { // Step 1: Initial aggregation - IMPORTANT: order is [spanExpr, byField] groupExprList = Arrays.asList(spanExpr, byField); - aggregateWithTrimming(groupExprList, List.of(node.getAggregateFunction()), context); + aggregateWithTrimming(groupExprList, List.of(node.getAggregateFunction()), context, false); // First rename the timestamp field (2nd to last) to @timestamp List fieldNames = context.relBuilder.peek().getRowType().getFieldNames(); From 9b7a891a4af4b4292827d9448bbc68ec58e8fef7 Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Thu, 23 Oct 2025 16:29:28 +0800 Subject: [PATCH 11/35] Add integration tests for chart command Signed-off-by: Yuanchun Shen --- .../calcite/remote/CalciteChartCommandIT.java | 325 +++++++++++++++--- 1 file changed, 280 insertions(+), 45 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java index b326f4ea073..ce6a63e3c24 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java @@ -5,58 +5,293 @@ package org.opensearch.sql.calcite.remote; -import org.json.JSONObject; -import org.junit.jupiter.api.Test; -import org.opensearch.sql.ppl.PPLIntegTestCase; - -import java.io.IOException; - import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_OTEL_LOGS; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_TIME_DATA; import static org.opensearch.sql.util.MatcherUtils.assertJsonEquals; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import static org.opensearch.sql.util.MatcherUtils.verifySchema; +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.ppl.PPLIntegTestCase; + public class CalciteChartCommandIT extends PPLIntegTestCase { - @Override - protected void init() throws Exception { - super.init(); - enableCalcite(); - loadIndex(Index.BANK); - loadIndex(Index.BANK_WITH_NULL_VALUES); - loadIndex(Index.OTELLOGS); - } - - @Test - public void testChartWithSingleGroupKey() throws IOException { - JSONObject result1 = executeQuery(String.format("source=%s | chart avg(balance) by gender", TEST_INDEX_BANK)); - verifySchema( - result1, - schema("avg(balance)", "double"), - schema("gender", "string")); - verifyDataRows(result1, rows(40488, "F"), rows(16377.25, "M")); - JSONObject result2 = executeQuery(String.format("source=%s | chart avg(balance) over gender", TEST_INDEX_BANK)); - assertJsonEquals(result1.toString(), result2.toString()); - } - - @Test - public void testChartWithMultipleGroupKeys() throws IOException { - JSONObject result1 = executeQuery(String.format("source=%s | chart avg(balance) by gender, age", TEST_INDEX_BANK)); - verifySchema( - result1, - schema("avg(balance)", "double"), - schema("gender", "string"), - schema("age", "string")); - verifyDataRows(result1, rows(40488, "F", "36"), rows(16377.25, "M", 36)); - JSONObject result2 = executeQuery(String.format("source=%s | chart avg(balance) over gender, age", TEST_INDEX_BANK)); - assertJsonEquals(result1.toString(), result2.toString()); - } - - // TODOs: - // Param nullstr: source=opensearch-sql_test_index_bank_with_null_values | eval age = cast(age as string) | chart nullstr='nil' max(account_number) over gender by age - // Param usenull: source=opensearch-sql_test_index_bank_with_null_values | eval age = cast(age as string) | chart usenull=false nullstr='nil' max(account_number) over gender by age - // Param limit = 0: source=bank | chart limit=0 avg(balance) over state by gender - // SPAN: + @Override + public void init() throws Exception { + super.init(); + enableCalcite(); + loadIndex(Index.BANK); + loadIndex(Index.BANK_WITH_NULL_VALUES); + loadIndex(Index.OTELLOGS); + loadIndex(Index.TIME_TEST_DATA); + } + + @Test + public void testChartWithSingleGroupKey() throws IOException { + JSONObject result1 = + executeQuery(String.format("source=%s | chart avg(balance) by gender", TEST_INDEX_BANK)); + verifySchema(result1, schema("avg(balance)", "double"), schema("gender", "string")); + verifyDataRows(result1, rows(40488, "F"), rows(16377.25, "M")); + JSONObject result2 = + executeQuery(String.format("source=%s | chart avg(balance) over gender", TEST_INDEX_BANK)); + assertJsonEquals(result1.toString(), result2.toString()); + } + + @Test + public void testChartWithMultipleGroupKeys() throws IOException { + JSONObject result1 = + executeQuery( + String.format("source=%s | chart avg(balance) over gender by age", TEST_INDEX_BANK)); + verifySchema( + result1, + schema("gender", "string"), + schema("age", "string"), + schema("avg(balance)", "double")); + verifyDataRows( + result1, + rows("F", "28", 32838), + rows("F", "39", 40540), + rows("M", "32", 39225), + rows("M", "33", 4180), + rows("M", "36", 11052), + rows("F", "34", 48086)); + JSONObject result2 = + executeQuery( + String.format("source=%s | chart avg(balance) by gender, age", TEST_INDEX_BANK)); + assertJsonEquals(result1.toString(), result2.toString()); + } + + @Test + public void testChartCombineOverByWithLimit0() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | chart limit=0 avg(balance) over state by gender", TEST_INDEX_BANK)); + verifySchema( + result, + schema("avg(balance)", "double"), + schema("state", "string"), + schema("gender", "string")); + verifyDataRows( + result, + rows(39225.0, "IL", "M"), + rows(48086.0, "IN", "F"), + rows(4180.0, "MD", "M"), + rows(40540.0, "PA", "F"), + rows(5686.0, "TN", "M"), + rows(32838.0, "VA", "F"), + rows(16418.0, "WA", "M")); + } + + @Test + public void testChartMaxBalanceByAgeSpan() throws IOException { + JSONObject result = + executeQuery( + String.format("source=%s | chart max(balance) by age span=10", TEST_INDEX_BANK)); + verifySchema(result, schema("max(balance)", "bigint"), schema("age", "int")); + verifyDataRows(result, rows(32838, 20), rows(48086, 30)); + } + + @Test + public void testChartMaxValueOverTimestampSpanWeekByCategory() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | chart max(value) over timestamp span=1week by category", + TEST_INDEX_TIME_DATA)); + verifySchema( + result, + schema("timestamp", "timestamp"), + schema("category", "string"), + schema("max(value)", "int")); + // Data spans from 2025-07-28 to 2025-08-01, all within same week + verifyDataRows( + result, + rows("2025-07-28 00:00:00", "A", 9367), + rows("2025-07-28 00:00:00", "B", 9521), + rows("2025-07-28 00:00:00", "C", 9187), + rows("2025-07-28 00:00:00", "D", 8736)); + } + + @Test + public void testChartMaxValueOverCategoryByTimestampSpanWeek() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | chart max(value) over category by timestamp span=1week", + TEST_INDEX_TIME_DATA)); + verifySchema( + result, + schema("category", "string"), + schema("timestamp", "string"), + schema("max(value)", "int")); + // All data within same week span + verifyDataRows( + result, + rows("A", "2025-07-28 00:00:00", 9367), + rows("B", "2025-07-28 00:00:00", 9521), + rows("C", "2025-07-28 00:00:00", 9187), + rows("D", "2025-07-28 00:00:00", 8736)); + } + + @Test + public void testChartMaxValueByTimestampSpanDayAndWeek() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | chart max(value) by timestamp span=1day, @timestamp span=2weeks", + TEST_INDEX_TIME_DATA)); + // column split are converted to string in order to be compatible with nullstr and otherstr + verifySchema( + result, + schema("timestamp", "timestamp"), + schema("@timestamp", "string"), + schema("max(value)", "int")); + // Data grouped by day spans + verifyDataRows( + result, + rows("2025-07-28 00:00:00", "2025-07-28 00:00:00", 9367), + rows("2025-07-29 00:00:00", "2025-07-28 00:00:00", 9521), + rows("2025-07-30 00:00:00", "2025-07-28 00:00:00", 9234), + rows("2025-07-31 00:00:00", "2025-07-28 00:00:00", 9318), + rows("2025-08-01 00:00:00", "2025-07-28 00:00:00", 9015)); + } + + @Test + public void testChartLimit0WithUseOther() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | chart limit=0 useother=true otherstr='max_among_other'" + + " max(severityNumber) over flags by severityText", + TEST_INDEX_OTEL_LOGS)); + verifySchema( + result, + schema("max(severityNumber)", "bigint"), + schema("flags", "bigint"), + schema("severityText", "string")); + verifyDataRows( + result, + rows(5, 0, "DEBUG"), + rows(6, 0, "DEBUG2"), + rows(7, 0, "DEBUG3"), + rows(8, 0, "DEBUG4"), + rows(17, 0, "ERROR"), + rows(18, 0, "ERROR2"), + rows(19, 0, "ERROR3"), + rows(20, 0, "ERROR4"), + rows(21, 0, "FATAL"), + rows(22, 0, "FATAL2"), + rows(23, 0, "FATAL3"), + rows(24, 0, "FATAL4"), + rows(9, 0, "INFO"), + rows(10, 0, "INFO2"), + rows(11, 0, "INFO3"), + rows(12, 0, "INFO4"), + rows(2, 0, "TRACE2"), + rows(3, 0, "TRACE3"), + rows(4, 0, "TRACE4"), + rows(13, 0, "WARN"), + rows(14, 0, "WARN2"), + rows(15, 0, "WARN3"), + rows(16, 0, "WARN4"), + rows(17, 1, "ERROR"), + rows(9, 1, "INFO"), + rows(1, 1, "TRACE")); + } + + @Test + public void testChartLimitTopWithUseOther() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | chart limit=top 2 useother=true otherstr='max_among_other'" + + " max(severityNumber) over flags by severityText", + TEST_INDEX_OTEL_LOGS)); + verifySchema( + result, + schema("flags", "bigint"), + schema("severityText", "string"), + schema("max(severityNumber)", "bigint")); + verifyDataRows( + result, + rows(1, "max_among_other", 17), + rows(0, "max_among_other", 22), + rows(0, "FATAL3", 23), + rows(0, "FATAL4", 24)); + } + + @Test + public void testChartLimitBottomWithUseOther() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | chart limit=bottom 2 useother=false otherstr='other_small_not_shown'" + + " max(severityNumber) over flags by severityText", + TEST_INDEX_OTEL_LOGS)); + verifySchema( + result, + schema("flags", "bigint"), + schema("severityText", "string"), + schema("max(severityNumber)", "bigint")); + verifyDataRows(result, rows(1, "TRACE", 1), rows(0, "TRACE2", 2)); + } + + @Test + public void testChartLimitTopWithMinAgg() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | chart limit=top 2 min(severityNumber) over flags by severityText", + TEST_INDEX_OTEL_LOGS)); + verifySchema( + result, + schema("flags", "bigint"), + schema("severityText", "string"), + schema("min(severityNumber)", "bigint")); + verifyDataRows( + result, + rows(1, "OTHER", 9), + rows(1, "TRACE", 1), + rows(0, "OTHER", 3), + rows(0, "TRACE2", 2)); + } + + @Test + public void testChartUseNullTrueWithNullStr() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | chart nullstr='nil' avg(balance) over gender by age span=10", + TEST_INDEX_BANK_WITH_NULL_VALUES)); + verifySchema( + result, + schema("gender", "string"), + schema("age", "string"), + schema("avg(balance)", "double")); + verifyDataRows( + result, + rows("M", "30", 21702.5), + rows("F", "30", 48086.0), + rows("F", "20", 32838.0), + rows("F", "nil", null)); + } + @Test + public void testChartUseNullFalseWithNullStr() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | chart usenull=false nullstr='not_shown' count() over gender by age" + + " span=10", + TEST_INDEX_BANK_WITH_NULL_VALUES)); + verifySchema( + result, schema("gender", "string"), schema("age", "string"), schema("count()", "bigint")); + verifyDataRows(result, rows("M", "30", 4), rows("F", "30", 1), rows("F", "20", 1)); + } } From 86173099f77da05ffd7e1b9f11a4c2865ed3ac99 Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Thu, 23 Oct 2025 18:19:39 +0800 Subject: [PATCH 12/35] Add unit tests Signed-off-by: Yuanchun Shen --- docs/user/ppl/cmd/chart.rst | 0 .../sql/ppl/parser/AstBuilderTest.java | 92 +++++++++++++++++++ .../ppl/parser/AstExpressionBuilderTest.java | 63 +++++++++++++ .../sql/ppl/utils/ArgumentFactoryTest.java | 38 ++++++++ 4 files changed, 193 insertions(+) create mode 100644 docs/user/ppl/cmd/chart.rst diff --git a/docs/user/ppl/cmd/chart.rst b/docs/user/ppl/cmd/chart.rst new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index d8b8e6e89ed..74d50f6e3ba 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -67,6 +67,7 @@ import org.mockito.Mockito; import org.opensearch.sql.ast.Node; import org.opensearch.sql.ast.dsl.AstDSL; +import org.opensearch.sql.ast.expression.AllFields; import org.opensearch.sql.ast.expression.DataType; import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.expression.ParseMethod; @@ -74,6 +75,7 @@ import org.opensearch.sql.ast.expression.PatternMode; import org.opensearch.sql.ast.expression.SpanUnit; import org.opensearch.sql.ast.tree.AD; +import org.opensearch.sql.ast.tree.Chart; import org.opensearch.sql.ast.tree.Kmeans; import org.opensearch.sql.ast.tree.ML; import org.opensearch.sql.ast.tree.RareTopN.CommandType; @@ -1448,4 +1450,94 @@ public void testReplaceCommandWithMultiplePairs() { // Test multiple pattern/replacement pairs plan("source=t | replace 'a' WITH 'A', 'b' WITH 'B' IN field"); } + + @Test + public void testChartCommandBasic() { + assertEqual( + "source=t | chart count() by age", + Chart.builder() + .child(relation("t")) + .columnSplit(alias("age", field("age"))) + .aggregationFunctions(List.of(alias("count()", aggregate("count", AllFields.of())))) + .arguments(emptyList()) + .build()); + } + + @Test + public void testChartCommandWithRowSplit() { + assertEqual( + "source=t | chart count() over status by age", + Chart.builder() + .child(relation("t")) + .rowSplit(alias("status", field("status"))) + .columnSplit(alias("age", field("age"))) + .aggregationFunctions(List.of(alias("count()", aggregate("count", AllFields.of())))) + .arguments(emptyList()) + .build()); + } + + @Test + public void testChartCommandWithMultipleAggregations() { + assertEqual( + "source=t | chart avg(salary), max(age) by department", + Chart.builder() + .child(relation("t")) + .columnSplit(alias("department", field("department"))) + .aggregationFunctions( + List.of( + alias("avg(salary)", aggregate("avg", field("salary"))), + alias("max(age)", aggregate("max", field("age"))))) + .arguments(emptyList()) + .build()); + } + + @Test + public void testChartCommandWithOptions() { + assertEqual( + "source=t | chart limit=10 useother=true count() by status", + Chart.builder() + .child(relation("t")) + .columnSplit(alias("status", field("status"))) + .aggregationFunctions(List.of(alias("count()", aggregate("count", AllFields.of())))) + .arguments( + exprList( + argument("limit", intLiteral(10)), + argument("top", booleanLiteral(true)), + argument("useother", booleanLiteral(true)))) + .build()); + } + + @Test + public void testChartCommandWithAllOptions() { + assertEqual( + "source=t | chart limit=5 useother=false otherstr='OTHER' usenull=true nullstr='NULL'" + + " avg(balance) by gender", + Chart.builder() + .child(relation("t")) + .columnSplit(alias("gender", field("gender"))) + .aggregationFunctions( + List.of(alias("avg(balance)", aggregate("avg", field("balance"))))) + .arguments( + exprList( + argument("limit", intLiteral(5)), + argument("top", booleanLiteral(true)), + argument("useother", booleanLiteral(false)), + argument("otherstr", stringLiteral("OTHER")), + argument("usenull", booleanLiteral(true)), + argument("nullstr", stringLiteral("NULL")))) + .build()); + } + + @Test + public void testChartCommandWithBottomLimit() { + assertEqual( + "source=t | chart limit=bottom 3 count() by category", + Chart.builder() + .child(relation("t")) + .columnSplit(alias("category", field("category"))) + .aggregationFunctions(List.of(alias("count()", aggregate("count", AllFields.of())))) + .arguments( + exprList(argument("limit", intLiteral(3)), argument("top", booleanLiteral(false)))) + .build()); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java index 6b0e0a081f8..5d10960ea6b 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java @@ -14,6 +14,7 @@ import static org.opensearch.sql.ast.dsl.AstDSL.allFields; import static org.opensearch.sql.ast.dsl.AstDSL.and; import static org.opensearch.sql.ast.dsl.AstDSL.argument; +import static org.opensearch.sql.ast.dsl.AstDSL.bin; import static org.opensearch.sql.ast.dsl.AstDSL.booleanLiteral; import static org.opensearch.sql.ast.dsl.AstDSL.caseWhen; import static org.opensearch.sql.ast.dsl.AstDSL.cast; @@ -1605,4 +1606,66 @@ public void testVisitSpanLiteral() { .useOther(true) .build()); } + + @Test + public void testBinOptionWithSpan() { + assertEqual( + "source=t | bin age span=10", + bin(relation("t"), field("age"), argument("span", intLiteral(10)))); + } + + @Test + public void testBinOptionWithBins() { + assertEqual( + "source=t | bin age bins=5", + bin(relation("t"), field("age"), argument("bins", intLiteral(5)))); + } + + @Test + public void testBinOptionWithMinspan() { + assertEqual( + "source=t | bin age minspan=100", + bin(relation("t"), field("age"), argument("minspan", intLiteral(100)))); + } + + @Test + public void testBinOptionWithAligntimeEarliest() { + assertEqual( + "source=t | bin age span=10 aligntime=earliest", + bin( + relation("t"), + field("age"), + argument("span", intLiteral(10)), + argument("aligntime", stringLiteral("earliest")))); + } + + @Test + public void testBinOptionWithAligntimeLiteralValue() { + assertEqual( + "source=t | bin age span=10 aligntime=1000", + bin( + relation("t"), + field("age"), + argument("span", intLiteral(10)), + argument("aligntime", intLiteral(1000)))); + } + + @Test + public void testBinOptionWithStartAndEnd() { + assertEqual( + "source=t | bin age bins=10 start=0 end=100", + bin( + relation("t"), + field("age"), + argument("bins", intLiteral(10)), + argument("start", intLiteral(0)), + argument("end", intLiteral(100)))); + } + + @Test + public void testBinOptionWithTimeSpan() { + assertEqual( + "source=t | bin timestamp span=1h", + bin(relation("t"), field("timestamp"), argument("span", stringLiteral("1h")))); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/ArgumentFactoryTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/ArgumentFactoryTest.java index adb9ec719e6..e268656a8d9 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/ArgumentFactoryTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/ArgumentFactoryTest.java @@ -20,8 +20,11 @@ import static org.opensearch.sql.ast.dsl.AstDSL.sort; import static org.opensearch.sql.ast.dsl.AstDSL.stringLiteral; +import com.google.common.collect.ImmutableList; import org.junit.Test; +import org.opensearch.sql.ast.expression.AllFields; import org.opensearch.sql.ast.expression.Argument; +import org.opensearch.sql.ast.tree.Chart; import org.opensearch.sql.ppl.parser.AstBuilderTest; public class ArgumentFactoryTest extends AstBuilderTest { @@ -100,6 +103,41 @@ public void testSortFieldArgument() { argument("type", stringLiteral("auto")))))); } + @Test + public void testChartCommandArguments() { + assertEqual( + "source=t | chart limit=5 useother=true otherstr='OTHER_VAL' usenull=false" + + " nullstr='NULL_VAL' count() by age", + Chart.builder() + .child(relation("t")) + .columnSplit(alias("age", field("age"))) + .aggregationFunctions( + ImmutableList.of(alias("count()", aggregate("count", AllFields.of())))) + .arguments( + exprList( + argument("limit", intLiteral(5)), + argument("top", booleanLiteral(true)), + argument("useother", booleanLiteral(true)), + argument("otherstr", stringLiteral("OTHER_VAL")), + argument("usenull", booleanLiteral(false)), + argument("nullstr", stringLiteral("NULL_VAL")))) + .build()); + } + + @Test + public void testChartCommandBottomArguments() { + assertEqual( + "source=t | chart limit=bottom 3 count() by status", + Chart.builder() + .child(relation("t")) + .columnSplit(alias("status", field("status"))) + .aggregationFunctions( + ImmutableList.of(alias("count()", aggregate("count", AllFields.of())))) + .arguments( + exprList(argument("limit", intLiteral(3)), argument("top", booleanLiteral(false)))) + .build()); + } + @Test public void testNoArgConstructorForArgumentFactoryShouldPass() { new ArgumentFactory(); From 3c4c13a447832c52bdef9d948bad8cafcefabba9 Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Thu, 23 Oct 2025 18:20:12 +0800 Subject: [PATCH 13/35] Add doc for chart command Signed-off-by: Yuanchun Shen --- docs/category.json | 1 + docs/user/ppl/cmd/chart.rst | 194 ++++++++++++++++++++++++++++++++++++ docs/user/ppl/index.rst | 2 + 3 files changed, 197 insertions(+) diff --git a/docs/category.json b/docs/category.json index d9605598800..aa1380f1efb 100644 --- a/docs/category.json +++ b/docs/category.json @@ -49,6 +49,7 @@ "user/ppl/cmd/stats.rst", "user/ppl/cmd/subquery.rst", "user/ppl/cmd/syntax.rst", + "user/ppl/cmd/chart.rst", "user/ppl/cmd/timechart.rst", "user/ppl/cmd/search.rst", "user/ppl/functions/statistical.rst", diff --git a/docs/user/ppl/cmd/chart.rst b/docs/user/ppl/cmd/chart.rst index e69de29bb2d..1633302676a 100644 --- a/docs/user/ppl/cmd/chart.rst +++ b/docs/user/ppl/cmd/chart.rst @@ -0,0 +1,194 @@ +============= +chart +============= + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 2 + + +Description +============ + +The ``chart`` command transforms search results by applying a statistical aggregation function and optionally grouping the data by one or two fields. The results are suitable for visualization as a two-dimension chart when grouping by two fields, where unique values in the second group key can be pivoted to column names. + +Version +======= +3.4.0 + +Syntax +============ + +.. code-block:: text + + chart + [limit=(top|bottom) ] [useother=] [usenull=] [nullstr=] [otherstr=] + + [ by ] | [over ] [ by ] + +**Parameters:** + +* **limit**: optional. Specifies the number of distinct values to display when using column split. + + * Default: 10 + * Syntax: ``limit=(top|bottom) `` or ``limit=`` (defaults to top) + * When there are more distinct values than the limit, the additional values are grouped into an "OTHER" category if useother is not set to false. + * Set to 0 to show all distinct values without any limit. + * Only applies when using column split (over...by clause). + +* **useother**: optional. Controls whether to create an "OTHER" category for values beyond the limit. + + * Default: true + * When set to false, only the top/bottom N values (based on limit) are shown without an "OTHER" category. + * When set to true, values beyond the limit are grouped into an "OTHER" category. + * Only applies when using column split and when there are more distinct values than the limit. + +* **usenull**: optional. Controls whether to include null values as a separate category. + + * Default: true + * When set to false, events with null values in the split-by field are excluded from results. + * When set to true, null values appear as a separate category. + +* **nullstr**: optional. Specifies the string to display for null values. + + * Default: "NULL" + * Only applies when usenull is set to true. + +* **otherstr**: optional. Specifies the string to display for the "OTHER" category. + + * Default: "OTHER" + * Only applies when useother is set to true and there are values beyond the limit. + +* **aggregation_function**: mandatory. The aggregation function to apply to the data. + + * Currently, only a single aggregation function is supported. + * Available functions: All aggregation functions supported by the :doc:`stats ` command. + +* **by**: optional. Groups the results by the specified field as rows. + + * If not specified, the aggregation is performed across all documents. + +* **over...by**: optional. Alternative syntax for grouping by multiple fields. + + * ``over by `` groups the results by both fields. + * The row_split field becomes the primary grouping dimension. + * The column_split field becomes the secondary grouping dimension. + * Results are returned as individual rows for each combination. + +Notes +===== + +* The ``chart`` command transforms results into a table format suitable for visualization. +* When using multiple grouping fields (over...by syntax), the output contains individual rows for each combination of the grouping fields. +* The limit parameter determines how many columns to show when there are many distinct values. +* Results are ordered by the aggregated values to determine top/bottom selections. + +Examples +======== + +Example 1: Basic aggregation without grouping +============================================== + +This example calculates the average balance across all accounts. + +PPL query:: + + os> source=accounts | chart avg(balance) + fetched rows / total rows = 1/1 + +--------------+ + | avg(balance) | + |--------------| + | 20482.25 | + +--------------+ + +Example 2: Group by single field +================================= + +This example calculates the count of accounts grouped by gender. + +PPL query:: + + os> source=accounts | chart count() by gender + fetched rows / total rows = 2/2 + +---------+--------+ + | count() | gender | + |---------+--------| + | 1 | F | + | 3 | M | + +---------+--------+ + +Example 3: Using over and by for multiple field grouping +======================================================== + +This example shows average balance grouped by both gender and age fields. + +PPL query:: + + os> source=accounts | chart avg(balance) over gender by age + fetched rows / total rows = 4/4 + +--------+-----+--------------+ + | gender | age | avg(balance) | + |--------+-----+--------------| + | F | 28 | 32838.0 | + | M | 32 | 39225.0 | + | M | 33 | 4180.0 | + | M | 36 | 5686.0 | + +--------+-----+--------------+ + +Example 4: Using basic limit functionality +======================================== + +This example limits the results to show only the top 1 age group. + +PPL query:: + + os> source=accounts | chart limit=1 count() over gender by age + fetched rows / total rows = 3/3 + +--------+-------+---------+ + | gender | age | count() | + |--------+-------+---------| + | M | OTHER | 2 | + | M | 33 | 1 | + | F | OTHER | 1 | + +--------+-------+---------+ + +Example 5: Using limit with other parameters +============================================= + +This example shows using limit with useother and custom otherstr parameters. + +PPL query:: + + os> source=accounts | chart limit=top 2 useother=true otherstr='remaining_accounts' max(balance) over state by gender + fetched rows / total rows = 4/4 + +-------+--------+--------------+ + | state | gender | max(balance) | + |-------+--------+--------------| + | TN | M | 5686 | + | MD | M | 4180 | + | IL | M | 39225 | + | VA | F | 32838 | + +-------+--------+--------------+ + +Example 6: Using span with chart command +======================================= + +This example demonstrates using span for grouping age ranges. + +PPL query:: + + os> source=accounts | chart max(balance) by age span=10 + fetched rows / total rows = 2/2 + +--------------+-----+ + | max(balance) | age | + |--------------+-----| + | 32838 | 20 | + | 39225 | 30 | + +--------------+-----+ + +Limitations +============ +* Only a single aggregation function is supported per chart command. +* When using both row and column splits, the column split field is converted to string type so that it can be used as column names. \ No newline at end of file diff --git a/docs/user/ppl/index.rst b/docs/user/ppl/index.rst index 17b4797df39..6792c161ed0 100644 --- a/docs/user/ppl/index.rst +++ b/docs/user/ppl/index.rst @@ -117,6 +117,8 @@ The query start with search command and then flowing a set of command delimited - `reverse command `_ - `table command `_ + + - `chart command `_ - `timechart command `_ From 5de82dd663be055979fa35fd110a8a56f94ca240 Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Fri, 24 Oct 2025 10:36:26 +0800 Subject: [PATCH 14/35] Prompt users that multiple agg is not supported Signed-off-by: Yuanchun Shen --- .../opensearch/sql/ppl/parser/AstBuilder.java | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index 00a4e10c0e6..cc489b27355 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -406,16 +406,7 @@ private ReplacePair buildReplacePair(OpenSearchPPLParser.ReplacePairContext ctx) /** Stats command. */ @Override public UnresolvedPlan visitStatsCommand(StatsCommandContext ctx) { - ImmutableList.Builder aggListBuilder = new ImmutableList.Builder<>(); - for (OpenSearchPPLParser.StatsAggTermContext aggCtx : ctx.statsAggTerm()) { - UnresolvedExpression aggExpression = internalVisitExpression(aggCtx.statsFunction()); - String name = - aggCtx.alias == null - ? getTextInQuery(aggCtx) - : StringUtils.unquoteIdentifier(aggCtx.alias.getText()); - Alias alias = new Alias(name, aggExpression); - aggListBuilder.add(alias); - } + List aggregations = parseAggTerms(ctx.statsAggTerm()); List groupList = Optional.ofNullable(ctx.statsByClause()) @@ -440,7 +431,7 @@ public UnresolvedPlan visitStatsCommand(StatsCommandContext ctx) { Aggregation aggregation = new Aggregation( - aggListBuilder.build(), + aggregations, Collections.emptyList(), groupList, span, @@ -615,8 +606,23 @@ public UnresolvedPlan visitChartCommand(OpenSearchPPLParser.ChartCommandContext UnresolvedExpression columnSplit = ctx.columnSplit() == null ? null : internalVisitExpression(ctx.columnSplit()); List arguments = ArgumentFactory.getArgumentList(ctx); + List aggList = parseAggTerms(ctx.statsAggTerm()); + if (aggList.size() > 1) { + throw new IllegalArgumentException( + "Chart command does not support multiple aggregation functions yet"); + } + return Chart.builder() + .rowSplit(rowSplit) + .columnSplit(columnSplit) + .aggregationFunctions(aggList) + .arguments(arguments) + .build(); + } + + private List parseAggTerms( + List statsAggTermContexts) { ImmutableList.Builder aggListBuilder = new ImmutableList.Builder<>(); - for (OpenSearchPPLParser.StatsAggTermContext aggCtx : ctx.statsAggTerm()) { + for (OpenSearchPPLParser.StatsAggTermContext aggCtx : statsAggTermContexts) { UnresolvedExpression aggExpression = internalVisitExpression(aggCtx.statsFunction()); String name = aggCtx.alias == null @@ -625,12 +631,7 @@ public UnresolvedPlan visitChartCommand(OpenSearchPPLParser.ChartCommandContext Alias alias = new Alias(name, aggExpression); aggListBuilder.add(alias); } - return Chart.builder() - .rowSplit(rowSplit) - .columnSplit(columnSplit) - .aggregationFunctions(aggListBuilder.build()) - .arguments(arguments) - .build(); + return aggListBuilder.build(); } /** Timechart command. */ From d3858ccf2dc0c475141f99493805f8a72abe230f Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Fri, 24 Oct 2025 14:05:37 +0800 Subject: [PATCH 15/35] Add explain ITs Signed-off-by: Yuanchun Shen --- .../sql/calcite/CalciteNoPushdownIT.java | 165 +++++++++--------- .../sql/calcite/remote/CalciteExplainIT.java | 62 +++++++ .../explain_chart_multiple_group_keys.yaml | 32 ++++ .../calcite/explain_chart_null_str.yaml | 36 ++++ .../explain_chart_single_group_key.yaml | 9 + .../calcite/explain_chart_timestamp_span.yaml | 26 +++ .../calcite/explain_chart_use_other.yaml | 26 +++ .../calcite/explain_chart_with_limit.yaml | 9 + .../calcite/explain_chart_with_span.yaml | 9 + .../explain_chart_multiple_group_keys.yaml | 35 ++++ .../explain_chart_multiple_groups.yaml | 35 ++++ .../explain_chart_null_str.yaml | 37 ++++ .../explain_chart_single_group.yaml | 13 ++ .../explain_chart_single_group_key.yaml | 13 ++ .../explain_chart_timestamp_span.yaml | 32 ++++ .../explain_chart_use_other.yaml | 30 ++++ .../explain_chart_with_limit.yaml | 13 ++ .../explain_chart_with_span.yaml | 14 ++ 18 files changed, 513 insertions(+), 83 deletions(-) create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_chart_multiple_group_keys.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_chart_null_str.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_chart_single_group_key.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_chart_timestamp_span.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_chart_use_other.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_limit.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_span.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_group_keys.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_groups.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_null_str.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_single_group.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_single_group_key.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_timestamp_span.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_use_other.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_limit.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_span.yaml diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java index 69507c71aa5..14e2bfdb4da 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java @@ -10,7 +10,6 @@ import org.junit.runner.RunWith; import org.junit.runners.Suite; import org.opensearch.sql.calcite.remote.*; -import org.opensearch.sql.calcite.tpch.CalcitePPLTpchIT; import org.opensearch.sql.ppl.PPLIntegTestCase; /** @@ -21,88 +20,88 @@ @RunWith(Suite.class) @Suite.SuiteClasses({ CalciteExplainIT.class, - CalciteArrayFunctionIT.class, - CalciteBinCommandIT.class, - CalciteConvertTZFunctionIT.class, - CalciteCsvFormatIT.class, - CalciteDataTypeIT.class, - CalciteDateTimeComparisonIT.class, - CalciteDateTimeFunctionIT.class, - CalciteDateTimeImplementationIT.class, - CalciteDedupCommandIT.class, - CalciteDescribeCommandIT.class, - CalciteExpandCommandIT.class, - CalciteFieldsCommandIT.class, - CalciteFillNullCommandIT.class, - CalciteFlattenCommandIT.class, - CalciteFlattenDocValueIT.class, - CalciteGeoIpFunctionsIT.class, - CalciteGeoPointFormatsIT.class, - CalciteHeadCommandIT.class, - CalciteInformationSchemaCommandIT.class, - CalciteIPComparisonIT.class, - CalciteIPFunctionsIT.class, - CalciteJsonFunctionsIT.class, - CalciteLegacyAPICompatibilityIT.class, - CalciteLikeQueryIT.class, - CalciteMathematicalFunctionIT.class, - CalciteMultisearchCommandIT.class, - CalciteMultiValueStatsIT.class, - CalciteNewAddedCommandsIT.class, - CalciteNowLikeFunctionIT.class, - CalciteObjectFieldOperateIT.class, - CalciteOperatorIT.class, - CalciteParseCommandIT.class, - CalcitePPLAggregationIT.class, - CalcitePPLAppendcolIT.class, - CalcitePPLAppendCommandIT.class, - CalcitePPLBasicIT.class, - CalcitePPLBuiltinDatetimeFunctionInvalidIT.class, - CalcitePPLBuiltinFunctionIT.class, - CalcitePPLBuiltinFunctionsNullIT.class, - CalcitePPLCaseFunctionIT.class, - CalcitePPLCastFunctionIT.class, - CalcitePPLConditionBuiltinFunctionIT.class, - CalcitePPLCryptographicFunctionIT.class, - CalcitePPLDedupIT.class, - CalcitePPLEventstatsIT.class, - CalcitePPLExistsSubqueryIT.class, - CalcitePPLExplainIT.class, - CalcitePPLFillnullIT.class, - CalcitePPLGrokIT.class, - CalcitePPLInSubqueryIT.class, - CalcitePPLIPFunctionIT.class, - CalcitePPLJoinIT.class, - CalcitePPLJsonBuiltinFunctionIT.class, - CalcitePPLLookupIT.class, - CalcitePPLParseIT.class, - CalcitePPLPatternsIT.class, - CalcitePPLPluginIT.class, - CalcitePPLRenameIT.class, - CalcitePPLScalarSubqueryIT.class, - CalcitePPLSortIT.class, - CalcitePPLStringBuiltinFunctionIT.class, - CalcitePPLTrendlineIT.class, - CalcitePrometheusDataSourceCommandsIT.class, - CalciteQueryAnalysisIT.class, - CalciteRareCommandIT.class, - CalciteRegexCommandIT.class, - CalciteRexCommandIT.class, - CalciteRenameCommandIT.class, - CalciteReplaceCommandIT.class, - CalciteResourceMonitorIT.class, - CalciteSearchCommandIT.class, - CalciteSettingsIT.class, - CalciteShowDataSourcesCommandIT.class, - CalciteSortCommandIT.class, - CalciteStatsCommandIT.class, - CalciteSystemFunctionIT.class, - CalciteTextFunctionIT.class, - CalciteTopCommandIT.class, - CalciteTrendlineCommandIT.class, - CalciteVisualizationFormatIT.class, - CalciteWhereCommandIT.class, - CalcitePPLTpchIT.class + // CalciteArrayFunctionIT.class, + // CalciteBinCommandIT.class, + // CalciteConvertTZFunctionIT.class, + // CalciteCsvFormatIT.class, + // CalciteDataTypeIT.class, + // CalciteDateTimeComparisonIT.class, + // CalciteDateTimeFunctionIT.class, + // CalciteDateTimeImplementationIT.class, + // CalciteDedupCommandIT.class, + // CalciteDescribeCommandIT.class, + // CalciteExpandCommandIT.class, + // CalciteFieldsCommandIT.class, + // CalciteFillNullCommandIT.class, + // CalciteFlattenCommandIT.class, + // CalciteFlattenDocValueIT.class, + // CalciteGeoIpFunctionsIT.class, + // CalciteGeoPointFormatsIT.class, + // CalciteHeadCommandIT.class, + // CalciteInformationSchemaCommandIT.class, + // CalciteIPComparisonIT.class, + // CalciteIPFunctionsIT.class, + // CalciteJsonFunctionsIT.class, + // CalciteLegacyAPICompatibilityIT.class, + // CalciteLikeQueryIT.class, + // CalciteMathematicalFunctionIT.class, + // CalciteMultisearchCommandIT.class, + // CalciteMultiValueStatsIT.class, + // CalciteNewAddedCommandsIT.class, + // CalciteNowLikeFunctionIT.class, + // CalciteObjectFieldOperateIT.class, + // CalciteOperatorIT.class, + // CalciteParseCommandIT.class, + // CalcitePPLAggregationIT.class, + // CalcitePPLAppendcolIT.class, + // CalcitePPLAppendCommandIT.class, + // CalcitePPLBasicIT.class, + // CalcitePPLBuiltinDatetimeFunctionInvalidIT.class, + // CalcitePPLBuiltinFunctionIT.class, + // CalcitePPLBuiltinFunctionsNullIT.class, + // CalcitePPLCaseFunctionIT.class, + // CalcitePPLCastFunctionIT.class, + // CalcitePPLConditionBuiltinFunctionIT.class, + // CalcitePPLCryptographicFunctionIT.class, + // CalcitePPLDedupIT.class, + // CalcitePPLEventstatsIT.class, + // CalcitePPLExistsSubqueryIT.class, + // CalcitePPLExplainIT.class, + // CalcitePPLFillnullIT.class, + // CalcitePPLGrokIT.class, + // CalcitePPLInSubqueryIT.class, + // CalcitePPLIPFunctionIT.class, + // CalcitePPLJoinIT.class, + // CalcitePPLJsonBuiltinFunctionIT.class, + // CalcitePPLLookupIT.class, + // CalcitePPLParseIT.class, + // CalcitePPLPatternsIT.class, + // CalcitePPLPluginIT.class, + // CalcitePPLRenameIT.class, + // CalcitePPLScalarSubqueryIT.class, + // CalcitePPLSortIT.class, + // CalcitePPLStringBuiltinFunctionIT.class, + // CalcitePPLTrendlineIT.class, + // CalcitePrometheusDataSourceCommandsIT.class, + // CalciteQueryAnalysisIT.class, + // CalciteRareCommandIT.class, + // CalciteRegexCommandIT.class, + // CalciteRexCommandIT.class, + // CalciteRenameCommandIT.class, + // CalciteReplaceCommandIT.class, + // CalciteResourceMonitorIT.class, + // CalciteSearchCommandIT.class, + // CalciteSettingsIT.class, + // CalciteShowDataSourcesCommandIT.class, + // CalciteSortCommandIT.class, + // CalciteStatsCommandIT.class, + // CalciteSystemFunctionIT.class, + // CalciteTextFunctionIT.class, + // CalciteTopCommandIT.class, + // CalciteTrendlineCommandIT.class, + // CalciteVisualizationFormatIT.class, + // CalciteWhereCommandIT.class, + // CalcitePPLTpchIT.class }) public class CalciteNoPushdownIT { private static boolean wasPushdownEnabled; diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 6528eb4d2b9..ed27e7fe8db 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -7,8 +7,10 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_LOGS; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_SIMPLE; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_OTEL_LOGS; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STRINGS; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_TIME_DATA; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WEBLOGS; @@ -31,6 +33,7 @@ public void init() throws Exception { enableCalcite(); setQueryBucketSize(1000); loadIndex(Index.BANK_WITH_STRING_VALUES); + loadIndex(Index.BANK_WITH_NULL_VALUES); loadIndex(Index.NESTED_SIMPLE); loadIndex(Index.TIME_TEST_DATA); loadIndex(Index.TIME_TEST_DATA2); @@ -1277,6 +1280,65 @@ public void testPushDownMinOrMaxAggOnDerivedField() throws IOException { TEST_INDEX_ACCOUNT))); } + @Test + public void testExplainChartWithSingleGroupKey() throws IOException { + assertYamlEqualsIgnoreId( + loadExpectedPlan("explain_chart_single_group_key.yaml"), + explainQueryYaml( + String.format("source=%s | chart avg(balance) by gender", TEST_INDEX_BANK))); + + assertYamlEqualsIgnoreId( + loadExpectedPlan("explain_chart_with_span.yaml"), + explainQueryYaml( + String.format("source=%s | chart max(balance) by age span=10", TEST_INDEX_BANK))); + + assertYamlEqualsIgnoreId( + loadExpectedPlan("explain_chart_timestamp_span.yaml"), + explainQueryYaml( + String.format( + "source=%s | chart max(value) over timestamp span=1week by category", + TEST_INDEX_TIME_DATA))); + } + + @Test + public void testExplainChartWithMultipleGroupKeys() throws IOException { + String expected = loadExpectedPlan("explain_chart_multiple_group_keys.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + String.format("source=%s | chart avg(balance) over gender by age", TEST_INDEX_BANK))); + } + + @Test + public void testExplainChartWithLimits() throws IOException { + String expected = loadExpectedPlan("explain_chart_with_limit.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + String.format( + "source=%s | chart limit=0 avg(balance) over state by gender", TEST_INDEX_BANK))); + + assertYamlEqualsIgnoreId( + loadExpectedPlan("explain_chart_use_other.yaml"), + explainQueryYaml( + String.format( + "source=%s | chart limit=2 useother=true otherstr='max_among_other'" + + " max(severityNumber) over flags by severityText", + TEST_INDEX_OTEL_LOGS))); + } + + @Test + public void testExplainChartWithNullStr() throws IOException { + String expected = loadExpectedPlan("explain_chart_null_str.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + String.format( + "source=%s | chart limit=10 usenull=true nullstr='nil' avg(balance) over gender by" + + " age span=10", + TEST_INDEX_BANK_WITH_NULL_VALUES))); + } + @Test public void testCasePushdownAsRangeQueryExplain() throws IOException { // CASE 1: Range - Metric diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_multiple_group_keys.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_multiple_group_keys.yaml new file mode 100644 index 00000000000..b4419f38e11 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_multiple_group_keys.yaml @@ -0,0 +1,32 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{1, 2}], avg(balance)=[AVG($0)]) + LogicalProject(avg(balance)=[$0], gender=[$1], age=[CASE(IS NULL($2), 'NULL', <=($5, 10), $2, 'OTHER')]) + LogicalJoin(condition=[=($2, $3)], joinType=[left]) + LogicalProject(avg(balance)=[$2], gender=[$0], age=[SAFE_CAST($1)]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(gender=[$4], age=[$10], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalProject(age=[$0], __grand_total__=[$1], __row_number__=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) + LogicalAggregate(group=[{1}], __grand_total__=[AVG($0)]) + LogicalProject(avg(balance)=[$2], age=[SAFE_CAST($1)]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(gender=[$4], age=[$10], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(balance)=[$t8]) + EnumerableAggregate(group=[{1, 2}], agg#0=[$SUM0($0)], agg#1=[COUNT($0)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t2)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t2, $t9)], proj#0..1=[{exprs}], age=[$t10]) + EnumerableMergeJoin(condition=[=($2, $3)], joinType=[left]) + EnumerableSort(sort0=[$2], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[SAFE_CAST($t1)], avg(balance)=[$t2], gender=[$t0], age=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) + EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:DOUBLE], expr#6=[CASE($t4, $t5, $t1)], expr#7=[/($t6, $t2)], age=[$t0], __grand_total__=[$t7]) + EnumerableAggregate(group=[{1}], agg#0=[$SUM0($0)], agg#1=[COUNT($0)]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[SAFE_CAST($t1)], avg(balance)=[$t0], $f1=[$t2]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2)), PROJECT->[avg(balance), age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_null_str.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_null_str.yaml new file mode 100644 index 00000000000..6a3a024b2b3 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_null_str.yaml @@ -0,0 +1,36 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{1, 2}], avg(balance)=[AVG($0)]) + LogicalProject(avg(balance)=[$0], gender=[$1], age=[CASE(IS NULL($2), 'nil', <=($5, 10), $2, 'OTHER')]) + LogicalJoin(condition=[=($2, $3)], joinType=[left]) + LogicalProject(avg(balance)=[$2], gender=[$0], age=[SAFE_CAST($1)]) + LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) + LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) + LogicalProject(age=[$0], __grand_total__=[$1], __row_number__=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) + LogicalAggregate(group=[{1}], __grand_total__=[AVG($0)]) + LogicalProject(avg(balance)=[$2], age=[SAFE_CAST($1)]) + LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) + LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(balance)=[$t8]) + EnumerableAggregate(group=[{1, 2}], agg#0=[$SUM0($0)], agg#1=[COUNT($0)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t2)], expr#6=['nil'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t2, $t9)], proj#0..1=[{exprs}], age=[$t10]) + EnumerableMergeJoin(condition=[=($2, $3)], joinType=[left]) + EnumerableSort(sort0=[$2], dir0=[ASC]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], expr#10=[SAFE_CAST($t1)], avg(balance)=[$t9], gender=[$t0], age=[$t10]) + EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], proj#0..1=[{exprs}], $f2=[$t5]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[gender, balance, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","balance","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) + EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:DOUBLE], expr#6=[CASE($t4, $t5, $t1)], expr#7=[/($t6, $t2)], age=[$t0], __grand_total__=[$t7]) + EnumerableAggregate(group=[{1}], agg#0=[$SUM0($0)], agg#1=[COUNT($0)]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], expr#10=[SAFE_CAST($t1)], avg(balance)=[$t9], age=[$t10]) + EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], proj#0..1=[{exprs}], $f2=[$t5]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[gender, balance, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","balance","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_single_group_key.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_single_group_key.yaml new file mode 100644 index 00000000000..3752736138f --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_single_group_key.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg(balance)=[$1], gender=[$0]) + LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)]) + LogicalProject(gender=[$4], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg(balance)=AVG($1)), PROJECT->[avg(balance), gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_timestamp_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_timestamp_span.yaml new file mode 100644 index 00000000000..9007f4da716 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_timestamp_span.yaml @@ -0,0 +1,26 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{1, 2}], max(value)=[MAX($0)]) + LogicalProject(max(value)=[$0], timestamp=[$1], category=[CASE(IS NULL($2), 'NULL', <=($5, 10), $2, 'OTHER')]) + LogicalJoin(condition=[=($2, $3)], joinType=[left]) + LogicalProject(max(value)=[$2], timestamp=[$1], category=[$0]) + LogicalAggregate(group=[{0, 2}], max(value)=[MAX($1)]) + LogicalProject(category=[$1], value=[$2], timestamp0=[SPAN($3, 1, 'w')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + LogicalProject(category=[$0], __grand_total__=[$1], __row_number__=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) + LogicalAggregate(group=[{1}], __grand_total__=[MAX($0)]) + LogicalProject(max(value)=[$2], category=[$0]) + LogicalAggregate(group=[{0, 2}], max(value)=[MAX($1)]) + LogicalProject(category=[$1], value=[$2], timestamp0=[SPAN($3, 1, 'w')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableAggregate(group=[{1, 2}], max(value)=[MAX($0)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t2)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t2, $t9)], proj#0..1=[{exprs}], category=[$t10]) + EnumerableMergeJoin(condition=[=($2, $3)], joinType=[left]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},max(value)=MAX($1)), PROJECT->[max(value), timestamp0, category], SORT->[2]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"timestamp0":{"date_histogram":{"field":"timestamp","missing_bucket":false,"order":"asc","calendar_interval":"1w"}}}]},"aggregations":{"max(value)":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], category=[$t0], $1=[$t2]) + EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},__grand_total__=MAX($1))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"__grand_total__":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_use_other.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_use_other.yaml new file mode 100644 index 00000000000..d6e799c3c36 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_use_other.yaml @@ -0,0 +1,26 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{1, 2}], max(severityNumber)=[MAX($0)]) + LogicalProject(max(severityNumber)=[$0], flags=[$1], severityText=[CASE(IS NULL($2), 'NULL', <=($5, 2), $2, 'max_among_other')]) + LogicalJoin(condition=[=($2, $3)], joinType=[left]) + LogicalProject(max(severityNumber)=[$2], flags=[$0], severityText=[$1]) + LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) + LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) + LogicalProject(severityText=[$0], __grand_total__=[$1], __row_number__=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) + LogicalAggregate(group=[{1}], __grand_total__=[MAX($0)]) + LogicalProject(max(severityNumber)=[$2], severityText=[$1]) + LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) + LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableAggregate(group=[{1, 2}], max(severityNumber)=[MAX($0)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t2)], expr#6=['NULL'], expr#7=[2], expr#8=[<=($t4, $t7)], expr#9=['max_among_other'], expr#10=[CASE($t5, $t6, $t8, $t2, $t9)], proj#0..1=[{exprs}], severityText=[$t10]) + EnumerableMergeJoin(condition=[=($2, $3)], joinType=[left]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},max(severityNumber)=MAX($2)), PROJECT->[max(severityNumber), flags, severityText], SORT->[2]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"severityText":{"terms":{"field":"severityText","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"flags":{"terms":{"field":"flags","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"max(severityNumber)":{"max":{"field":"severityNumber"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], severityText=[$t0], $1=[$t2]) + EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},__grand_total__=MAX($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"severityText":{"terms":{"field":"severityText","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"__grand_total__":{"max":{"field":"severityNumber"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_limit.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_limit.yaml new file mode 100644 index 00000000000..3077f16152c --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_limit.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg(balance)=[$2], state=[$0], gender=[$1]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(state=[$9], gender=[$4], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2)), PROJECT->[avg(balance), state, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_span.yaml new file mode 100644 index 00000000000..b6af45e0974 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_span.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(max(balance)=[$1], age=[$0]) + LogicalAggregate(group=[{1}], max(balance)=[MAX($0)]) + LogicalProject(balance=[$7], age0=[SPAN($10, 10, null:NULL)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},max(balance)=MAX($0)), PROJECT->[max(balance), age0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age0":{"histogram":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc","interval":10.0}}}]},"aggregations":{"max(balance)":{"max":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_group_keys.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_group_keys.yaml new file mode 100644 index 00000000000..11e45e502bf --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_group_keys.yaml @@ -0,0 +1,35 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{1, 2}], avg(balance)=[AVG($0)]) + LogicalProject(avg(balance)=[$0], gender=[$1], age=[CASE(IS NULL($2), 'NULL', <=($5, 10), $2, 'OTHER')]) + LogicalJoin(condition=[=($2, $3)], joinType=[left]) + LogicalProject(avg(balance)=[$2], gender=[$0], age=[SAFE_CAST($1)]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(gender=[$4], age=[$10], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalProject(age=[$0], __grand_total__=[$1], __row_number__=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) + LogicalAggregate(group=[{1}], __grand_total__=[AVG($0)]) + LogicalProject(avg(balance)=[$2], age=[SAFE_CAST($1)]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(gender=[$4], age=[$10], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(balance)=[$t8]) + EnumerableAggregate(group=[{1, 2}], agg#0=[$SUM0($0)], agg#1=[COUNT($0)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t2)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t2, $t9)], proj#0..1=[{exprs}], age=[$t10]) + EnumerableMergeJoin(condition=[=($2, $3)], joinType=[left]) + EnumerableSort(sort0=[$2], dir0=[ASC]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], expr#10=[SAFE_CAST($t1)], avg(balance)=[$t9], gender=[$t0], age=[$t10]) + EnumerableAggregate(group=[{4, 10}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) + EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:DOUBLE], expr#6=[CASE($t4, $t5, $t1)], expr#7=[/($t6, $t2)], age=[$t0], __grand_total__=[$t7]) + EnumerableAggregate(group=[{1}], agg#0=[$SUM0($0)], agg#1=[COUNT($0)]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], expr#10=[SAFE_CAST($t1)], avg(balance)=[$t9], age=[$t10]) + EnumerableAggregate(group=[{4, 10}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_groups.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_groups.yaml new file mode 100644 index 00000000000..5d641cb3929 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_groups.yaml @@ -0,0 +1,35 @@ +logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{1, 2}], avg(balance)=[AVG($0)]) + LogicalProject(avg(balance)=[$0], gender=[$1], age=[CASE(IS NULL($2), 'NULL', <=($5, 10), $2, 'OTHER')]) + LogicalJoin(condition=[=($2, $3)], joinType=[left]) + LogicalProject(avg(balance)=[$2], gender=[$0], age=[SAFE_CAST($1)]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(gender=[$4], age=[$10], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalProject(age=[$0], __grand_total__=[$1], __row_number__=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) + LogicalAggregate(group=[{1}], __grand_total__=[AVG($0)]) + LogicalProject(avg(balance)=[$2], age=[SAFE_CAST($1)]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(gender=[$4], age=[$10], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + +physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(balance)=[$t8]) + EnumerableAggregate(group=[{1, 2}], agg#0=[$SUM0($0)], agg#1=[COUNT($0)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t2)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t2, $t9)], proj#0..1=[{exprs}], age=[$t10]) + EnumerableMergeJoin(condition=[=($2, $3)], joinType=[left]) + EnumerableSort(sort0=[$2], dir0=[ASC]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], expr#10=[SAFE_CAST($t1)], avg(balance)=[$t9], gender=[$t0], age=[$t10]) + EnumerableAggregate(group=[{4, 10}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) + EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:DOUBLE], expr#6=[CASE($t4, $t5, $t1)], expr#7=[/($t6, $t2)], age=[$t0], __grand_total__=[$t7]) + EnumerableAggregate(group=[{1}], agg#0=[$SUM0($0)], agg#1=[COUNT($0)]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], expr#10=[SAFE_CAST($t1)], avg(balance)=[$t9], age=[$t10]) + EnumerableAggregate(group=[{4, 10}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_null_str.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_null_str.yaml new file mode 100644 index 00000000000..0c34016836b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_null_str.yaml @@ -0,0 +1,37 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{1, 2}], avg(balance)=[AVG($0)]) + LogicalProject(avg(balance)=[$0], gender=[$1], age=[CASE(IS NULL($2), 'nil', <=($5, 10), $2, 'OTHER')]) + LogicalJoin(condition=[=($2, $3)], joinType=[left]) + LogicalProject(avg(balance)=[$2], gender=[$0], age=[SAFE_CAST($1)]) + LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) + LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) + LogicalProject(age=[$0], __grand_total__=[$1], __row_number__=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) + LogicalAggregate(group=[{1}], __grand_total__=[AVG($0)]) + LogicalProject(avg(balance)=[$2], age=[SAFE_CAST($1)]) + LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) + LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(balance)=[$t8]) + EnumerableAggregate(group=[{1, 2}], agg#0=[$SUM0($0)], agg#1=[COUNT($0)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t2)], expr#6=['nil'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t2, $t9)], proj#0..1=[{exprs}], age=[$t10]) + EnumerableMergeJoin(condition=[=($2, $3)], joinType=[left]) + EnumerableSort(sort0=[$2], dir0=[ASC]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], expr#10=[SAFE_CAST($t1)], avg(balance)=[$t9], gender=[$t0], age=[$t10]) + EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[10], expr#14=[null:NULL], expr#15=[SPAN($t5, $t13, $t14)], gender=[$t4], balance=[$t3], age0=[$t15]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) + EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:DOUBLE], expr#6=[CASE($t4, $t5, $t1)], expr#7=[/($t6, $t2)], age=[$t0], __grand_total__=[$t7]) + EnumerableAggregate(group=[{1}], agg#0=[$SUM0($0)], agg#1=[COUNT($0)]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], expr#10=[SAFE_CAST($t1)], avg(balance)=[$t9], age=[$t10]) + EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[10], expr#14=[null:NULL], expr#15=[SPAN($t5, $t13, $t14)], gender=[$t4], balance=[$t3], age0=[$t15]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) + diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_single_group.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_single_group.yaml new file mode 100644 index 00000000000..0cf28205b97 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_single_group.yaml @@ -0,0 +1,13 @@ +logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg(balance)=[$1], gender=[$0]) + LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)]) + LogicalProject(gender=[$4], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + +physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], avg(balance)=[$t8], gender=[$t0]) + EnumerableAggregate(group=[{4}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_single_group_key.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_single_group_key.yaml new file mode 100644 index 00000000000..b9e6ff9f735 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_single_group_key.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg(balance)=[$1], gender=[$0]) + LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)]) + LogicalProject(gender=[$4], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], avg(balance)=[$t8], gender=[$t0]) + EnumerableAggregate(group=[{4}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_timestamp_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_timestamp_span.yaml new file mode 100644 index 00000000000..913fd20b8bc --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_timestamp_span.yaml @@ -0,0 +1,32 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{1, 2}], max(value)=[MAX($0)]) + LogicalProject(max(value)=[$0], timestamp=[$1], category=[CASE(IS NULL($2), 'NULL', <=($5, 10), $2, 'OTHER')]) + LogicalJoin(condition=[=($2, $3)], joinType=[left]) + LogicalProject(max(value)=[$2], timestamp=[$1], category=[$0]) + LogicalAggregate(group=[{0, 2}], max(value)=[MAX($1)]) + LogicalProject(category=[$1], value=[$2], timestamp0=[SPAN($3, 1, 'w')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + LogicalProject(category=[$0], __grand_total__=[$1], __row_number__=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) + LogicalAggregate(group=[{1}], __grand_total__=[MAX($0)]) + LogicalProject(max(value)=[$2], category=[$0]) + LogicalAggregate(group=[{0, 2}], max(value)=[MAX($1)]) + LogicalProject(category=[$1], value=[$2], timestamp0=[SPAN($3, 1, 'w')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableAggregate(group=[{1, 2}], max(value)=[MAX($0)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t2)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t2, $t9)], proj#0..1=[{exprs}], category=[$t10]) + EnumerableMergeJoin(condition=[=($2, $3)], joinType=[left]) + EnumerableSort(sort0=[$2], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], max(value)=[$t2], timestamp=[$t1], category=[$t0]) + EnumerableAggregate(group=[{0, 2}], max(value)=[MAX($1)]) + EnumerableCalc(expr#0..9=[{inputs}], expr#10=[1], expr#11=['w'], expr#12=[SPAN($t3, $t10, $t11)], category=[$t1], value=[$t2], timestamp0=[$t12]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], category=[$t0], $1=[$t2]) + EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableAggregate(group=[{1}], __grand_total__=[MAX($2)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_use_other.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_use_other.yaml new file mode 100644 index 00000000000..022072f3ae7 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_use_other.yaml @@ -0,0 +1,30 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{1, 2}], max(severityNumber)=[MAX($0)]) + LogicalProject(max(severityNumber)=[$0], flags=[$1], severityText=[CASE(IS NULL($2), 'NULL', <=($5, 2), $2, 'max_among_other')]) + LogicalJoin(condition=[=($2, $3)], joinType=[left]) + LogicalProject(max(severityNumber)=[$2], flags=[$0], severityText=[$1]) + LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) + LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) + LogicalProject(severityText=[$0], __grand_total__=[$1], __row_number__=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) + LogicalAggregate(group=[{1}], __grand_total__=[MAX($0)]) + LogicalProject(max(severityNumber)=[$2], severityText=[$1]) + LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) + LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableAggregate(group=[{1, 2}], max(severityNumber)=[MAX($0)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t2)], expr#6=['NULL'], expr#7=[2], expr#8=[<=($t4, $t7)], expr#9=['max_among_other'], expr#10=[CASE($t5, $t6, $t8, $t2, $t9)], proj#0..1=[{exprs}], severityText=[$t10]) + EnumerableMergeJoin(condition=[=($2, $3)], joinType=[left]) + EnumerableSort(sort0=[$2], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], max(severityNumber)=[$t2], flags=[$t1], severityText=[$t0]) + EnumerableAggregate(group=[{7, 23}], max(severityNumber)=[MAX($163)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], severityText=[$t0], $1=[$t2]) + EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableAggregate(group=[{7}], __grand_total__=[MAX($163)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_limit.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_limit.yaml new file mode 100644 index 00000000000..e4ff0a172ce --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_limit.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg(balance)=[$2], state=[$0], gender=[$1]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(state=[$9], gender=[$4], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], avg(balance)=[$t9], state=[$t1], gender=[$t0]) + EnumerableAggregate(group=[{4, 9}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_span.yaml new file mode 100644 index 00000000000..6e8f8777170 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_span.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(max(balance)=[$1], age=[$0]) + LogicalAggregate(group=[{1}], max(balance)=[MAX($0)]) + LogicalProject(balance=[$7], age0=[SPAN($10, 10, null:NULL)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..1=[{inputs}], max(balance)=[$t1], age=[$t0]) + EnumerableAggregate(group=[{1}], max(balance)=[MAX($0)]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[10], expr#20=[null:NULL], expr#21=[SPAN($t10, $t19, $t20)], balance=[$t7], age0=[$t21]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + From 8f4d6d48c9a6d8353bebf0d5e7c9fca72c022032 Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Fri, 24 Oct 2025 14:50:11 +0800 Subject: [PATCH 16/35] Remove unimplemented support for multiple aggregations in chart command Signed-off-by: Yuanchun Shen --- .../org/opensearch/sql/ast/tree/Chart.java | 2 +- .../sql/calcite/CalciteRelNodeVisitor.java | 58 +++++++++++-------- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 4 +- .../opensearch/sql/ppl/parser/AstBuilder.java | 8 +-- .../sql/ppl/parser/AstBuilderTest.java | 26 ++------- .../sql/ppl/utils/ArgumentFactoryTest.java | 7 +-- 6 files changed, 46 insertions(+), 59 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/Chart.java b/core/src/main/java/org/opensearch/sql/ast/tree/Chart.java index 02e0878e12d..d0f982edce6 100644 --- a/core/src/main/java/org/opensearch/sql/ast/tree/Chart.java +++ b/core/src/main/java/org/opensearch/sql/ast/tree/Chart.java @@ -34,7 +34,7 @@ public class Chart extends UnresolvedPlan { private UnresolvedPlan child; private UnresolvedExpression rowSplit; private UnresolvedExpression columnSplit; - private List aggregationFunctions; + private UnresolvedExpression aggregationFunction; private List arguments; @Override diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 60138475fe9..4e31965d8ba 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -2030,30 +2030,31 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { ArgumentMap argMap = ArgumentMap.of(node.getArguments()); List groupExprList = Stream.of(node.getRowSplit(), node.getColumnSplit()).filter(Objects::nonNull).toList(); - Boolean useNull = (Boolean) argMap.getOrDefault("usenull", Chart.DEFAULT_USE_NULL).getValue(); + ChartConfig config = ChartConfig.fromArguments(argMap); Aggregation aggregation = new Aggregation( - node.getAggregationFunctions(), + List.of(node.getAggregationFunction()), List.of(), groupExprList, null, - List.of(new Argument(Argument.BUCKET_NULLABLE, AstDSL.booleanLiteral(useNull)))); + List.of(new Argument(Argument.BUCKET_NULLABLE, AstDSL.booleanLiteral(config.useNull)))); RelNode aggregated = visitAggregation(aggregation, context); // If row or column split does not present or limit equals 0, this is the same as `stats agg - // [group by col]` + // [group by col]` because all truncating is performed on the column split Integer limit = (Integer) argMap.getOrDefault("limit", Chart.DEFAULT_LIMIT).getValue(); if (node.getRowSplit() == null || node.getColumnSplit() == null || Objects.equals(limit, 0)) { return aggregated; } - String aggFunctionName = getAggFunctionName(node.getAggregationFunctions().getFirst()); - Optional aggFuncNameOptional = BuiltinFunctionName.of(aggFunctionName); - if (aggFuncNameOptional.isEmpty()) { - throw new IllegalArgumentException( - StringUtils.format("Unrecognized aggregation function: %s", aggFunctionName)); - } - BuiltinFunctionName aggFunction = aggFuncNameOptional.get(); + String aggFunctionName = getAggFunctionName(node.getAggregationFunction()); + BuiltinFunctionName aggFunction = + BuiltinFunctionName.of(aggFunctionName) + .orElseThrow( + () -> + new IllegalArgumentException( + StringUtils.format( + "Unrecognized aggregation function: %s", aggFunctionName))); // Convert the column split to string if necessary: column split was supposed to be pivoted to // column names. This guarantees that its type compatibility with useother and usenull @@ -2070,12 +2071,6 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { relBuilder.project(relBuilder.field(0), relBuilder.field(1), colSplit); aggregated = relBuilder.peek(); - Boolean top = (Boolean) argMap.getOrDefault("top", Chart.DEFAULT_TOP).getValue(); - Boolean useOther = - (Boolean) argMap.getOrDefault("useother", Chart.DEFAULT_USE_OTHER).getValue(); - String otherStr = (String) argMap.getOrDefault("otherstr", Chart.DEFAULT_OTHER_STR).getValue(); - String nullStr = (String) argMap.getOrDefault("nullstr", Chart.DEFAULT_NULL_STR).getValue(); - // 0: agg; 2: column-split relBuilder.project(relBuilder.field(0), relBuilder.field(2)); // 1: column split; 0: agg @@ -2087,7 +2082,7 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { // Apply sorting: for MIN/EARLIEST, reverse the top/bottom logic boolean smallestFirst = aggFunction == BuiltinFunctionName.MIN || aggFunction == BuiltinFunctionName.EARLIEST; - if (top != smallestFirst) { + if (config.top != smallestFirst) { grandTotal = relBuilder.desc(grandTotal); } @@ -2120,26 +2115,26 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { relBuilder.literal(limit)); RexNode nullCondition = relBuilder.isNull(colSplitPostJoin); RexNode columnSplitExpr; - if (!useOther) { + if (!config.useOther) { relBuilder.filter(lteCondition); } - if (useNull) { + if (config.useNull) { columnSplitExpr = relBuilder.call( SqlStdOperatorTable.CASE, nullCondition, - relBuilder.literal(nullStr), + relBuilder.literal(config.nullStr), lteCondition, relBuilder.field(2), - relBuilder.literal(otherStr)); + relBuilder.literal(config.otherStr)); } else { columnSplitExpr = relBuilder.call( SqlStdOperatorTable.CASE, lteCondition, relBuilder.field(2), - relBuilder.literal(otherStr)); + relBuilder.literal(config.otherStr)); } String aggFieldName = relBuilder.peek().getRowType().getFieldNames().getFirst(); @@ -2153,6 +2148,21 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { return relBuilder.peek(); } + private record ChartConfig( + int limit, boolean top, boolean useOther, boolean useNull, String otherStr, String nullStr) { + static ChartConfig fromArguments(ArgumentMap argMap) { + int limit = (Integer) argMap.getOrDefault("limit", Chart.DEFAULT_LIMIT).getValue(); + boolean top = (Boolean) argMap.getOrDefault("top", Chart.DEFAULT_TOP).getValue(); + boolean useOther = + (Boolean) argMap.getOrDefault("useother", Chart.DEFAULT_USE_OTHER).getValue(); + boolean useNull = (Boolean) argMap.getOrDefault("usenull", Chart.DEFAULT_USE_NULL).getValue(); + String otherStr = + (String) argMap.getOrDefault("otherstr", Chart.DEFAULT_OTHER_STR).getValue(); + String nullStr = (String) argMap.getOrDefault("nullstr", Chart.DEFAULT_NULL_STR).getValue(); + return new ChartConfig(limit, top, useOther, useNull, otherStr, nullStr); + } + } + /** Transforms timechart command into SQL-based operations. */ @Override public RelNode visitTimechart( @@ -2162,7 +2172,7 @@ public RelNode visitTimechart( // Extract parameters UnresolvedExpression spanExpr = node.getBinExpression(); - List groupExprList = Arrays.asList(spanExpr); + List groupExprList; // Handle no by field case if (node.getByField() == null) { diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 1623e02376b..a7d5aa674e8 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -260,8 +260,8 @@ reverseCommand ; chartCommand - : CHART chartOptions* statsAggTerm (COMMA statsAggTerm)* (OVER rowSplit)? (BY columnSplit)? - | CHART chartOptions* statsAggTerm (COMMA statsAggTerm)* BY rowSplit (COMMA)? columnSplit + : CHART chartOptions* statsAggTerm (OVER rowSplit)? (BY columnSplit)? + | CHART chartOptions* statsAggTerm BY rowSplit (COMMA)? columnSplit ; chartOptions diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index cc489b27355..4af902db53a 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -606,15 +606,11 @@ public UnresolvedPlan visitChartCommand(OpenSearchPPLParser.ChartCommandContext UnresolvedExpression columnSplit = ctx.columnSplit() == null ? null : internalVisitExpression(ctx.columnSplit()); List arguments = ArgumentFactory.getArgumentList(ctx); - List aggList = parseAggTerms(ctx.statsAggTerm()); - if (aggList.size() > 1) { - throw new IllegalArgumentException( - "Chart command does not support multiple aggregation functions yet"); - } + UnresolvedExpression aggFunction = parseAggTerms(List.of(ctx.statsAggTerm())).getFirst(); return Chart.builder() .rowSplit(rowSplit) .columnSplit(columnSplit) - .aggregationFunctions(aggList) + .aggregationFunction(aggFunction) .arguments(arguments) .build(); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index 74d50f6e3ba..621302e6531 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -1458,7 +1458,7 @@ public void testChartCommandBasic() { Chart.builder() .child(relation("t")) .columnSplit(alias("age", field("age"))) - .aggregationFunctions(List.of(alias("count()", aggregate("count", AllFields.of())))) + .aggregationFunction(alias("count()", aggregate("count", AllFields.of()))) .arguments(emptyList()) .build()); } @@ -1471,22 +1471,7 @@ public void testChartCommandWithRowSplit() { .child(relation("t")) .rowSplit(alias("status", field("status"))) .columnSplit(alias("age", field("age"))) - .aggregationFunctions(List.of(alias("count()", aggregate("count", AllFields.of())))) - .arguments(emptyList()) - .build()); - } - - @Test - public void testChartCommandWithMultipleAggregations() { - assertEqual( - "source=t | chart avg(salary), max(age) by department", - Chart.builder() - .child(relation("t")) - .columnSplit(alias("department", field("department"))) - .aggregationFunctions( - List.of( - alias("avg(salary)", aggregate("avg", field("salary"))), - alias("max(age)", aggregate("max", field("age"))))) + .aggregationFunction(alias("count()", aggregate("count", AllFields.of()))) .arguments(emptyList()) .build()); } @@ -1498,7 +1483,7 @@ public void testChartCommandWithOptions() { Chart.builder() .child(relation("t")) .columnSplit(alias("status", field("status"))) - .aggregationFunctions(List.of(alias("count()", aggregate("count", AllFields.of())))) + .aggregationFunction(alias("count()", aggregate("count", AllFields.of()))) .arguments( exprList( argument("limit", intLiteral(10)), @@ -1515,8 +1500,7 @@ public void testChartCommandWithAllOptions() { Chart.builder() .child(relation("t")) .columnSplit(alias("gender", field("gender"))) - .aggregationFunctions( - List.of(alias("avg(balance)", aggregate("avg", field("balance"))))) + .aggregationFunction(alias("avg(balance)", aggregate("avg", field("balance")))) .arguments( exprList( argument("limit", intLiteral(5)), @@ -1535,7 +1519,7 @@ public void testChartCommandWithBottomLimit() { Chart.builder() .child(relation("t")) .columnSplit(alias("category", field("category"))) - .aggregationFunctions(List.of(alias("count()", aggregate("count", AllFields.of())))) + .aggregationFunction(alias("count()", aggregate("count", AllFields.of()))) .arguments( exprList(argument("limit", intLiteral(3)), argument("top", booleanLiteral(false)))) .build()); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/ArgumentFactoryTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/ArgumentFactoryTest.java index e268656a8d9..f5b389146c7 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/ArgumentFactoryTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/ArgumentFactoryTest.java @@ -20,7 +20,6 @@ import static org.opensearch.sql.ast.dsl.AstDSL.sort; import static org.opensearch.sql.ast.dsl.AstDSL.stringLiteral; -import com.google.common.collect.ImmutableList; import org.junit.Test; import org.opensearch.sql.ast.expression.AllFields; import org.opensearch.sql.ast.expression.Argument; @@ -111,8 +110,7 @@ public void testChartCommandArguments() { Chart.builder() .child(relation("t")) .columnSplit(alias("age", field("age"))) - .aggregationFunctions( - ImmutableList.of(alias("count()", aggregate("count", AllFields.of())))) + .aggregationFunction(alias("count()", aggregate("count", AllFields.of()))) .arguments( exprList( argument("limit", intLiteral(5)), @@ -131,8 +129,7 @@ public void testChartCommandBottomArguments() { Chart.builder() .child(relation("t")) .columnSplit(alias("status", field("status"))) - .aggregationFunctions( - ImmutableList.of(alias("count()", aggregate("count", AllFields.of())))) + .aggregationFunction(alias("count()", aggregate("count", AllFields.of()))) .arguments( exprList(argument("limit", intLiteral(3)), argument("top", booleanLiteral(false)))) .build()); From b14a764f1e36de8d7814d6e2d6040d76b405468b Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Tue, 28 Oct 2025 20:03:10 +0800 Subject: [PATCH 17/35] Add unit tests for chart command Signed-off-by: Yuanchun Shen --- .../sql/ppl/calcite/CalcitePPLChartTest.java | 381 ++++++++++++++++++ 1 file changed, 381 insertions(+) create mode 100644 ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLChartTest.java diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLChartTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLChartTest.java new file mode 100644 index 00000000000..c49f0344900 --- /dev/null +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLChartTest.java @@ -0,0 +1,381 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl.calcite; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.RequiredArgsConstructor; +import org.apache.calcite.DataContext; +import org.apache.calcite.config.CalciteConnectionConfig; +import org.apache.calcite.linq4j.Enumerable; +import org.apache.calcite.linq4j.Linq4j; +import org.apache.calcite.plan.RelTraitDef; +import org.apache.calcite.rel.RelCollations; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rel.type.RelProtoDataType; +import org.apache.calcite.schema.ScannableTable; +import org.apache.calcite.schema.Schema; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.schema.Statistic; +import org.apache.calcite.schema.Statistics; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.parser.SqlParser; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.test.CalciteAssert; +import org.apache.calcite.tools.Frameworks; +import org.apache.calcite.tools.Programs; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.junit.Test; +import org.opensearch.sql.ast.tree.UnresolvedPlan; +import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; +import org.opensearch.sql.ppl.parser.AstBuilder; + +public class CalcitePPLChartTest extends CalcitePPLAbstractTest { + + public CalcitePPLChartTest() { + super(CalciteAssert.SchemaSpec.SCOTT_WITH_TEMPORAL); + } + + @Override + protected Frameworks.ConfigBuilder config(CalciteAssert.SchemaSpec... schemaSpecs) { + final SchemaPlus rootSchema = Frameworks.createRootSchema(true); + final SchemaPlus schema = CalciteAssert.addSchema(rootSchema, schemaSpecs); + // Add events table for chart tests - similar to bank data used in integration tests + ImmutableList rows = + ImmutableList.of( + new Object[] {32838, "F", 28, "VA", java.sql.Timestamp.valueOf("2024-07-01 00:00:00")}, + new Object[] {40540, "F", 39, "PA", java.sql.Timestamp.valueOf("2024-07-01 00:01:00")}, + new Object[] {39225, "M", 32, "IL", java.sql.Timestamp.valueOf("2024-07-01 00:02:00")}, + new Object[] {4180, "M", 33, "MD", java.sql.Timestamp.valueOf("2024-07-01 00:03:00")}, + new Object[] {11052, "M", 36, "WA", java.sql.Timestamp.valueOf("2024-07-01 00:04:00")}, + new Object[] {48086, "F", 34, "IN", java.sql.Timestamp.valueOf("2024-07-01 00:05:00")}); + schema.add("bank", new BankTable(rows)); + + // Add time_data table for span tests + ImmutableList timeRows = + ImmutableList.of( + new Object[] {java.sql.Timestamp.valueOf("2025-07-28 00:00:00"), "A", 9367}, + new Object[] {java.sql.Timestamp.valueOf("2025-07-29 00:00:00"), "B", 9521}, + new Object[] {java.sql.Timestamp.valueOf("2025-07-30 00:00:00"), "C", 9187}, + new Object[] {java.sql.Timestamp.valueOf("2025-07-31 00:00:00"), "D", 8736}, + new Object[] {java.sql.Timestamp.valueOf("2025-08-01 00:00:00"), "A", 9015}); + schema.add("time_data", new TimeDataTable(timeRows)); + + return Frameworks.newConfigBuilder() + .parserConfig(SqlParser.Config.DEFAULT) + .defaultSchema(schema) + .traitDefs((List) null) + .programs(Programs.heuristicJoinOrder(Programs.RULE_SET, true, 2)); + } + + @Test + public void testChartWithSingleGroupKey() { + String ppl = "source=bank | chart avg(balance) by gender"; + + RelNode root = getRelNode(ppl); + String expectedSparkSql = + "SELECT AVG(`balance`) `avg(balance)`, `gender`\n" + + "FROM `scott`.`bank`\n" + + "GROUP BY `gender`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testChartWithOverSyntax() { + String ppl = "source=bank | chart avg(balance) over gender"; + + RelNode root = getRelNode(ppl); + String expectedSparkSql = + "SELECT AVG(`balance`) `avg(balance)`, `gender`\n" + + "FROM `scott`.`bank`\n" + + "GROUP BY `gender`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testChartWithMultipleGroupKeys() { + String ppl = "source=bank | chart avg(balance) over gender by age"; + + RelNode root = getRelNode(ppl); + String expectedSparkSql = + "SELECT `t1`.`gender`, CASE WHEN `t1`.`age` IS NULL THEN 'NULL' WHEN `t6`.`__row_number__`" + + " <= 10 THEN `t1`.`age` ELSE 'OTHER' END `age`, AVG(`t1`.`avg(balance)`)" + + " `avg(balance)`\n" + + "FROM (SELECT AVG(`balance`) `avg(balance)`, `gender`, SAFE_CAST(`age` AS STRING)" + + " `age`\n" + + "FROM `scott`.`bank`\n" + + "GROUP BY `gender`, `age`) `t1`\n" + + "LEFT JOIN (SELECT `age`, AVG(`avg(balance)`) `__grand_total__`, ROW_NUMBER() OVER" + + " (ORDER BY AVG(`avg(balance)`) DESC) `__row_number__`\n" + + "FROM (SELECT AVG(`balance`) `avg(balance)`, SAFE_CAST(`age` AS STRING) `age`\n" + + "FROM `scott`.`bank`\n" + + "GROUP BY `gender`, `age`) `t4`\n" + + "GROUP BY `age`) `t6` ON `t1`.`age` = `t6`.`age`\n" + + "GROUP BY `t1`.`gender`, CASE WHEN `t1`.`age` IS NULL THEN 'NULL' WHEN" + + " `t6`.`__row_number__` <= 10 THEN `t1`.`age` ELSE 'OTHER' END"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testChartWithMultipleGroupKeysAlternativeSyntax() { + String ppl = "source=bank | chart avg(balance) by gender, age"; + + RelNode root = getRelNode(ppl); + String expectedSparkSql = + "SELECT `t1`.`gender`, CASE WHEN `t1`.`age` IS NULL THEN 'NULL' WHEN `t6`.`__row_number__`" + + " <= 10 THEN `t1`.`age` ELSE 'OTHER' END `age`, AVG(`t1`.`avg(balance)`)" + + " `avg(balance)`\n" + + "FROM (SELECT AVG(`balance`) `avg(balance)`, `gender`, SAFE_CAST(`age` AS STRING)" + + " `age`\n" + + "FROM `scott`.`bank`\n" + + "GROUP BY `gender`, `age`) `t1`\n" + + "LEFT JOIN (SELECT `age`, AVG(`avg(balance)`) `__grand_total__`, ROW_NUMBER() OVER" + + " (ORDER BY AVG(`avg(balance)`) DESC) `__row_number__`\n" + + "FROM (SELECT AVG(`balance`) `avg(balance)`, SAFE_CAST(`age` AS STRING) `age`\n" + + "FROM `scott`.`bank`\n" + + "GROUP BY `gender`, `age`) `t4`\n" + + "GROUP BY `age`) `t6` ON `t1`.`age` = `t6`.`age`\n" + + "GROUP BY `t1`.`gender`, CASE WHEN `t1`.`age` IS NULL THEN 'NULL' WHEN" + + " `t6`.`__row_number__` <= 10 THEN `t1`.`age` ELSE 'OTHER' END"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testChartWithLimit() { + String ppl = "source=bank | chart limit=2 avg(balance) by gender"; + + RelNode root = getRelNode(ppl); + String expectedSparkSql = + "SELECT AVG(`balance`) `avg(balance)`, `gender`\n" + + "FROM `scott`.`bank`\n" + + "GROUP BY `gender`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testChartWithLimitZero() { + String ppl = "source=bank | chart limit=0 avg(balance) over state by gender"; + + RelNode root = getRelNode(ppl); + String expectedSparkSql = + "SELECT AVG(`balance`) `avg(balance)`, `state`, `gender`\n" + + "FROM `scott`.`bank`\n" + + "GROUP BY `state`, `gender`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testChartWithSpan() { + String ppl = "source=bank | chart max(balance) by age span=10"; + + RelNode root = getRelNode(ppl); + String expectedSparkSql = + "SELECT MAX(`balance`) `max(balance)`, `SPAN`(`age`, 10, NULL) `age`\n" + + "FROM `scott`.`bank`\n" + + "GROUP BY `SPAN`(`age`, 10, NULL)"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testChartWithTimeSpan() { + String ppl = "source=time_data | chart max(value) over timestamp span=1week by category"; + + RelNode root = getRelNode(ppl); + String expectedSparkSql = + "SELECT `t1`.`timestamp`, CASE WHEN `t1`.`category` IS NULL THEN 'NULL' WHEN" + + " `t6`.`__row_number__` <= 10 THEN `t1`.`category` ELSE 'OTHER' END `category`," + + " MAX(`t1`.`max(value)`) `max(value)`\n" + + "FROM (SELECT MAX(`value`) `max(value)`, `SPAN`(`timestamp`, 1, 'w') `timestamp`," + + " `category`\n" + + "FROM `scott`.`time_data`\n" + + "GROUP BY `category`, `SPAN`(`timestamp`, 1, 'w')) `t1`\n" + + "LEFT JOIN (SELECT `category`, MAX(`max(value)`) `__grand_total__`, ROW_NUMBER() OVER" + + " (ORDER BY MAX(`max(value)`) DESC) `__row_number__`\n" + + "FROM (SELECT MAX(`value`) `max(value)`, `category`\n" + + "FROM `scott`.`time_data`\n" + + "GROUP BY `category`, `SPAN`(`timestamp`, 1, 'w')) `t4`\n" + + "GROUP BY `category`) `t6` ON `t1`.`category` = `t6`.`category`\n" + + "GROUP BY `t1`.`timestamp`, CASE WHEN `t1`.`category` IS NULL THEN 'NULL' WHEN" + + " `t6`.`__row_number__` <= 10 THEN `t1`.`category` ELSE 'OTHER' END"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testChartWithUseOtherTrue() { + String ppl = "source=bank | chart useother=true avg(balance) by gender"; + + RelNode root = getRelNode(ppl); + String expectedSparkSql = + "SELECT AVG(`balance`) `avg(balance)`, `gender`\n" + + "FROM `scott`.`bank`\n" + + "GROUP BY `gender`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testChartWithUseOtherFalse() { + String ppl = "source=bank | chart useother=false limit=2 avg(balance) by gender"; + + RelNode root = getRelNode(ppl); + String expectedSparkSql = + "SELECT AVG(`balance`) `avg(balance)`, `gender`\n" + + "FROM `scott`.`bank`\n" + + "GROUP BY `gender`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testChartWithOtherStr() { + String ppl = "source=bank | chart limit=1 otherstr='other_values' avg(balance) by gender"; + + RelNode root = getRelNode(ppl); + String expectedSparkSql = + "SELECT AVG(`balance`) `avg(balance)`, `gender`\n" + + "FROM `scott`.`bank`\n" + + "GROUP BY `gender`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testChartWithNullStr() { + String ppl = "source=bank | chart nullstr='null_values' avg(balance) by gender"; + + RelNode root = getRelNode(ppl); + String expectedSparkSql = + "SELECT AVG(`balance`) `avg(balance)`, `gender`\n" + + "FROM `scott`.`bank`\n" + + "GROUP BY `gender`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testChartWithUseNull() { + String ppl = "source=bank | chart usenull=false avg(balance) by gender"; + + RelNode root = getRelNode(ppl); + String expectedSparkSql = + "SELECT AVG(`balance`) `avg(balance)`, `gender`\n" + + "FROM `scott`.`bank`\n" + + "WHERE `gender` IS NOT NULL\n" + + "GROUP BY `gender`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + private UnresolvedPlan parsePPL(String query) { + PPLSyntaxParser parser = new PPLSyntaxParser(); + AstBuilder astBuilder = new AstBuilder(query); + return astBuilder.visit(parser.parse(query)); + } + + @RequiredArgsConstructor + public static class BankTable implements ScannableTable { + private final ImmutableList rows; + + protected final RelProtoDataType protoRowType = + factory -> + factory + .builder() + .add("balance", SqlTypeName.INTEGER) + .nullable(true) + .add("gender", SqlTypeName.VARCHAR) + .nullable(true) + .add("age", SqlTypeName.INTEGER) + .nullable(true) + .add("state", SqlTypeName.VARCHAR) + .nullable(true) + .add("timestamp", SqlTypeName.TIMESTAMP) + .nullable(true) + .build(); + + @Override + public Enumerable<@Nullable Object[]> scan(DataContext root) { + return Linq4j.asEnumerable(rows); + } + + @Override + public RelDataType getRowType(RelDataTypeFactory typeFactory) { + return protoRowType.apply(typeFactory); + } + + @Override + public Statistic getStatistic() { + return Statistics.of(0d, ImmutableList.of(), RelCollations.createSingleton(0)); + } + + @Override + public Schema.TableType getJdbcTableType() { + return Schema.TableType.TABLE; + } + + @Override + public boolean isRolledUp(String column) { + return false; + } + + @Override + public boolean rolledUpColumnValidInsideAgg( + String column, + SqlCall call, + @Nullable SqlNode parent, + @Nullable CalciteConnectionConfig config) { + return false; + } + } + + @RequiredArgsConstructor + public static class TimeDataTable implements ScannableTable { + private final ImmutableList rows; + + protected final RelProtoDataType protoRowType = + factory -> + factory + .builder() + .add("timestamp", SqlTypeName.TIMESTAMP) + .nullable(true) + .add("category", SqlTypeName.VARCHAR) + .nullable(true) + .add("value", SqlTypeName.INTEGER) + .nullable(true) + .build(); + + @Override + public Enumerable<@Nullable Object[]> scan(DataContext root) { + return Linq4j.asEnumerable(rows); + } + + @Override + public RelDataType getRowType(RelDataTypeFactory typeFactory) { + return protoRowType.apply(typeFactory); + } + + @Override + public Statistic getStatistic() { + return Statistics.of(0d, ImmutableList.of(), RelCollations.createSingleton(0)); + } + + @Override + public Schema.TableType getJdbcTableType() { + return Schema.TableType.TABLE; + } + + @Override + public boolean isRolledUp(String column) { + return false; + } + + @Override + public boolean rolledUpColumnValidInsideAgg( + String column, + SqlCall call, + @Nullable SqlNode parent, + @Nullable CalciteConnectionConfig config) { + return false; + } + } +} From 7126c82dd4afaf27bce0a4f6583530ab58c0d47f Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Wed, 29 Oct 2025 09:55:18 +0800 Subject: [PATCH 18/35] Remove irrelevant yaml test Signed-off-by: Yuanchun Shen --- .../rest-api-spec/test/issues/4582.yml | 120 ------------------ 1 file changed, 120 deletions(-) delete mode 100644 integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4582.yml diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4582.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4582.yml deleted file mode 100644 index 27973484d6c..00000000000 --- a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4582.yml +++ /dev/null @@ -1,120 +0,0 @@ -setup: - - do: - query.settings: - body: - transient: - plugins.calcite.enabled : true - - do: - indices.create: - index: test_timechart_4582 - body: - mappings: - properties: - "@timestamp": - type: date_nanos - severityNumber: - type: long - severityText: - type: keyword - body: - type: text - - do: - bulk: - index: test_timechart_4582 - refresh: true - body: - - '{"index": {}}' - - '{"@timestamp": "2024-01-15T10:30:04.567890123Z", "severityNumber": 9, "severityText": "INFO", "body": "Info message"}' - - '{"index": {}}' - - '{"@timestamp": "2024-01-15T10:30:05.567890123Z", "severityNumber": 13, "severityText": "WARN", "body": "Warning message"}' - - '{"index": {}}' - - '{"@timestamp": "2024-01-15T10:30:06.567890123Z", "severityNumber": 17, "severityText": "ERROR", "body": "Error message"}' - - '{"index": {}}' - - '{"@timestamp": "2024-01-15T10:30:07.567890123Z", "severityNumber": 21, "severityText": "FATAL", "body": "Fatal message"}' - - '{"index": {}}' - - '{"@timestamp": "2024-01-15T10:30:08.567890123Z", "severityNumber": 24, "severityText": "FATAL4", "body": "Fatal4 message"}' - - '{"index": {}}' - - '{"@timestamp": "2024-01-15T10:30:09.567890123Z", "severityNumber": 23, "severityText": "DEBUG", "body": "Debug message"}' - - '{"index": {}}' - - '{"@timestamp": "2024-01-15T10:30:10.567890123Z", "severityNumber": 20, "severityText": "TRACE", "body": "Trace message"}' - - '{"index": {}}' - - '{"@timestamp": "2024-01-15T10:30:11.567890123Z", "severityNumber": 22, "severityText": "CUSTOM", "body": "Custom message"}' - ---- -teardown: - - do: - query.settings: - body: - transient: - plugins.calcite.enabled : false - ---- -"timechart max aggregation with limit should not sum OTHER values": - - skip: - features: - - headers - - allowed_warnings - - do: - headers: - Content-Type: 'application/json' - ppl: - body: - query: source=test_timechart_4582 | timechart limit=1 span=10seconds max(severityNumber) by severityText - - - match: { total: 3 } - - match: { "schema": [{"name": "@timestamp", "type": "timestamp"}, {"name": "severityText", "type": "string"}, {"name": "max(severityNumber)", "type": "bigint"}] } - - match: { "datarows": [["2024-01-15 10:30:00", "FATAL4", 24], ["2024-01-15 10:30:00", "OTHER", 23], ["2024-01-15 10:30:10", "OTHER",22]] } - ---- -"timechart min aggregation with limit should not sum OTHER values": - - skip: - features: - - headers - - allowed_warnings - - do: - headers: - Content-Type: 'application/json' - ppl: - body: - query: source=test_timechart_4582 | timechart limit=2 span=1d min(severityNumber) by severityText - - - match: { total: 3 } - - match: { "schema": [{"name": "@timestamp", "type": "timestamp"}, {"name": "severityText", "type": "string"}, {"name": "min(severityNumber)", "type": "bigint"}] } - - match: { "datarows": [["2024-01-15 00:00:00", "INFO", 9], ["2024-01-15 00:00:00", "OTHER", 17], ["2024-01-15 00:00:00", "WARN", 13]] } - ---- -"timechart earliest aggregation with limit should not sum OTHER values": - - skip: - features: - - headers - - allowed_warnings - - do: - headers: - Content-Type: 'application/json' - ppl: - body: - query: source=test_timechart_4582 | timechart limit=2 span=30seconds earliest(@timestamp) by severityText - - - match: { total: 3 } - - match: { "schema": [{"name": "@timestamp", "type": "timestamp"}, {"name": "severityText", "type": "string"}, {"name": "earliest(@timestamp)", "type": "timestamp"}] } - - match: { "datarows": [ - ["2024-01-15 10:30:00", "INFO", "2024-01-15 10:30:04.567890123"], - ["2024-01-15 10:30:00", "OTHER", "2024-01-15 10:30:06.567890123"], - ["2024-01-15 10:30:00", "WARN", "2024-01-15 10:30:05.567890123"]] } - ---- -"timechart count aggregation with limit should sum OTHER values": - - skip: - features: - - headers - - allowed_warnings - - do: - headers: - Content-Type: 'application/json' - ppl: - body: - query: source=test_timechart_4582 | timechart limit=3 span=1min count() by severityText - - - match: { total: 4 } - - match: { "schema": [{"name": "@timestamp", "type": "timestamp"}, {"name": "severityText", "type": "string"}, {"name": "count", "type": "bigint"}] } - - match: { "datarows": [["2024-01-15 10:30:00", "CUSTOM", 1], ["2024-01-15 10:30:00", "DEBUG", 1], ["2024-01-15 10:30:00", "ERROR", 1], ["2024-01-15 10:30:00", "OTHER", 5]] } From 7d294c7569fa5a496096f70dbb85889b0fc48560 Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Wed, 29 Oct 2025 15:50:34 +0800 Subject: [PATCH 19/35] Tweak chart.rst Signed-off-by: Yuanchun Shen --- docs/user/ppl/cmd/chart.rst | 114 ++++++++++++++++++++---------------- 1 file changed, 64 insertions(+), 50 deletions(-) diff --git a/docs/user/ppl/cmd/chart.rst b/docs/user/ppl/cmd/chart.rst index 1633302676a..ed5058f4f38 100644 --- a/docs/user/ppl/cmd/chart.rst +++ b/docs/user/ppl/cmd/chart.rst @@ -1,6 +1,6 @@ -============= +===== chart -============= +===== .. rubric:: Table of contents @@ -10,7 +10,7 @@ chart Description -============ +=========== The ``chart`` command transforms search results by applying a statistical aggregation function and optionally grouping the data by one or two fields. The results are suitable for visualization as a two-dimension chart when grouping by two fields, where unique values in the second group key can be pivoted to column names. @@ -19,7 +19,7 @@ Version 3.4.0 Syntax -============ +====== .. code-block:: text @@ -34,16 +34,16 @@ Syntax * Default: 10 * Syntax: ``limit=(top|bottom) `` or ``limit=`` (defaults to top) - * When there are more distinct values than the limit, the additional values are grouped into an "OTHER" category if useother is not set to false. + * When there are more distinct column split values than the limit, the additional values are grouped into an "OTHER" category if ``useother`` is not set to false. * Set to 0 to show all distinct values without any limit. - * Only applies when using column split (over...by clause). + * Only applies when column split presents (by 2 fields or over...by... coexists). -* **useother**: optional. Controls whether to create an "OTHER" category for values beyond the limit. +* **useother**: optional. Controls whether to create an "OTHER" category for distinct column values beyond the limit. * Default: true - * When set to false, only the top/bottom N values (based on limit) are shown without an "OTHER" category. - * When set to true, values beyond the limit are grouped into an "OTHER" category. - * Only applies when using column split and when there are more distinct values than the limit. + * When set to false, only the top/bottom N distinct values (based on limit) are shown without an "OTHER" category. + * When set to true, distinct values beyond the limit are grouped into an "OTHER" category. + * Only applies when using column split and when there are more distinct column values than the limit. * **usenull**: optional. Controls whether to include null values as a separate category. @@ -51,45 +51,42 @@ Syntax * When set to false, events with null values in the split-by field are excluded from results. * When set to true, null values appear as a separate category. -* **nullstr**: optional. Specifies the string to display for null values. +* **nullstr**: optional. Specifies the category name for rows that do not contain the column split value. * Default: "NULL" - * Only applies when usenull is set to true. + * Only applies when ``usenull`` is set to true. -* **otherstr**: optional. Specifies the string to display for the "OTHER" category. +* **otherstr**: optional. Specifies the category name for the "OTHER" category. * Default: "OTHER" - * Only applies when useother is set to true and there are values beyond the limit. + * Only applies when ``useother`` is set to true and there are values beyond the limit. * **aggregation_function**: mandatory. The aggregation function to apply to the data. * Currently, only a single aggregation function is supported. - * Available functions: All aggregation functions supported by the :doc:`stats ` command. + * Available functions: aggregation functions supported by the `stats `_ command. -* **by**: optional. Groups the results by the specified field as rows. +* **by**: optional. Groups the results by either one field (row split) or two fields (row split and column split) + * ``limit``, ``useother``, and ``usenull`` apply to the column split + * Results are returned as individual rows for each combination. * If not specified, the aggregation is performed across all documents. -* **over...by**: optional. Alternative syntax for grouping by multiple fields. +* **over...by...**: optional. Alternative syntax for grouping by multiple fields. * ``over by `` groups the results by both fields. - * The row_split field becomes the primary grouping dimension. - * The column_split field becomes the secondary grouping dimension. - * Results are returned as individual rows for each combination. + * Using ``over`` alone on one field is equivalent to ``by `` Notes ===== -* The ``chart`` command transforms results into a table format suitable for visualization. -* When using multiple grouping fields (over...by syntax), the output contains individual rows for each combination of the grouping fields. -* The limit parameter determines how many columns to show when there are many distinct values. -* Results are ordered by the aggregated values to determine top/bottom selections. +* The column split field in the result will become strings so that they are compatible with ``nullstr`` and ``otherstr`` and can be used as column names once pivoted. Examples ======== Example 1: Basic aggregation without grouping -============================================== +--------------------------------------------- This example calculates the average balance across all accounts. @@ -104,7 +101,7 @@ PPL query:: +--------------+ Example 2: Group by single field -================================= +-------------------------------- This example calculates the count of accounts grouped by gender. @@ -120,7 +117,7 @@ PPL query:: +---------+--------+ Example 3: Using over and by for multiple field grouping -======================================================== +-------------------------------------------------------- This example shows average balance grouped by both gender and age fields. @@ -138,7 +135,7 @@ PPL query:: +--------+-----+--------------+ Example 4: Using basic limit functionality -======================================== +------------------------------------------ This example limits the results to show only the top 1 age group. @@ -155,40 +152,57 @@ PPL query:: +--------+-------+---------+ Example 5: Using limit with other parameters -============================================= +-------------------------------------------- This example shows using limit with useother and custom otherstr parameters. PPL query:: - os> source=accounts | chart limit=top 2 useother=true otherstr='remaining_accounts' max(balance) over state by gender + os> source=accounts | chart limit=top 1 useother=true otherstr='minor_gender' count() over state by gender + fetched rows / total rows = 4/4 + +-------+--------------+---------+ + | state | gender | count() | + |-------+--------------+---------| + | TN | M | 1 | + | MD | M | 1 | + | VA | minor_gender | 1 | + | IL | M | 1 | + +-------+--------------+---------+ + +Example 6: Using null parameters +-------------------------------- + +This example shows using limit with usenull and custom nullstr parameters. + +PPL query:: + + os> source=accounts | chart usenull=true nullstr='employer not specified' count() over firstname by employer fetched rows / total rows = 4/4 - +-------+--------+--------------+ - | state | gender | max(balance) | - |-------+--------+--------------| - | TN | M | 5686 | - | MD | M | 4180 | - | IL | M | 39225 | - | VA | F | 32838 | - +-------+--------+--------------+ - -Example 6: Using span with chart command -======================================= + +-----------+------------------------+---------+ + | firstname | employer | count() | + |-----------+------------------------+---------| + | Nanette | Quility | 1 | + | Amber | Pyrami | 1 | + | Dale | employer not specified | 1 | + | Hattie | Netagy | 1 | + +-----------+------------------------+---------+ + +Example 7: Using chart command with span +---------------------------------------- This example demonstrates using span for grouping age ranges. PPL query:: - os> source=accounts | chart max(balance) by age span=10 + os> source=accounts | chart max(balance) by age span=10, gender fetched rows / total rows = 2/2 - +--------------+-----+ - | max(balance) | age | - |--------------+-----| - | 32838 | 20 | - | 39225 | 30 | - +--------------+-----+ + +-----+--------+--------------+ + | age | gender | max(balance) | + |-----+--------+--------------| + | 30 | M | 39225 | + | 20 | F | 32838 | + +-----+--------+--------------+ Limitations -============ +=========== * Only a single aggregation function is supported per chart command. -* When using both row and column splits, the column split field is converted to string type so that it can be used as column names. \ No newline at end of file From 9bfb577ceb2baf6dc5d8f7670e1c022039c99d14 Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Wed, 29 Oct 2025 16:52:59 +0800 Subject: [PATCH 20/35] Swap the order of chart output to ensure metrics come last Signed-off-by: Yuanchun Shen --- .../sql/calcite/CalciteRelNodeVisitor.java | 72 ++++++++++------ docs/user/ppl/cmd/chart.rst | 12 +-- .../calcite/remote/CalciteChartCommandIT.java | 82 +++++++++---------- 3 files changed, 93 insertions(+), 73 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 4e31965d8ba..63af492b6f7 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -1108,6 +1108,19 @@ private Pair, List> resolveAttributesForAggregation( @Override public RelNode visitAggregation(Aggregation node, CalcitePlanContext context) { + visitAggregation(node, context, true); + return context.relBuilder.peek(); + } + + /** + * Visits an aggregation node and builds the corresponding Calcite RelNode. + * + * @param node the aggregation node containing group expressions and aggregation functions + * @param context the Calcite plan context for building RelNodes + * @param aggFirst if true, aggregation results (metrics) appear first in output schema (agg, + * group-by fields); if false, group expressions appear first (group-by fields, agg). + */ + private void visitAggregation(Aggregation node, CalcitePlanContext context, boolean aggFirst) { visitChildren(node, context); List aggExprList = node.getAggExprList(); @@ -1152,8 +1165,6 @@ public RelNode visitAggregation(Aggregation node, CalcitePlanContext context) { aggregateWithTrimming(groupExprList, aggExprList, context, toAddHintsOnAggregate); // schema reordering - // As an example, in command `stats count() by colA, colB`, - // the sequence of output schema is "count, colA, colB". List outputFields = context.relBuilder.fields(); int numOfOutputFields = outputFields.size(); int numOfAggList = aggExprList.size(); @@ -1161,8 +1172,6 @@ public RelNode visitAggregation(Aggregation node, CalcitePlanContext context) { // Add aggregation results first List aggRexList = outputFields.subList(numOfOutputFields - numOfAggList, numOfOutputFields); - reordered.addAll(aggRexList); - // Add group by columns List aliasedGroupByList = aggregationAttributes.getLeft().stream() .map(this::extractAliasLiteral) @@ -1171,10 +1180,17 @@ public RelNode visitAggregation(Aggregation node, CalcitePlanContext context) { .map(context.relBuilder::field) .map(f -> (RexNode) f) .toList(); - reordered.addAll(aliasedGroupByList); + if (aggFirst) { + // As an example, in command `stats count() by colA, colB`, + // the sequence of output schema is "count, colA, colB". + reordered.addAll(aggRexList); + // Add group by columns + reordered.addAll(aliasedGroupByList); + } else { + reordered.addAll(aliasedGroupByList); + reordered.addAll(aggRexList); + } context.relBuilder.project(reordered); - - return context.relBuilder.peek(); } private Optional getTimeSpanField(UnresolvedExpression expr) { @@ -2038,7 +2054,13 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { groupExprList, null, List.of(new Argument(Argument.BUCKET_NULLABLE, AstDSL.booleanLiteral(config.useNull)))); - RelNode aggregated = visitAggregation(aggregation, context); + visitAggregation(aggregation, context, false); + RelBuilder relBuilder = context.relBuilder; + String columnSplitName = + relBuilder.peek().getRowType().getFieldNames().size() > 2 + ? relBuilder.peek().getRowType().getFieldNames().get(1) + : null; + RelNode aggregated = context.relBuilder.peek(); // If row or column split does not present or limit equals 0, this is the same as `stats agg // [group by col]` because all truncating is performed on the column split @@ -2058,9 +2080,8 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { // Convert the column split to string if necessary: column split was supposed to be pivoted to // column names. This guarantees that its type compatibility with useother and usenull - RelBuilder relBuilder = context.relBuilder; - RexNode colSplit = relBuilder.field(2); - String columSplitName = relBuilder.peek().getRowType().getFieldNames().getLast(); + RexNode colSplit = relBuilder.field(1); + String columSplitName = relBuilder.peek().getRowType().getFieldNames().get(1); if (!SqlTypeUtil.isCharacter(colSplit.getType())) { colSplit = relBuilder.alias( @@ -2068,15 +2089,14 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { UserDefinedFunctionUtils.NULLABLE_STRING, colSplit, true, true), columSplitName); } - relBuilder.project(relBuilder.field(0), relBuilder.field(1), colSplit); + relBuilder.project(relBuilder.field(0), colSplit, relBuilder.field(2)); aggregated = relBuilder.peek(); - // 0: agg; 2: column-split - relBuilder.project(relBuilder.field(0), relBuilder.field(2)); - // 1: column split; 0: agg + // 1: column-split, 2: agg + relBuilder.project(relBuilder.field(1), relBuilder.field(2)); relBuilder.aggregate( - relBuilder.groupKey(relBuilder.field(1)), - buildAggCall(context.relBuilder, aggFunction, relBuilder.field(0)) + relBuilder.groupKey(relBuilder.field(0)), + buildAggCall(context.relBuilder, aggFunction, relBuilder.field(1)) .as("__grand_total__")); // results: group key, agg calls RexNode grandTotal = relBuilder.field("__grand_total__"); // Apply sorting: for MIN/EARLIEST, reverse the top/bottom logic @@ -2105,9 +2125,9 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { // on column-split = group key relBuilder.join( - JoinRelType.LEFT, relBuilder.equals(relBuilder.field(2, 0, 2), relBuilder.field(2, 1, 0))); + JoinRelType.LEFT, relBuilder.equals(relBuilder.field(2, 0, 1), relBuilder.field(2, 1, 0))); - RexNode colSplitPostJoin = relBuilder.field(2); + RexNode colSplitPostJoin = relBuilder.field(1); RexNode lteCondition = relBuilder.call( SqlStdOperatorTable.LESS_THAN_OR_EQUAL, @@ -2126,25 +2146,25 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { nullCondition, relBuilder.literal(config.nullStr), lteCondition, - relBuilder.field(2), + relBuilder.field(1), // col split relBuilder.literal(config.otherStr)); } else { columnSplitExpr = relBuilder.call( SqlStdOperatorTable.CASE, lteCondition, - relBuilder.field(2), + relBuilder.field(1), relBuilder.literal(config.otherStr)); } - String aggFieldName = relBuilder.peek().getRowType().getFieldNames().getFirst(); + String aggFieldName = relBuilder.peek().getRowType().getFieldNames().get(2); relBuilder.project( relBuilder.field(0), - relBuilder.field(1), - relBuilder.alias(columnSplitExpr, columSplitName)); + relBuilder.alias(columnSplitExpr, columnSplitName), + relBuilder.field(2)); relBuilder.aggregate( - relBuilder.groupKey(relBuilder.field(1), relBuilder.field(2)), - buildAggCall(context.relBuilder, aggFunction, relBuilder.field(0)).as(aggFieldName)); + relBuilder.groupKey(relBuilder.field(0), relBuilder.field(1)), + buildAggCall(context.relBuilder, aggFunction, relBuilder.field(2)).as(aggFieldName)); return relBuilder.peek(); } diff --git a/docs/user/ppl/cmd/chart.rst b/docs/user/ppl/cmd/chart.rst index ed5058f4f38..ed1cd2a453e 100644 --- a/docs/user/ppl/cmd/chart.rst +++ b/docs/user/ppl/cmd/chart.rst @@ -109,12 +109,12 @@ PPL query:: os> source=accounts | chart count() by gender fetched rows / total rows = 2/2 - +---------+--------+ - | count() | gender | - |---------+--------| - | 1 | F | - | 3 | M | - +---------+--------+ + +--------+---------+ + | gender | count() | + |--------+---------| + | F | 1 | + | M | 3 | + +--------+---------+ Example 3: Using over and by for multiple field grouping -------------------------------------------------------- diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java index ce6a63e3c24..310e8451b22 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java @@ -35,8 +35,8 @@ public void init() throws Exception { public void testChartWithSingleGroupKey() throws IOException { JSONObject result1 = executeQuery(String.format("source=%s | chart avg(balance) by gender", TEST_INDEX_BANK)); - verifySchema(result1, schema("avg(balance)", "double"), schema("gender", "string")); - verifyDataRows(result1, rows(40488, "F"), rows(16377.25, "M")); + verifySchema(result1, schema("gender", "string"), schema("avg(balance)", "double")); + verifyDataRows(result1, rows("F", 40488), rows("M", 16377.25)); JSONObject result2 = executeQuery(String.format("source=%s | chart avg(balance) over gender", TEST_INDEX_BANK)); assertJsonEquals(result1.toString(), result2.toString()); @@ -74,18 +74,18 @@ public void testChartCombineOverByWithLimit0() throws IOException { "source=%s | chart limit=0 avg(balance) over state by gender", TEST_INDEX_BANK)); verifySchema( result, - schema("avg(balance)", "double"), schema("state", "string"), - schema("gender", "string")); + schema("gender", "string"), + schema("avg(balance)", "double")); verifyDataRows( result, - rows(39225.0, "IL", "M"), - rows(48086.0, "IN", "F"), - rows(4180.0, "MD", "M"), - rows(40540.0, "PA", "F"), - rows(5686.0, "TN", "M"), - rows(32838.0, "VA", "F"), - rows(16418.0, "WA", "M")); + rows("IL", "M", 39225.0), + rows("IN", "F", 48086.0), + rows("MD", "M", 4180.0), + rows("PA", "F", 40540.0), + rows("TN", "M", 5686.0), + rows("VA", "F", 32838.0), + rows("WA", "M", 16418.0)); } @Test @@ -93,8 +93,8 @@ public void testChartMaxBalanceByAgeSpan() throws IOException { JSONObject result = executeQuery( String.format("source=%s | chart max(balance) by age span=10", TEST_INDEX_BANK)); - verifySchema(result, schema("max(balance)", "bigint"), schema("age", "int")); - verifyDataRows(result, rows(32838, 20), rows(48086, 30)); + verifySchema(result, schema("age", "int"), schema("max(balance)", "bigint")); + verifyDataRows(result, rows(20, 32838), rows(30, 48086)); } @Test @@ -172,37 +172,37 @@ public void testChartLimit0WithUseOther() throws IOException { TEST_INDEX_OTEL_LOGS)); verifySchema( result, - schema("max(severityNumber)", "bigint"), schema("flags", "bigint"), - schema("severityText", "string")); + schema("severityText", "string"), + schema("max(severityNumber)", "bigint")); verifyDataRows( result, - rows(5, 0, "DEBUG"), - rows(6, 0, "DEBUG2"), - rows(7, 0, "DEBUG3"), - rows(8, 0, "DEBUG4"), - rows(17, 0, "ERROR"), - rows(18, 0, "ERROR2"), - rows(19, 0, "ERROR3"), - rows(20, 0, "ERROR4"), - rows(21, 0, "FATAL"), - rows(22, 0, "FATAL2"), - rows(23, 0, "FATAL3"), - rows(24, 0, "FATAL4"), - rows(9, 0, "INFO"), - rows(10, 0, "INFO2"), - rows(11, 0, "INFO3"), - rows(12, 0, "INFO4"), - rows(2, 0, "TRACE2"), - rows(3, 0, "TRACE3"), - rows(4, 0, "TRACE4"), - rows(13, 0, "WARN"), - rows(14, 0, "WARN2"), - rows(15, 0, "WARN3"), - rows(16, 0, "WARN4"), - rows(17, 1, "ERROR"), - rows(9, 1, "INFO"), - rows(1, 1, "TRACE")); + rows(0, "DEBUG", 5), + rows(0, "DEBUG2", 6), + rows(0, "DEBUG3", 7), + rows(0, "DEBUG4", 8), + rows(0, "ERROR", 17), + rows(0, "ERROR2", 18), + rows(0, "ERROR3", 19), + rows(0, "ERROR4", 20), + rows(0, "FATAL", 21), + rows(0, "FATAL2", 22), + rows(0, "FATAL3", 23), + rows(0, "FATAL4", 24), + rows(0, "INFO", 9), + rows(0, "INFO2", 10), + rows(0, "INFO3", 11), + rows(0, "INFO4", 12), + rows(0, "TRACE2", 2), + rows(0, "TRACE3", 3), + rows(0, "TRACE4", 4), + rows(0, "WARN", 13), + rows(0, "WARN2", 14), + rows(0, "WARN3", 15), + rows(0, "WARN4", 16), + rows(1, "ERROR", 17), + rows(1, "INFO", 9), + rows(1, "TRACE", 1)); } @Test From 2c8d6324106489c7512f6816e3bb528f72848b45 Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Wed, 29 Oct 2025 19:38:37 +0800 Subject: [PATCH 21/35] Filter rows without col split when calculate grand total Signed-off-by: Yuanchun Shen --- .../sql/calcite/CalciteRelNodeVisitor.java | 12 +- .../sql/calcite/CalciteNoPushdownIT.java | 165 +++++++++--------- .../calcite/remote/CalciteChartCommandIT.java | 20 +++ .../explain_chart_multiple_group_keys.yaml | 35 ++-- .../calcite/explain_chart_null_str.yaml | 33 ++-- .../explain_chart_single_group_key.yaml | 9 +- .../calcite/explain_chart_timestamp_span.yaml | 29 +-- .../calcite/explain_chart_use_other.yaml | 34 ++-- .../calcite/explain_chart_with_limit.yaml | 9 +- .../calcite/explain_chart_with_span.yaml | 4 +- .../explain_chart_multiple_group_keys.yaml | 34 ++-- .../explain_chart_null_str.yaml | 34 ++-- .../explain_chart_single_group_key.yaml | 10 +- .../explain_chart_timestamp_span.yaml | 33 ++-- .../explain_chart_use_other.yaml | 40 +++-- .../explain_chart_with_limit.yaml | 10 +- .../explain_chart_with_span.yaml | 10 +- .../sql/ppl/calcite/CalcitePPLChartTest.java | 59 ++++--- 18 files changed, 303 insertions(+), 277 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 63af492b6f7..eb73fbceea2 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -2094,11 +2094,14 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { // 1: column-split, 2: agg relBuilder.project(relBuilder.field(1), relBuilder.field(2)); + // Make sure that rows who don't have a column split not interfere grand total calculation + relBuilder.filter(relBuilder.isNotNull(relBuilder.field(0))); + final String GRAND_TOTAL_COL = "__grand_total__"; relBuilder.aggregate( relBuilder.groupKey(relBuilder.field(0)), buildAggCall(context.relBuilder, aggFunction, relBuilder.field(1)) - .as("__grand_total__")); // results: group key, agg calls - RexNode grandTotal = relBuilder.field("__grand_total__"); + .as(GRAND_TOTAL_COL)); // results: group key, agg calls + RexNode grandTotal = relBuilder.field(GRAND_TOTAL_COL); // Apply sorting: for MIN/EARLIEST, reverse the top/bottom logic boolean smallestFirst = aggFunction == BuiltinFunctionName.MIN || aggFunction == BuiltinFunctionName.EARLIEST; @@ -2108,6 +2111,7 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { // Always set it to null last so that it does not interfere with top / bottom calculation grandTotal = relBuilder.nullsLast(grandTotal); + final String ROW_NUM_COL = "__row_number__"; RexNode rowNum = PlanUtils.makeOver( context, @@ -2117,7 +2121,7 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { List.of(), List.of(grandTotal), WindowFrame.toCurrentRow()); - relBuilder.projectPlus(relBuilder.alias(rowNum, "__row_number__")); + relBuilder.projectPlus(relBuilder.alias(rowNum, ROW_NUM_COL)); RelNode ranked = relBuilder.build(); relBuilder.push(aggregated); @@ -2131,7 +2135,7 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { RexNode lteCondition = relBuilder.call( SqlStdOperatorTable.LESS_THAN_OR_EQUAL, - relBuilder.field("__row_number__"), + relBuilder.field(ROW_NUM_COL), relBuilder.literal(limit)); RexNode nullCondition = relBuilder.isNull(colSplitPostJoin); RexNode columnSplitExpr; diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java index 14e2bfdb4da..69507c71aa5 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java @@ -10,6 +10,7 @@ import org.junit.runner.RunWith; import org.junit.runners.Suite; import org.opensearch.sql.calcite.remote.*; +import org.opensearch.sql.calcite.tpch.CalcitePPLTpchIT; import org.opensearch.sql.ppl.PPLIntegTestCase; /** @@ -20,88 +21,88 @@ @RunWith(Suite.class) @Suite.SuiteClasses({ CalciteExplainIT.class, - // CalciteArrayFunctionIT.class, - // CalciteBinCommandIT.class, - // CalciteConvertTZFunctionIT.class, - // CalciteCsvFormatIT.class, - // CalciteDataTypeIT.class, - // CalciteDateTimeComparisonIT.class, - // CalciteDateTimeFunctionIT.class, - // CalciteDateTimeImplementationIT.class, - // CalciteDedupCommandIT.class, - // CalciteDescribeCommandIT.class, - // CalciteExpandCommandIT.class, - // CalciteFieldsCommandIT.class, - // CalciteFillNullCommandIT.class, - // CalciteFlattenCommandIT.class, - // CalciteFlattenDocValueIT.class, - // CalciteGeoIpFunctionsIT.class, - // CalciteGeoPointFormatsIT.class, - // CalciteHeadCommandIT.class, - // CalciteInformationSchemaCommandIT.class, - // CalciteIPComparisonIT.class, - // CalciteIPFunctionsIT.class, - // CalciteJsonFunctionsIT.class, - // CalciteLegacyAPICompatibilityIT.class, - // CalciteLikeQueryIT.class, - // CalciteMathematicalFunctionIT.class, - // CalciteMultisearchCommandIT.class, - // CalciteMultiValueStatsIT.class, - // CalciteNewAddedCommandsIT.class, - // CalciteNowLikeFunctionIT.class, - // CalciteObjectFieldOperateIT.class, - // CalciteOperatorIT.class, - // CalciteParseCommandIT.class, - // CalcitePPLAggregationIT.class, - // CalcitePPLAppendcolIT.class, - // CalcitePPLAppendCommandIT.class, - // CalcitePPLBasicIT.class, - // CalcitePPLBuiltinDatetimeFunctionInvalidIT.class, - // CalcitePPLBuiltinFunctionIT.class, - // CalcitePPLBuiltinFunctionsNullIT.class, - // CalcitePPLCaseFunctionIT.class, - // CalcitePPLCastFunctionIT.class, - // CalcitePPLConditionBuiltinFunctionIT.class, - // CalcitePPLCryptographicFunctionIT.class, - // CalcitePPLDedupIT.class, - // CalcitePPLEventstatsIT.class, - // CalcitePPLExistsSubqueryIT.class, - // CalcitePPLExplainIT.class, - // CalcitePPLFillnullIT.class, - // CalcitePPLGrokIT.class, - // CalcitePPLInSubqueryIT.class, - // CalcitePPLIPFunctionIT.class, - // CalcitePPLJoinIT.class, - // CalcitePPLJsonBuiltinFunctionIT.class, - // CalcitePPLLookupIT.class, - // CalcitePPLParseIT.class, - // CalcitePPLPatternsIT.class, - // CalcitePPLPluginIT.class, - // CalcitePPLRenameIT.class, - // CalcitePPLScalarSubqueryIT.class, - // CalcitePPLSortIT.class, - // CalcitePPLStringBuiltinFunctionIT.class, - // CalcitePPLTrendlineIT.class, - // CalcitePrometheusDataSourceCommandsIT.class, - // CalciteQueryAnalysisIT.class, - // CalciteRareCommandIT.class, - // CalciteRegexCommandIT.class, - // CalciteRexCommandIT.class, - // CalciteRenameCommandIT.class, - // CalciteReplaceCommandIT.class, - // CalciteResourceMonitorIT.class, - // CalciteSearchCommandIT.class, - // CalciteSettingsIT.class, - // CalciteShowDataSourcesCommandIT.class, - // CalciteSortCommandIT.class, - // CalciteStatsCommandIT.class, - // CalciteSystemFunctionIT.class, - // CalciteTextFunctionIT.class, - // CalciteTopCommandIT.class, - // CalciteTrendlineCommandIT.class, - // CalciteVisualizationFormatIT.class, - // CalciteWhereCommandIT.class, - // CalcitePPLTpchIT.class + CalciteArrayFunctionIT.class, + CalciteBinCommandIT.class, + CalciteConvertTZFunctionIT.class, + CalciteCsvFormatIT.class, + CalciteDataTypeIT.class, + CalciteDateTimeComparisonIT.class, + CalciteDateTimeFunctionIT.class, + CalciteDateTimeImplementationIT.class, + CalciteDedupCommandIT.class, + CalciteDescribeCommandIT.class, + CalciteExpandCommandIT.class, + CalciteFieldsCommandIT.class, + CalciteFillNullCommandIT.class, + CalciteFlattenCommandIT.class, + CalciteFlattenDocValueIT.class, + CalciteGeoIpFunctionsIT.class, + CalciteGeoPointFormatsIT.class, + CalciteHeadCommandIT.class, + CalciteInformationSchemaCommandIT.class, + CalciteIPComparisonIT.class, + CalciteIPFunctionsIT.class, + CalciteJsonFunctionsIT.class, + CalciteLegacyAPICompatibilityIT.class, + CalciteLikeQueryIT.class, + CalciteMathematicalFunctionIT.class, + CalciteMultisearchCommandIT.class, + CalciteMultiValueStatsIT.class, + CalciteNewAddedCommandsIT.class, + CalciteNowLikeFunctionIT.class, + CalciteObjectFieldOperateIT.class, + CalciteOperatorIT.class, + CalciteParseCommandIT.class, + CalcitePPLAggregationIT.class, + CalcitePPLAppendcolIT.class, + CalcitePPLAppendCommandIT.class, + CalcitePPLBasicIT.class, + CalcitePPLBuiltinDatetimeFunctionInvalidIT.class, + CalcitePPLBuiltinFunctionIT.class, + CalcitePPLBuiltinFunctionsNullIT.class, + CalcitePPLCaseFunctionIT.class, + CalcitePPLCastFunctionIT.class, + CalcitePPLConditionBuiltinFunctionIT.class, + CalcitePPLCryptographicFunctionIT.class, + CalcitePPLDedupIT.class, + CalcitePPLEventstatsIT.class, + CalcitePPLExistsSubqueryIT.class, + CalcitePPLExplainIT.class, + CalcitePPLFillnullIT.class, + CalcitePPLGrokIT.class, + CalcitePPLInSubqueryIT.class, + CalcitePPLIPFunctionIT.class, + CalcitePPLJoinIT.class, + CalcitePPLJsonBuiltinFunctionIT.class, + CalcitePPLLookupIT.class, + CalcitePPLParseIT.class, + CalcitePPLPatternsIT.class, + CalcitePPLPluginIT.class, + CalcitePPLRenameIT.class, + CalcitePPLScalarSubqueryIT.class, + CalcitePPLSortIT.class, + CalcitePPLStringBuiltinFunctionIT.class, + CalcitePPLTrendlineIT.class, + CalcitePrometheusDataSourceCommandsIT.class, + CalciteQueryAnalysisIT.class, + CalciteRareCommandIT.class, + CalciteRegexCommandIT.class, + CalciteRexCommandIT.class, + CalciteRenameCommandIT.class, + CalciteReplaceCommandIT.class, + CalciteResourceMonitorIT.class, + CalciteSearchCommandIT.class, + CalciteSettingsIT.class, + CalciteShowDataSourcesCommandIT.class, + CalciteSortCommandIT.class, + CalciteStatsCommandIT.class, + CalciteSystemFunctionIT.class, + CalciteTextFunctionIT.class, + CalciteTopCommandIT.class, + CalciteTrendlineCommandIT.class, + CalciteVisualizationFormatIT.class, + CalciteWhereCommandIT.class, + CalcitePPLTpchIT.class }) public class CalciteNoPushdownIT { private static boolean wasPushdownEnabled; diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java index 310e8451b22..c25296d9cd8 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java @@ -29,6 +29,7 @@ public void init() throws Exception { loadIndex(Index.BANK_WITH_NULL_VALUES); loadIndex(Index.OTELLOGS); loadIndex(Index.TIME_TEST_DATA); + loadIndex(Index.EVENTS_NULL); } @Test @@ -282,6 +283,25 @@ public void testChartUseNullTrueWithNullStr() throws IOException { rows("F", "nil", null)); } + @Test + public void testChartWithNullAndLimit() throws IOException { + JSONObject result = + executeQuery("source=events_null | chart limit=3 count() over @timestamp span=1d by host"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("host", "string"), + schema("count()", "bigint")); + + verifyDataRows( + result, + rows("2024-07-01 00:00:00", "db-01", 1), + rows("2024-07-01 00:00:00", "web-01", 2), + rows("2024-07-01 00:00:00", "web-02", 2), + rows("2024-07-01 00:00:00", "NULL", 1)); + } + @Test public void testChartUseNullFalseWithNullStr() throws IOException { JSONObject result = diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_multiple_group_keys.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_multiple_group_keys.yaml index b4419f38e11..b41cebbda4e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_multiple_group_keys.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_multiple_group_keys.yaml @@ -1,32 +1,33 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalAggregate(group=[{1, 2}], avg(balance)=[AVG($0)]) - LogicalProject(avg(balance)=[$0], gender=[$1], age=[CASE(IS NULL($2), 'NULL', <=($5, 10), $2, 'OTHER')]) - LogicalJoin(condition=[=($2, $3)], joinType=[left]) - LogicalProject(avg(balance)=[$2], gender=[$0], age=[SAFE_CAST($1)]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(gender=[$0], age=[CASE(IS NULL($1), 'NULL', <=($5, 10), $1, 'OTHER')], avg(balance)=[$2]) + LogicalJoin(condition=[=($1, $3)], joinType=[left]) + LogicalProject(gender=[$0], age=[SAFE_CAST($1)], avg(balance)=[$2]) LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) LogicalProject(gender=[$4], age=[$10], balance=[$7]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) LogicalProject(age=[$0], __grand_total__=[$1], __row_number__=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) - LogicalAggregate(group=[{1}], __grand_total__=[AVG($0)]) - LogicalProject(avg(balance)=[$2], age=[SAFE_CAST($1)]) - LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) - LogicalProject(gender=[$4], age=[$10], balance=[$7]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalAggregate(group=[{0}], __grand_total__=[AVG($1)]) + LogicalFilter(condition=[IS NOT NULL($0)]) + LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(gender=[$4], age=[$10], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(balance)=[$t8]) - EnumerableAggregate(group=[{1, 2}], agg#0=[$SUM0($0)], agg#1=[COUNT($0)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t2)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t2, $t9)], proj#0..1=[{exprs}], age=[$t10]) - EnumerableMergeJoin(condition=[=($2, $3)], joinType=[left]) - EnumerableSort(sort0=[$2], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[SAFE_CAST($t1)], avg(balance)=[$t2], gender=[$t0], age=[$t3]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) + EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) + EnumerableSort(sort0=[$1], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[SAFE_CAST($t1)], gender=[$t0], age=[$t3], avg(balance)=[$t2]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:DOUBLE], expr#6=[CASE($t4, $t5, $t1)], expr#7=[/($t6, $t2)], age=[$t0], __grand_total__=[$t7]) - EnumerableAggregate(group=[{1}], agg#0=[$SUM0($0)], agg#1=[COUNT($0)]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[SAFE_CAST($t1)], avg(balance)=[$t0], $f1=[$t2]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2)), PROJECT->[avg(balance), age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[SAFE_CAST($t0)], expr#3=[IS NOT NULL($t2)], $f0=[$t2], avg(balance)=[$t1], $condition=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2)), PROJECT->[age, avg(balance)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_null_str.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_null_str.yaml index 6a3a024b2b3..3f60744f069 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_null_str.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_null_str.yaml @@ -1,27 +1,28 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalAggregate(group=[{1, 2}], avg(balance)=[AVG($0)]) - LogicalProject(avg(balance)=[$0], gender=[$1], age=[CASE(IS NULL($2), 'nil', <=($5, 10), $2, 'OTHER')]) - LogicalJoin(condition=[=($2, $3)], joinType=[left]) - LogicalProject(avg(balance)=[$2], gender=[$0], age=[SAFE_CAST($1)]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(gender=[$0], age=[CASE(IS NULL($1), 'nil', <=($5, 10), $1, 'OTHER')], avg(balance)=[$2]) + LogicalJoin(condition=[=($1, $3)], joinType=[left]) + LogicalProject(gender=[$0], age=[SAFE_CAST($1)], avg(balance)=[$2]) LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) LogicalProject(age=[$0], __grand_total__=[$1], __row_number__=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) - LogicalAggregate(group=[{1}], __grand_total__=[AVG($0)]) - LogicalProject(avg(balance)=[$2], age=[SAFE_CAST($1)]) - LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) - LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) + LogicalAggregate(group=[{0}], __grand_total__=[AVG($1)]) + LogicalFilter(condition=[IS NOT NULL($0)]) + LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) + LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) + LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(balance)=[$t8]) - EnumerableAggregate(group=[{1, 2}], agg#0=[$SUM0($0)], agg#1=[COUNT($0)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t2)], expr#6=['nil'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t2, $t9)], proj#0..1=[{exprs}], age=[$t10]) - EnumerableMergeJoin(condition=[=($2, $3)], joinType=[left]) - EnumerableSort(sort0=[$2], dir0=[ASC]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], expr#10=[SAFE_CAST($t1)], avg(balance)=[$t9], gender=[$t0], age=[$t10]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['nil'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) + EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) + EnumerableSort(sort0=[$1], dir0=[ASC]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t10]) EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], proj#0..1=[{exprs}], $f2=[$t5]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[gender, balance, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","balance","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) @@ -29,8 +30,8 @@ calcite: EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:DOUBLE], expr#6=[CASE($t4, $t5, $t1)], expr#7=[/($t6, $t2)], age=[$t0], __grand_total__=[$t7]) - EnumerableAggregate(group=[{1}], agg#0=[$SUM0($0)], agg#1=[COUNT($0)]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], expr#10=[SAFE_CAST($t1)], avg(balance)=[$t9], age=[$t10]) + EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], expr#11=[IS NOT NULL($t4)], age=[$t4], avg(balance)=[$t10], $condition=[$t11]) EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], proj#0..1=[{exprs}], $f2=[$t5]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[gender, balance, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","balance","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_single_group_key.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_single_group_key.yaml index 3752736138f..be22ae9c011 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_single_group_key.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_single_group_key.yaml @@ -1,9 +1,8 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(avg(balance)=[$1], gender=[$0]) - LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)]) - LogicalProject(gender=[$4], balance=[$7]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)]) + LogicalProject(gender=[$4], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg(balance)=AVG($1)), PROJECT->[avg(balance), gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg(balance)=AVG($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_timestamp_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_timestamp_span.yaml index 9007f4da716..7fb6a77dce2 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_timestamp_span.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_timestamp_span.yaml @@ -1,26 +1,27 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalAggregate(group=[{1, 2}], max(value)=[MAX($0)]) - LogicalProject(max(value)=[$0], timestamp=[$1], category=[CASE(IS NULL($2), 'NULL', <=($5, 10), $2, 'OTHER')]) - LogicalJoin(condition=[=($2, $3)], joinType=[left]) - LogicalProject(max(value)=[$2], timestamp=[$1], category=[$0]) + LogicalAggregate(group=[{0, 1}], max(value)=[MAX($2)]) + LogicalProject(timestamp=[$0], category=[CASE(IS NULL($1), 'NULL', <=($5, 10), $1, 'OTHER')], max(value)=[$2]) + LogicalJoin(condition=[=($1, $3)], joinType=[left]) + LogicalProject(timestamp=[$1], category=[$0], max(value)=[$2]) LogicalAggregate(group=[{0, 2}], max(value)=[MAX($1)]) LogicalProject(category=[$1], value=[$2], timestamp0=[SPAN($3, 1, 'w')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) LogicalProject(category=[$0], __grand_total__=[$1], __row_number__=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) - LogicalAggregate(group=[{1}], __grand_total__=[MAX($0)]) - LogicalProject(max(value)=[$2], category=[$0]) - LogicalAggregate(group=[{0, 2}], max(value)=[MAX($1)]) - LogicalProject(category=[$1], value=[$2], timestamp0=[SPAN($3, 1, 'w')]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + LogicalAggregate(group=[{0}], __grand_total__=[MAX($1)]) + LogicalFilter(condition=[IS NOT NULL($0)]) + LogicalProject(category=[$0], max(value)=[$2]) + LogicalAggregate(group=[{0, 2}], max(value)=[MAX($1)]) + LogicalProject(category=[$1], value=[$2], timestamp0=[SPAN($3, 1, 'w')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableAggregate(group=[{1, 2}], max(value)=[MAX($0)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t2)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t2, $t9)], proj#0..1=[{exprs}], category=[$t10]) - EnumerableMergeJoin(condition=[=($2, $3)], joinType=[left]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},max(value)=MAX($1)), PROJECT->[max(value), timestamp0, category], SORT->[2]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"timestamp0":{"date_histogram":{"field":"timestamp","missing_bucket":false,"order":"asc","calendar_interval":"1w"}}}]},"aggregations":{"max(value)":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableAggregate(group=[{0, 1}], max(value)=[MAX($2)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], timestamp=[$t0], category=[$t10], max(value)=[$t2]) + EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},max(value)=MAX($1)), PROJECT->[timestamp0, category, max(value)], SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"timestamp0":{"date_histogram":{"field":"timestamp","missing_bucket":false,"order":"asc","calendar_interval":"1w"}}}]},"aggregations":{"max(value)":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], category=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},__grand_total__=MAX($1))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"__grand_total__":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[FILTER->IS NOT NULL($1), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},__grand_total__=MAX($1))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"category","boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"__grand_total__":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_use_other.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_use_other.yaml index d6e799c3c36..ffb167e148e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_use_other.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_use_other.yaml @@ -1,26 +1,26 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalAggregate(group=[{1, 2}], max(severityNumber)=[MAX($0)]) - LogicalProject(max(severityNumber)=[$0], flags=[$1], severityText=[CASE(IS NULL($2), 'NULL', <=($5, 2), $2, 'max_among_other')]) - LogicalJoin(condition=[=($2, $3)], joinType=[left]) - LogicalProject(max(severityNumber)=[$2], flags=[$0], severityText=[$1]) - LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) - LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) + LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) + LogicalProject(flags=[$0], severityText=[CASE(IS NULL($1), 'NULL', <=($5, 2), $1, 'max_among_other')], max(severityNumber)=[$2]) + LogicalJoin(condition=[=($1, $3)], joinType=[left]) + LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) + LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) LogicalProject(severityText=[$0], __grand_total__=[$1], __row_number__=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) - LogicalAggregate(group=[{1}], __grand_total__=[MAX($0)]) - LogicalProject(max(severityNumber)=[$2], severityText=[$1]) - LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) - LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) + LogicalAggregate(group=[{0}], __grand_total__=[MAX($1)]) + LogicalFilter(condition=[IS NOT NULL($0)]) + LogicalProject(severityText=[$1], max(severityNumber)=[$2]) + LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) + LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableAggregate(group=[{1, 2}], max(severityNumber)=[MAX($0)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t2)], expr#6=['NULL'], expr#7=[2], expr#8=[<=($t4, $t7)], expr#9=['max_among_other'], expr#10=[CASE($t5, $t6, $t8, $t2, $t9)], proj#0..1=[{exprs}], severityText=[$t10]) - EnumerableMergeJoin(condition=[=($2, $3)], joinType=[left]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},max(severityNumber)=MAX($2)), PROJECT->[max(severityNumber), flags, severityText], SORT->[2]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"severityText":{"terms":{"field":"severityText","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"flags":{"terms":{"field":"flags","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"max(severityNumber)":{"max":{"field":"severityNumber"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[2], expr#8=[<=($t4, $t7)], expr#9=['max_among_other'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], flags=[$t0], severityText=[$t10], max(severityNumber)=[$t2]) + EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},max(severityNumber)=MAX($2)), SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"severityText":{"terms":{"field":"severityText","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"flags":{"terms":{"field":"flags","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"max(severityNumber)":{"max":{"field":"severityNumber"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], severityText=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},__grand_total__=MAX($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"severityText":{"terms":{"field":"severityText","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"__grand_total__":{"max":{"field":"severityNumber"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[FILTER->IS NOT NULL($7), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},__grand_total__=MAX($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"severityText","boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"severityText":{"terms":{"field":"severityText","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"__grand_total__":{"max":{"field":"severityNumber"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_limit.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_limit.yaml index 3077f16152c..4c97f6b8506 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_limit.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_limit.yaml @@ -1,9 +1,8 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(avg(balance)=[$2], state=[$0], gender=[$1]) - LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) - LogicalProject(state=[$9], gender=[$4], balance=[$7]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(state=[$9], gender=[$4], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2)), PROJECT->[avg(balance), state, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_span.yaml index b6af45e0974..5b6b68572c2 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_span.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_span.yaml @@ -1,9 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(max(balance)=[$1], age=[$0]) + LogicalProject(age=[$0], max(balance)=[$1]) LogicalAggregate(group=[{1}], max(balance)=[MAX($0)]) LogicalProject(balance=[$7], age0=[SPAN($10, 10, null:NULL)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},max(balance)=MAX($0)), PROJECT->[max(balance), age0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age0":{"histogram":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc","interval":10.0}}}]},"aggregations":{"max(balance)":{"max":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},max(balance)=MAX($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age0":{"histogram":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc","interval":10.0}}}]},"aggregations":{"max(balance)":{"max":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_group_keys.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_group_keys.yaml index 11e45e502bf..5cf91c3d322 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_group_keys.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_group_keys.yaml @@ -1,35 +1,35 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalAggregate(group=[{1, 2}], avg(balance)=[AVG($0)]) - LogicalProject(avg(balance)=[$0], gender=[$1], age=[CASE(IS NULL($2), 'NULL', <=($5, 10), $2, 'OTHER')]) - LogicalJoin(condition=[=($2, $3)], joinType=[left]) - LogicalProject(avg(balance)=[$2], gender=[$0], age=[SAFE_CAST($1)]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(gender=[$0], age=[CASE(IS NULL($1), 'NULL', <=($5, 10), $1, 'OTHER')], avg(balance)=[$2]) + LogicalJoin(condition=[=($1, $3)], joinType=[left]) + LogicalProject(gender=[$0], age=[SAFE_CAST($1)], avg(balance)=[$2]) LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) LogicalProject(gender=[$4], age=[$10], balance=[$7]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) LogicalProject(age=[$0], __grand_total__=[$1], __row_number__=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) - LogicalAggregate(group=[{1}], __grand_total__=[AVG($0)]) - LogicalProject(avg(balance)=[$2], age=[SAFE_CAST($1)]) - LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) - LogicalProject(gender=[$4], age=[$10], balance=[$7]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalAggregate(group=[{0}], __grand_total__=[AVG($1)]) + LogicalFilter(condition=[IS NOT NULL($0)]) + LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(gender=[$4], age=[$10], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(balance)=[$t8]) - EnumerableAggregate(group=[{1, 2}], agg#0=[$SUM0($0)], agg#1=[COUNT($0)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t2)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t2, $t9)], proj#0..1=[{exprs}], age=[$t10]) - EnumerableMergeJoin(condition=[=($2, $3)], joinType=[left]) - EnumerableSort(sort0=[$2], dir0=[ASC]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], expr#10=[SAFE_CAST($t1)], avg(balance)=[$t9], gender=[$t0], age=[$t10]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) + EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) + EnumerableSort(sort0=[$1], dir0=[ASC]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t10]) EnumerableAggregate(group=[{4, 10}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:DOUBLE], expr#6=[CASE($t4, $t5, $t1)], expr#7=[/($t6, $t2)], age=[$t0], __grand_total__=[$t7]) - EnumerableAggregate(group=[{1}], agg#0=[$SUM0($0)], agg#1=[COUNT($0)]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], expr#10=[SAFE_CAST($t1)], avg(balance)=[$t9], age=[$t10]) + EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], expr#11=[IS NOT NULL($t4)], age=[$t4], avg(balance)=[$t10], $condition=[$t11]) EnumerableAggregate(group=[{4, 10}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_null_str.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_null_str.yaml index 0c34016836b..22daa2c447c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_null_str.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_null_str.yaml @@ -1,27 +1,28 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalAggregate(group=[{1, 2}], avg(balance)=[AVG($0)]) - LogicalProject(avg(balance)=[$0], gender=[$1], age=[CASE(IS NULL($2), 'nil', <=($5, 10), $2, 'OTHER')]) - LogicalJoin(condition=[=($2, $3)], joinType=[left]) - LogicalProject(avg(balance)=[$2], gender=[$0], age=[SAFE_CAST($1)]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(gender=[$0], age=[CASE(IS NULL($1), 'nil', <=($5, 10), $1, 'OTHER')], avg(balance)=[$2]) + LogicalJoin(condition=[=($1, $3)], joinType=[left]) + LogicalProject(gender=[$0], age=[SAFE_CAST($1)], avg(balance)=[$2]) LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) LogicalProject(age=[$0], __grand_total__=[$1], __row_number__=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) - LogicalAggregate(group=[{1}], __grand_total__=[AVG($0)]) - LogicalProject(avg(balance)=[$2], age=[SAFE_CAST($1)]) - LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) - LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) + LogicalAggregate(group=[{0}], __grand_total__=[AVG($1)]) + LogicalFilter(condition=[IS NOT NULL($0)]) + LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) + LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) + LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(balance)=[$t8]) - EnumerableAggregate(group=[{1, 2}], agg#0=[$SUM0($0)], agg#1=[COUNT($0)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t2)], expr#6=['nil'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t2, $t9)], proj#0..1=[{exprs}], age=[$t10]) - EnumerableMergeJoin(condition=[=($2, $3)], joinType=[left]) - EnumerableSort(sort0=[$2], dir0=[ASC]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], expr#10=[SAFE_CAST($t1)], avg(balance)=[$t9], gender=[$t0], age=[$t10]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['nil'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) + EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) + EnumerableSort(sort0=[$1], dir0=[ASC]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t10]) EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) EnumerableCalc(expr#0..12=[{inputs}], expr#13=[10], expr#14=[null:NULL], expr#15=[SPAN($t5, $t13, $t14)], gender=[$t4], balance=[$t3], age0=[$t15]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) @@ -29,9 +30,8 @@ calcite: EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:DOUBLE], expr#6=[CASE($t4, $t5, $t1)], expr#7=[/($t6, $t2)], age=[$t0], __grand_total__=[$t7]) - EnumerableAggregate(group=[{1}], agg#0=[$SUM0($0)], agg#1=[COUNT($0)]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], expr#10=[SAFE_CAST($t1)], avg(balance)=[$t9], age=[$t10]) + EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], expr#11=[IS NOT NULL($t4)], age=[$t4], avg(balance)=[$t10], $condition=[$t11]) EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) EnumerableCalc(expr#0..12=[{inputs}], expr#13=[10], expr#14=[null:NULL], expr#15=[SPAN($t5, $t13, $t14)], gender=[$t4], balance=[$t3], age0=[$t15]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) - diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_single_group_key.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_single_group_key.yaml index b9e6ff9f735..5af244df172 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_single_group_key.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_single_group_key.yaml @@ -1,13 +1,11 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(avg(balance)=[$1], gender=[$0]) - LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)]) - LogicalProject(gender=[$4], balance=[$7]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)]) + LogicalProject(gender=[$4], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], avg(balance)=[$t8], gender=[$t0]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], gender=[$t0], avg(balance)=[$t8]) EnumerableAggregate(group=[{4}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_timestamp_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_timestamp_span.yaml index 913fd20b8bc..bdbe39dc2ae 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_timestamp_span.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_timestamp_span.yaml @@ -1,26 +1,27 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalAggregate(group=[{1, 2}], max(value)=[MAX($0)]) - LogicalProject(max(value)=[$0], timestamp=[$1], category=[CASE(IS NULL($2), 'NULL', <=($5, 10), $2, 'OTHER')]) - LogicalJoin(condition=[=($2, $3)], joinType=[left]) - LogicalProject(max(value)=[$2], timestamp=[$1], category=[$0]) + LogicalAggregate(group=[{0, 1}], max(value)=[MAX($2)]) + LogicalProject(timestamp=[$0], category=[CASE(IS NULL($1), 'NULL', <=($5, 10), $1, 'OTHER')], max(value)=[$2]) + LogicalJoin(condition=[=($1, $3)], joinType=[left]) + LogicalProject(timestamp=[$1], category=[$0], max(value)=[$2]) LogicalAggregate(group=[{0, 2}], max(value)=[MAX($1)]) LogicalProject(category=[$1], value=[$2], timestamp0=[SPAN($3, 1, 'w')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) LogicalProject(category=[$0], __grand_total__=[$1], __row_number__=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) - LogicalAggregate(group=[{1}], __grand_total__=[MAX($0)]) - LogicalProject(max(value)=[$2], category=[$0]) - LogicalAggregate(group=[{0, 2}], max(value)=[MAX($1)]) - LogicalProject(category=[$1], value=[$2], timestamp0=[SPAN($3, 1, 'w')]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + LogicalAggregate(group=[{0}], __grand_total__=[MAX($1)]) + LogicalFilter(condition=[IS NOT NULL($0)]) + LogicalProject(category=[$0], max(value)=[$2]) + LogicalAggregate(group=[{0, 2}], max(value)=[MAX($1)]) + LogicalProject(category=[$1], value=[$2], timestamp0=[SPAN($3, 1, 'w')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableAggregate(group=[{1, 2}], max(value)=[MAX($0)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t2)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t2, $t9)], proj#0..1=[{exprs}], category=[$t10]) - EnumerableMergeJoin(condition=[=($2, $3)], joinType=[left]) - EnumerableSort(sort0=[$2], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], max(value)=[$t2], timestamp=[$t1], category=[$t0]) + EnumerableAggregate(group=[{0, 1}], max(value)=[MAX($2)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], timestamp=[$t0], category=[$t10], max(value)=[$t2]) + EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) + EnumerableSort(sort0=[$1], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], timestamp=[$t1], category=[$t0], max(value)=[$t2]) EnumerableAggregate(group=[{0, 2}], max(value)=[MAX($1)]) EnumerableCalc(expr#0..9=[{inputs}], expr#10=[1], expr#11=['w'], expr#12=[SPAN($t3, $t10, $t11)], category=[$t1], value=[$t2], timestamp0=[$t12]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) @@ -28,5 +29,5 @@ calcite: EnumerableCalc(expr#0..2=[{inputs}], category=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{1}], __grand_total__=[MAX($2)]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) - + EnumerableCalc(expr#0..9=[{inputs}], expr#10=[IS NOT NULL($t1)], proj#0..9=[{exprs}], $condition=[$t10]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_use_other.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_use_other.yaml index 022072f3ae7..be986e25077 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_use_other.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_use_other.yaml @@ -1,30 +1,32 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalAggregate(group=[{1, 2}], max(severityNumber)=[MAX($0)]) - LogicalProject(max(severityNumber)=[$0], flags=[$1], severityText=[CASE(IS NULL($2), 'NULL', <=($5, 2), $2, 'max_among_other')]) - LogicalJoin(condition=[=($2, $3)], joinType=[left]) - LogicalProject(max(severityNumber)=[$2], flags=[$0], severityText=[$1]) - LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) - LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) + LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) + LogicalProject(flags=[$0], severityText=[CASE(IS NULL($1), 'NULL', <=($5, 2), $1, 'max_among_other')], max(severityNumber)=[$2]) + LogicalJoin(condition=[=($1, $3)], joinType=[left]) + LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) + LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) LogicalProject(severityText=[$0], __grand_total__=[$1], __row_number__=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) - LogicalAggregate(group=[{1}], __grand_total__=[MAX($0)]) - LogicalProject(max(severityNumber)=[$2], severityText=[$1]) - LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) - LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) + LogicalAggregate(group=[{0}], __grand_total__=[MAX($1)]) + LogicalFilter(condition=[IS NOT NULL($0)]) + LogicalProject(severityText=[$1], max(severityNumber)=[$2]) + LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) + LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableAggregate(group=[{1, 2}], max(severityNumber)=[MAX($0)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t2)], expr#6=['NULL'], expr#7=[2], expr#8=[<=($t4, $t7)], expr#9=['max_among_other'], expr#10=[CASE($t5, $t6, $t8, $t2, $t9)], proj#0..1=[{exprs}], severityText=[$t10]) - EnumerableMergeJoin(condition=[=($2, $3)], joinType=[left]) - EnumerableSort(sort0=[$2], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], max(severityNumber)=[$t2], flags=[$t1], severityText=[$t0]) + EnumerableAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[2], expr#8=[<=($t4, $t7)], expr#9=['max_among_other'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], flags=[$t0], severityText=[$t10], max(severityNumber)=[$t2]) + EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) + EnumerableSort(sort0=[$1], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], flags=[$t1], severityText=[$t0], max(severityNumber)=[$t2]) EnumerableAggregate(group=[{7, 23}], max(severityNumber)=[MAX($163)]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], severityText=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - EnumerableAggregate(group=[{7}], __grand_total__=[MAX($163)]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) + EnumerableAggregate(group=[{0}], __grand_total__=[MAX($2)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t0)], proj#0..2=[{exprs}], $condition=[$t3]) + EnumerableAggregate(group=[{7, 23}], max(severityNumber)=[MAX($163)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_limit.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_limit.yaml index e4ff0a172ce..0c23fbaf627 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_limit.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_limit.yaml @@ -1,13 +1,11 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(avg(balance)=[$2], state=[$0], gender=[$1]) - LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) - LogicalProject(state=[$9], gender=[$4], balance=[$7]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(state=[$9], gender=[$4], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], avg(balance)=[$t9], state=[$t1], gender=[$t0]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], state=[$t1], gender=[$t0], avg(balance)=[$t9]) EnumerableAggregate(group=[{4, 9}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_span.yaml index 6e8f8777170..8708b777a13 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_span.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_span.yaml @@ -1,14 +1,12 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(max(balance)=[$1], age=[$0]) + LogicalProject(age=[$0], max(balance)=[$1]) LogicalAggregate(group=[{1}], max(balance)=[MAX($0)]) LogicalProject(balance=[$7], age0=[SPAN($10, 10, null:NULL)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..1=[{inputs}], max(balance)=[$t1], age=[$t0]) - EnumerableAggregate(group=[{1}], max(balance)=[MAX($0)]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[10], expr#20=[null:NULL], expr#21=[SPAN($t10, $t19, $t20)], balance=[$t7], age0=[$t21]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - + EnumerableAggregate(group=[{1}], max(balance)=[MAX($0)]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[10], expr#20=[null:NULL], expr#21=[SPAN($t10, $t19, $t20)], balance=[$t7], age0=[$t21]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLChartTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLChartTest.java index c49f0344900..a29dc230949 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLChartTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLChartTest.java @@ -80,7 +80,7 @@ public void testChartWithSingleGroupKey() { RelNode root = getRelNode(ppl); String expectedSparkSql = - "SELECT AVG(`balance`) `avg(balance)`, `gender`\n" + "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" + "GROUP BY `gender`"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -92,7 +92,7 @@ public void testChartWithOverSyntax() { RelNode root = getRelNode(ppl); String expectedSparkSql = - "SELECT AVG(`balance`) `avg(balance)`, `gender`\n" + "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" + "GROUP BY `gender`"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -104,21 +104,22 @@ public void testChartWithMultipleGroupKeys() { RelNode root = getRelNode(ppl); String expectedSparkSql = - "SELECT `t1`.`gender`, CASE WHEN `t1`.`age` IS NULL THEN 'NULL' WHEN `t6`.`__row_number__`" + "SELECT `t1`.`gender`, CASE WHEN `t1`.`age` IS NULL THEN 'NULL' WHEN `t7`.`__row_number__`" + " <= 10 THEN `t1`.`age` ELSE 'OTHER' END `age`, AVG(`t1`.`avg(balance)`)" + " `avg(balance)`\n" - + "FROM (SELECT AVG(`balance`) `avg(balance)`, `gender`, SAFE_CAST(`age` AS STRING)" - + " `age`\n" + + "FROM (SELECT `gender`, SAFE_CAST(`age` AS STRING) `age`, AVG(`balance`)" + + " `avg(balance)`\n" + "FROM `scott`.`bank`\n" + "GROUP BY `gender`, `age`) `t1`\n" + "LEFT JOIN (SELECT `age`, AVG(`avg(balance)`) `__grand_total__`, ROW_NUMBER() OVER" + " (ORDER BY AVG(`avg(balance)`) DESC) `__row_number__`\n" - + "FROM (SELECT AVG(`balance`) `avg(balance)`, SAFE_CAST(`age` AS STRING) `age`\n" + + "FROM (SELECT SAFE_CAST(`age` AS STRING) `age`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" + "GROUP BY `gender`, `age`) `t4`\n" - + "GROUP BY `age`) `t6` ON `t1`.`age` = `t6`.`age`\n" + + "WHERE `age` IS NOT NULL\n" + + "GROUP BY `age`) `t7` ON `t1`.`age` = `t7`.`age`\n" + "GROUP BY `t1`.`gender`, CASE WHEN `t1`.`age` IS NULL THEN 'NULL' WHEN" - + " `t6`.`__row_number__` <= 10 THEN `t1`.`age` ELSE 'OTHER' END"; + + " `t7`.`__row_number__` <= 10 THEN `t1`.`age` ELSE 'OTHER' END"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -128,21 +129,22 @@ public void testChartWithMultipleGroupKeysAlternativeSyntax() { RelNode root = getRelNode(ppl); String expectedSparkSql = - "SELECT `t1`.`gender`, CASE WHEN `t1`.`age` IS NULL THEN 'NULL' WHEN `t6`.`__row_number__`" + "SELECT `t1`.`gender`, CASE WHEN `t1`.`age` IS NULL THEN 'NULL' WHEN `t7`.`__row_number__`" + " <= 10 THEN `t1`.`age` ELSE 'OTHER' END `age`, AVG(`t1`.`avg(balance)`)" + " `avg(balance)`\n" - + "FROM (SELECT AVG(`balance`) `avg(balance)`, `gender`, SAFE_CAST(`age` AS STRING)" - + " `age`\n" + + "FROM (SELECT `gender`, SAFE_CAST(`age` AS STRING) `age`, AVG(`balance`)" + + " `avg(balance)`\n" + "FROM `scott`.`bank`\n" + "GROUP BY `gender`, `age`) `t1`\n" + "LEFT JOIN (SELECT `age`, AVG(`avg(balance)`) `__grand_total__`, ROW_NUMBER() OVER" + " (ORDER BY AVG(`avg(balance)`) DESC) `__row_number__`\n" - + "FROM (SELECT AVG(`balance`) `avg(balance)`, SAFE_CAST(`age` AS STRING) `age`\n" + + "FROM (SELECT SAFE_CAST(`age` AS STRING) `age`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" + "GROUP BY `gender`, `age`) `t4`\n" - + "GROUP BY `age`) `t6` ON `t1`.`age` = `t6`.`age`\n" + + "WHERE `age` IS NOT NULL\n" + + "GROUP BY `age`) `t7` ON `t1`.`age` = `t7`.`age`\n" + "GROUP BY `t1`.`gender`, CASE WHEN `t1`.`age` IS NULL THEN 'NULL' WHEN" - + " `t6`.`__row_number__` <= 10 THEN `t1`.`age` ELSE 'OTHER' END"; + + " `t7`.`__row_number__` <= 10 THEN `t1`.`age` ELSE 'OTHER' END"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -152,7 +154,7 @@ public void testChartWithLimit() { RelNode root = getRelNode(ppl); String expectedSparkSql = - "SELECT AVG(`balance`) `avg(balance)`, `gender`\n" + "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" + "GROUP BY `gender`"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -164,7 +166,7 @@ public void testChartWithLimitZero() { RelNode root = getRelNode(ppl); String expectedSparkSql = - "SELECT AVG(`balance`) `avg(balance)`, `state`, `gender`\n" + "SELECT `state`, `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" + "GROUP BY `state`, `gender`"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -176,7 +178,7 @@ public void testChartWithSpan() { RelNode root = getRelNode(ppl); String expectedSparkSql = - "SELECT MAX(`balance`) `max(balance)`, `SPAN`(`age`, 10, NULL) `age`\n" + "SELECT `SPAN`(`age`, 10, NULL) `age`, MAX(`balance`) `max(balance)`\n" + "FROM `scott`.`bank`\n" + "GROUP BY `SPAN`(`age`, 10, NULL)"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -189,20 +191,21 @@ public void testChartWithTimeSpan() { RelNode root = getRelNode(ppl); String expectedSparkSql = "SELECT `t1`.`timestamp`, CASE WHEN `t1`.`category` IS NULL THEN 'NULL' WHEN" - + " `t6`.`__row_number__` <= 10 THEN `t1`.`category` ELSE 'OTHER' END `category`," + + " `t7`.`__row_number__` <= 10 THEN `t1`.`category` ELSE 'OTHER' END `category`," + " MAX(`t1`.`max(value)`) `max(value)`\n" - + "FROM (SELECT MAX(`value`) `max(value)`, `SPAN`(`timestamp`, 1, 'w') `timestamp`," - + " `category`\n" + + "FROM (SELECT `SPAN`(`timestamp`, 1, 'w') `timestamp`, `category`, MAX(`value`)" + + " `max(value)`\n" + "FROM `scott`.`time_data`\n" + "GROUP BY `category`, `SPAN`(`timestamp`, 1, 'w')) `t1`\n" + "LEFT JOIN (SELECT `category`, MAX(`max(value)`) `__grand_total__`, ROW_NUMBER() OVER" + " (ORDER BY MAX(`max(value)`) DESC) `__row_number__`\n" - + "FROM (SELECT MAX(`value`) `max(value)`, `category`\n" + + "FROM (SELECT `category`, MAX(`value`) `max(value)`\n" + "FROM `scott`.`time_data`\n" + "GROUP BY `category`, `SPAN`(`timestamp`, 1, 'w')) `t4`\n" - + "GROUP BY `category`) `t6` ON `t1`.`category` = `t6`.`category`\n" + + "WHERE `category` IS NOT NULL\n" + + "GROUP BY `category`) `t7` ON `t1`.`category` = `t7`.`category`\n" + "GROUP BY `t1`.`timestamp`, CASE WHEN `t1`.`category` IS NULL THEN 'NULL' WHEN" - + " `t6`.`__row_number__` <= 10 THEN `t1`.`category` ELSE 'OTHER' END"; + + " `t7`.`__row_number__` <= 10 THEN `t1`.`category` ELSE 'OTHER' END"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -212,7 +215,7 @@ public void testChartWithUseOtherTrue() { RelNode root = getRelNode(ppl); String expectedSparkSql = - "SELECT AVG(`balance`) `avg(balance)`, `gender`\n" + "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" + "GROUP BY `gender`"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -224,7 +227,7 @@ public void testChartWithUseOtherFalse() { RelNode root = getRelNode(ppl); String expectedSparkSql = - "SELECT AVG(`balance`) `avg(balance)`, `gender`\n" + "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" + "GROUP BY `gender`"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -236,7 +239,7 @@ public void testChartWithOtherStr() { RelNode root = getRelNode(ppl); String expectedSparkSql = - "SELECT AVG(`balance`) `avg(balance)`, `gender`\n" + "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" + "GROUP BY `gender`"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -248,7 +251,7 @@ public void testChartWithNullStr() { RelNode root = getRelNode(ppl); String expectedSparkSql = - "SELECT AVG(`balance`) `avg(balance)`, `gender`\n" + "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" + "GROUP BY `gender`"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -260,7 +263,7 @@ public void testChartWithUseNull() { RelNode root = getRelNode(ppl); String expectedSparkSql = - "SELECT AVG(`balance`) `avg(balance)`, `gender`\n" + "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" + "WHERE `gender` IS NOT NULL\n" + "GROUP BY `gender`"; From 1fe81b3298d0f546363e54b5dff35080b70db73a Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Wed, 29 Oct 2025 20:39:17 +0800 Subject: [PATCH 22/35] Chores: tweak code order Signed-off-by: Yuanchun Shen --- .../org/opensearch/sql/calcite/CalciteRelNodeVisitor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index eb73fbceea2..11b652244cb 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -2137,12 +2137,12 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { SqlStdOperatorTable.LESS_THAN_OR_EQUAL, relBuilder.field(ROW_NUM_COL), relBuilder.literal(limit)); - RexNode nullCondition = relBuilder.isNull(colSplitPostJoin); - RexNode columnSplitExpr; if (!config.useOther) { relBuilder.filter(lteCondition); } + RexNode nullCondition = relBuilder.isNull(colSplitPostJoin); + RexNode columnSplitExpr; if (config.useNull) { columnSplitExpr = relBuilder.call( From d7949ef96bec690a02e362e71bf314b706950503 Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Wed, 29 Oct 2025 22:21:59 +0800 Subject: [PATCH 23/35] Add anonymize test to chart command Signed-off-by: Yuanchun Shen --- .../sql/ppl/utils/PPLQueryDataAnonymizer.java | 37 +++++++++++++++++++ .../ppl/utils/PPLQueryDataAnonymizerTest.java | 28 ++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java index e392a682cef..ea173f894a2 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -56,6 +56,7 @@ import org.opensearch.sql.ast.tree.Append; import org.opensearch.sql.ast.tree.AppendCol; import org.opensearch.sql.ast.tree.Bin; +import org.opensearch.sql.ast.tree.Chart; import org.opensearch.sql.ast.tree.CountBin; import org.opensearch.sql.ast.tree.Dedupe; import org.opensearch.sql.ast.tree.DefaultBin; @@ -520,6 +521,42 @@ public String visitTimechart(Timechart node, String context) { return StringUtils.format("%s%s", child, timechartCommand.toString()); } + @Override + public String visitChart(Chart node, String context) { + String child = node.getChild().get(0).accept(this, context); + StringBuilder chartCommand = new StringBuilder(); + chartCommand.append(" | chart"); + + for (Argument arg : node.getArguments()) { + String argName = arg.getArgName(); + // Skip the auto-generated "top" parameter that's added when limit is specified + if ("top".equals(argName)) { + continue; + } + if ("limit".equals(argName) || "useother".equals(argName) || "usenull".equals(argName)) { + chartCommand.append(" ").append(argName).append("=").append(MASK_LITERAL); + } else if ("otherstr".equals(argName) || "nullstr".equals(argName)) { + chartCommand.append(" ").append(argName).append("=").append(MASK_LITERAL); + } + } + + chartCommand.append(" ").append(visitExpression(node.getAggregationFunction())); + + if (node.getRowSplit() != null && node.getColumnSplit() != null) { + chartCommand + .append(" by ") + .append(visitExpression(node.getRowSplit())) + .append(" ") + .append(visitExpression(node.getColumnSplit())); + } else if (node.getRowSplit() != null) { + chartCommand.append(" by ").append(visitExpression(node.getRowSplit())); + } else if (node.getColumnSplit() != null) { + chartCommand.append(" by ").append(visitExpression(node.getColumnSplit())); + } + + return StringUtils.format("%s%s", child, chartCommand.toString()); + } + public String visitRex(Rex node, String context) { String child = node.getChild().get(0).accept(this, context); String field = visitExpression(node.getField()); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index 6de9acacfe1..811e3fb62de 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -232,6 +232,34 @@ public void testTimechartCommand() { anonymize("source=t | timechart count() by host")); } + @Test + public void testChartCommand() { + assertEquals( + "source=table | chart count(identifier) by identifier identifier", + anonymize("source=t | chart count(age) by gender country")); + } + + @Test + public void testChartCommandWithParameters() { + assertEquals( + "source=table | chart limit=*** useother=*** avg(identifier) by identifier", + anonymize("source=t | chart limit=5 useother=false avg(balance) by state")); + } + + @Test + public void testChartCommandOver() { + assertEquals( + "source=table | chart avg(identifier) by identifier", + anonymize("source=t | chart avg(balance) over gender")); + } + + @Test + public void testChartCommandOverBy() { + assertEquals( + "source=table | chart sum(identifier) by identifier identifier", + anonymize("source=t | chart sum(amount) over gender by age")); + } + // todo, sort order is ignored, it doesn't impact the log analysis. @Test public void testSortCommandWithOptions() { From 7bef2027a743c7d165b78fc0b2dc0769d6fb8d1b Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Fri, 31 Oct 2025 12:59:51 +0800 Subject: [PATCH 24/35] Change grammart from limit=top 10 to limit=top10 Signed-off-by: Yuanchun Shen --- .../sql/calcite/CalciteRelNodeVisitor.java | 17 ++++++----------- .../calcite/remote/CalciteChartCommandIT.java | 6 +++--- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 3 ++- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 3 ++- .../sql/ppl/utils/ArgumentFactory.java | 15 +++++++++++++-- .../sql/ppl/parser/AstBuilderTest.java | 4 ++-- .../sql/ppl/utils/ArgumentFactoryTest.java | 2 +- 7 files changed, 29 insertions(+), 21 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index ddfbed811d2..753c6bd89fc 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -915,7 +915,8 @@ private boolean isCountField(RexCall call) { private Pair, List> aggregateWithTrimming( List groupExprList, List aggExprList, - CalcitePlanContext context) { + CalcitePlanContext context, + boolean hintBucketNonNull) { Pair, List> resolved = resolveAttributesForAggregation(groupExprList, aggExprList, context); List resolvedGroupByList = resolved.getLeft(); @@ -1019,6 +1020,7 @@ private Pair, List> aggregateWithTrimming( List intendedGroupKeyAliases = getGroupKeyNamesAfterAggregation(reResolved.getLeft()); context.relBuilder.aggregate( context.relBuilder.groupKey(reResolved.getLeft()), reResolved.getRight()); + if (hintBucketNonNull) addIgnoreNullBucketHintToAggregate(context); // During aggregation, Calcite projects both input dependencies and output group-by fields. // When names conflict, Calcite adds numeric suffixes (e.g., "value0"). // Apply explicit renaming to restore the intended aliases. @@ -1141,10 +1143,7 @@ private void visitAggregation(Aggregation node, CalcitePlanContext context, bool } Pair, List> aggregationAttributes = - aggregateWithTrimming(groupExprList, aggExprList, context); - if (toAddHintsOnAggregate) { - addIgnoreNullBucketHintToAggregate(context); - } + aggregateWithTrimming(groupExprList, aggExprList, context, toAddHintsOnAggregate); // schema reordering List outputFields = context.relBuilder.fields(); @@ -1896,11 +1895,7 @@ public RelNode visitRareTopN(RareTopN node, CalcitePlanContext context) { .map(context.relBuilder::isNotNull) .toList()); } - aggregateWithTrimming(groupExprList, aggExprList, context); - - if (toAddHintsOnAggregate) { - addIgnoreNullBucketHintToAggregate(context); - } + aggregateWithTrimming(groupExprList, aggExprList, context, toAddHintsOnAggregate); // 2. add a window column List partitionKeys = rexVisitor.analyze(node.getGroupExprList(), context); @@ -2262,7 +2257,7 @@ public RelNode visitTimechart( try { // Step 1: Initial aggregation - IMPORTANT: order is [spanExpr, byField] groupExprList = Arrays.asList(spanExpr, byField); - aggregateWithTrimming(groupExprList, List.of(node.getAggregateFunction()), context); + aggregateWithTrimming(groupExprList, List.of(node.getAggregateFunction()), context, false); // First rename the timestamp field (2nd to last) to @timestamp List fieldNames = context.relBuilder.peek().getRowType().getFieldNames(); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java index c25296d9cd8..46b067ac9d7 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java @@ -211,7 +211,7 @@ public void testChartLimitTopWithUseOther() throws IOException { JSONObject result = executeQuery( String.format( - "source=%s | chart limit=top 2 useother=true otherstr='max_among_other'" + "source=%s | chart limit=top2 useother=true otherstr='max_among_other'" + " max(severityNumber) over flags by severityText", TEST_INDEX_OTEL_LOGS)); verifySchema( @@ -232,7 +232,7 @@ public void testChartLimitBottomWithUseOther() throws IOException { JSONObject result = executeQuery( String.format( - "source=%s | chart limit=bottom 2 useother=false otherstr='other_small_not_shown'" + "source=%s | chart limit=bottom2 useother=false otherstr='other_small_not_shown'" + " max(severityNumber) over flags by severityText", TEST_INDEX_OTEL_LOGS)); verifySchema( @@ -248,7 +248,7 @@ public void testChartLimitTopWithMinAgg() throws IOException { JSONObject result = executeQuery( String.format( - "source=%s | chart limit=top 2 min(severityNumber) over flags by severityText", + "source=%s | chart limit=top2 min(severityNumber) over flags by severityText", TEST_INDEX_OTEL_LOGS)); verifySchema( result, diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index f6bad7aa568..65451fae908 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -94,7 +94,8 @@ COST: 'COST'; EXTENDED: 'EXTENDED'; OVERRIDE: 'OVERRIDE'; OVERWRITE: 'OVERWRITE'; -BOTTOM: 'BOTTOM'; +TOP_K: 'TOP'[0-9]+; +BOTTOM_K: 'BOTTOM'[0-9]+; // SORT FIELD KEYWORDS // TODO #3180: Fix broken sort functionality diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 341e5cb0d6f..51837b74a82 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -264,7 +264,8 @@ chartCommand ; chartOptions - : LIMIT EQUAL (TOP | BOTTOM)? integerLiteral + : LIMIT EQUAL integerLiteral + | LIMIT EQUAL (TOP_K | BOTTOM_K) | USEOTHER EQUAL booleanLiteral | OTHERSTR EQUAL stringLiteral | USENULL EQUAL booleanLiteral diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java index 1c4796a6596..e99bef9966a 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java @@ -185,9 +185,20 @@ public static List getArgumentList(ChartCommandContext ctx) { List arguments = new ArrayList<>(); for (var optionCtx : ctx.chartOptions()) { if (optionCtx.LIMIT() != null) { - arguments.add(new Argument("limit", getArgumentValue(optionCtx.integerLiteral()))); + Literal limit; + if (optionCtx.integerLiteral() != null) { + limit = getArgumentValue(optionCtx.integerLiteral()); + } else { + limit = + AstDSL.intLiteral( + Integer.parseInt( + (optionCtx.TOP_K() != null ? optionCtx.TOP_K() : optionCtx.BOTTOM_K()) + .getText() + .replaceAll("[^0-9-]", ""))); + } + arguments.add(new Argument("limit", limit)); // not specified | top presents -> true; bottom presents -> false - arguments.add(new Argument("top", AstDSL.booleanLiteral(optionCtx.BOTTOM() == null))); + arguments.add(new Argument("top", AstDSL.booleanLiteral(optionCtx.BOTTOM_K() == null))); } else if (optionCtx.USEOTHER() != null) { arguments.add(new Argument("useother", getArgumentValue(optionCtx.booleanLiteral()))); } else if (optionCtx.OTHERSTR() != null) { diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index 7da90c861b2..f1464e31065 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -1535,7 +1535,7 @@ public void testChartCommandWithOptions() { @Test public void testChartCommandWithAllOptions() { assertEqual( - "source=t | chart limit=5 useother=false otherstr='OTHER' usenull=true nullstr='NULL'" + "source=t | chart limit=top5 useother=false otherstr='OTHER' usenull=true nullstr='NULL'" + " avg(balance) by gender", Chart.builder() .child(relation("t")) @@ -1555,7 +1555,7 @@ public void testChartCommandWithAllOptions() { @Test public void testChartCommandWithBottomLimit() { assertEqual( - "source=t | chart limit=bottom 3 count() by category", + "source=t | chart limit=bottom3 count() by category", Chart.builder() .child(relation("t")) .columnSplit(alias("category", field("category"))) diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/ArgumentFactoryTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/ArgumentFactoryTest.java index f5b389146c7..dc2a9d66061 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/ArgumentFactoryTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/ArgumentFactoryTest.java @@ -125,7 +125,7 @@ public void testChartCommandArguments() { @Test public void testChartCommandBottomArguments() { assertEqual( - "source=t | chart limit=bottom 3 count() by status", + "source=t | chart limit=bottom3 count() by status", Chart.builder() .child(relation("t")) .columnSplit(alias("status", field("status"))) From 9da3bd285cdfe85051fcbf2503225484bafa22c6 Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Fri, 31 Oct 2025 13:35:03 +0800 Subject: [PATCH 25/35] Update chart doc Signed-off-by: Yuanchun Shen --- docs/user/ppl/cmd/chart.rst | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/docs/user/ppl/cmd/chart.rst b/docs/user/ppl/cmd/chart.rst index ed1cd2a453e..5824ef704bc 100644 --- a/docs/user/ppl/cmd/chart.rst +++ b/docs/user/ppl/cmd/chart.rst @@ -32,24 +32,26 @@ Syntax * **limit**: optional. Specifies the number of distinct values to display when using column split. - * Default: 10 - * Syntax: ``limit=(top|bottom) `` or ``limit=`` (defaults to top) - * When there are more distinct column split values than the limit, the additional values are grouped into an "OTHER" category if ``useother`` is not set to false. - * Set to 0 to show all distinct values without any limit. + * Default: top10 + * Syntax: ``limit=(top|bottom)`` or ``limit=`` (defaults to top) + * When ``limit=K`` is set, the top or bottom K distinct column split values are retained; the additional values are grouped into an "OTHER" category if ``useother`` is not set to false. + * Set limit to 0 to show all distinct values without any limit. + * Use ``limit=topK`` or ``limit=bottomK`` to specify whether to retain the top or bottom K column categories. The ranking is based on the aggregated values for each distinct column-split value. For example, ``chart limit=top3 count() by a b`` retains the 3 most common b categories; ``chart limit=top5 min(value) by a b`` selects the 5 b categories that contains smallest aggregated values. If not specified, top is used by default. * Only applies when column split presents (by 2 fields or over...by... coexists). -* **useother**: optional. Controls whether to create an "OTHER" category for distinct column values beyond the limit. +* **useother**: optional. Controls whether to create an "OTHER" category for distinct column-split values beyond the limit. * Default: true - * When set to false, only the top/bottom N distinct values (based on limit) are shown without an "OTHER" category. + * When set to false, only the top/bottom N distinct column-split values (based on limit) are shown without an "OTHER" category. * When set to true, distinct values beyond the limit are grouped into an "OTHER" category. - * Only applies when using column split and when there are more distinct column values than the limit. + * Only applies when using column split and when there are more distinct column-split values than the limit. -* **usenull**: optional. Controls whether to include null values as a separate category. +* **usenull**: optional. Controls whether to group events without a column split (i.e. whose column split is null) into a separate "NULL" category. * Default: true - * When set to false, events with null values in the split-by field are excluded from results. - * When set to true, null values appear as a separate category. + * When ``usenull=false``, events with a null column split are excluded from results. + * When ``usenull=true``, events with a null column split are grouped into a separate "NULL" category. + * ``usenull`` only applies to column split. Null values in the row split are handled in the same way as normal aggregations. * **nullstr**: optional. Specifies the category name for rows that do not contain the column split value. @@ -80,7 +82,8 @@ Syntax Notes ===== -* The column split field in the result will become strings so that they are compatible with ``nullstr`` and ``otherstr`` and can be used as column names once pivoted. +* The fields generated by column splitting are converted to strings so that they are compatible with ``nullstr`` and ``otherstr`` and can be used as column names once pivoted. +* The aggregation metric appears as the last column in the result. Result columns are ordered as: [row-split] [column-split] [aggregation-metrics] Examples ======== @@ -119,7 +122,7 @@ PPL query:: Example 3: Using over and by for multiple field grouping -------------------------------------------------------- -This example shows average balance grouped by both gender and age fields. +This example shows average balance grouped by both gender and age fields. Note that the age column in the result is converted to string type. PPL query:: @@ -137,7 +140,7 @@ PPL query:: Example 4: Using basic limit functionality ------------------------------------------ -This example limits the results to show only the top 1 age group. +This example limits the results to show only the top 1 age group. Note that the age column in the result is converted to string type. PPL query:: @@ -158,7 +161,7 @@ This example shows using limit with useother and custom otherstr parameters. PPL query:: - os> source=accounts | chart limit=top 1 useother=true otherstr='minor_gender' count() over state by gender + os> source=accounts | chart limit=top1 useother=true otherstr='minor_gender' count() over state by gender fetched rows / total rows = 4/4 +-------+--------------+---------+ | state | gender | count() | From befede183b6350198600c592b4064e5e2158bfba Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Tue, 4 Nov 2025 13:24:07 +0800 Subject: [PATCH 26/35] Rename __row_number__ for chart to _row_number_chart_ Signed-off-by: Yuanchun Shen --- .../sql/calcite/CalciteRelNodeVisitor.java | 5 +-- .../sql/calcite/utils/PlanUtils.java | 1 + .../explain_chart_multiple_group_keys.yaml | 2 +- .../calcite/explain_chart_null_str.yaml | 2 +- .../calcite/explain_chart_timestamp_span.yaml | 2 +- .../calcite/explain_chart_use_other.yaml | 2 +- .../explain_chart_multiple_group_keys.yaml | 2 +- .../explain_chart_multiple_groups.yaml | 2 +- .../explain_chart_null_str.yaml | 2 +- .../explain_chart_timestamp_span.yaml | 2 +- .../explain_chart_use_other.yaml | 2 +- .../explain_timechart.yaml | 38 ++++++++++--------- .../sql/ppl/calcite/CalcitePPLChartTest.java | 26 ++++++------- 13 files changed, 45 insertions(+), 43 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 75942c042b6..34be0ed1ead 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -2459,7 +2459,6 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { // Always set it to null last so that it does not interfere with top / bottom calculation grandTotal = relBuilder.nullsLast(grandTotal); - final String ROW_NUM_COL = "__row_number__"; RexNode rowNum = PlanUtils.makeOver( context, @@ -2469,7 +2468,7 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { List.of(), List.of(grandTotal), WindowFrame.toCurrentRow()); - relBuilder.projectPlus(relBuilder.alias(rowNum, ROW_NUM_COL)); + relBuilder.projectPlus(relBuilder.alias(rowNum, PlanUtils.ROW_NUMBER_COLUMN_FOR_CHART)); RelNode ranked = relBuilder.build(); relBuilder.push(aggregated); @@ -2483,7 +2482,7 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { RexNode lteCondition = relBuilder.call( SqlStdOperatorTable.LESS_THAN_OR_EQUAL, - relBuilder.field(ROW_NUM_COL), + relBuilder.field(PlanUtils.ROW_NUMBER_COLUMN_FOR_CHART), relBuilder.literal(limit)); if (!config.useOther) { relBuilder.filter(lteCondition); diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java index fefab6d57ce..ded7ba541a4 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java @@ -66,6 +66,7 @@ public interface PlanUtils { String ROW_NUMBER_COLUMN_FOR_MAIN = "_row_number_main_"; String ROW_NUMBER_COLUMN_FOR_SUBSEARCH = "_row_number_subsearch_"; String ROW_NUMBER_COLUMN_FOR_STREAMSTATS = "__stream_seq__"; + String ROW_NUMBER_COLUMN_FOR_CHART = "_row_number_chart_"; static SpanUnit intervalUnitToSpanUnit(IntervalUnit unit) { return switch (unit) { diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_multiple_group_keys.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_multiple_group_keys.yaml index b41cebbda4e..30938d7b649 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_multiple_group_keys.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_multiple_group_keys.yaml @@ -8,7 +8,7 @@ calcite: LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) LogicalProject(gender=[$4], age=[$10], balance=[$7]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - LogicalProject(age=[$0], __grand_total__=[$1], __row_number__=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) + LogicalProject(age=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) LogicalAggregate(group=[{0}], __grand_total__=[AVG($1)]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_null_str.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_null_str.yaml index 3f60744f069..8fc0cfe490f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_null_str.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_null_str.yaml @@ -8,7 +8,7 @@ calcite: LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) - LogicalProject(age=[$0], __grand_total__=[$1], __row_number__=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) + LogicalProject(age=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) LogicalAggregate(group=[{0}], __grand_total__=[AVG($1)]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_timestamp_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_timestamp_span.yaml index 7fb6a77dce2..1e22843fd78 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_timestamp_span.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_timestamp_span.yaml @@ -8,7 +8,7 @@ calcite: LogicalAggregate(group=[{0, 2}], max(value)=[MAX($1)]) LogicalProject(category=[$1], value=[$2], timestamp0=[SPAN($3, 1, 'w')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) - LogicalProject(category=[$0], __grand_total__=[$1], __row_number__=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) + LogicalProject(category=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) LogicalAggregate(group=[{0}], __grand_total__=[MAX($1)]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(category=[$0], max(value)=[$2]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_use_other.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_use_other.yaml index ffb167e148e..de71751630e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_use_other.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_use_other.yaml @@ -7,7 +7,7 @@ calcite: LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) - LogicalProject(severityText=[$0], __grand_total__=[$1], __row_number__=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) + LogicalProject(severityText=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) LogicalAggregate(group=[{0}], __grand_total__=[MAX($1)]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(severityText=[$1], max(severityNumber)=[$2]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_group_keys.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_group_keys.yaml index 5cf91c3d322..ec4c159bece 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_group_keys.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_group_keys.yaml @@ -8,7 +8,7 @@ calcite: LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) LogicalProject(gender=[$4], age=[$10], balance=[$7]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - LogicalProject(age=[$0], __grand_total__=[$1], __row_number__=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) + LogicalProject(age=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) LogicalAggregate(group=[{0}], __grand_total__=[AVG($1)]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_groups.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_groups.yaml index 5d641cb3929..8409ddf0604 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_groups.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_groups.yaml @@ -7,7 +7,7 @@ logical: | LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) LogicalProject(gender=[$4], age=[$10], balance=[$7]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - LogicalProject(age=[$0], __grand_total__=[$1], __row_number__=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) + LogicalProject(age=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) LogicalAggregate(group=[{1}], __grand_total__=[AVG($0)]) LogicalProject(avg(balance)=[$2], age=[SAFE_CAST($1)]) LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_null_str.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_null_str.yaml index 22daa2c447c..f7e95b6a716 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_null_str.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_null_str.yaml @@ -8,7 +8,7 @@ calcite: LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) - LogicalProject(age=[$0], __grand_total__=[$1], __row_number__=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) + LogicalProject(age=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) LogicalAggregate(group=[{0}], __grand_total__=[AVG($1)]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_timestamp_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_timestamp_span.yaml index bdbe39dc2ae..d5ce3f2febe 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_timestamp_span.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_timestamp_span.yaml @@ -8,7 +8,7 @@ calcite: LogicalAggregate(group=[{0, 2}], max(value)=[MAX($1)]) LogicalProject(category=[$1], value=[$2], timestamp0=[SPAN($3, 1, 'w')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) - LogicalProject(category=[$0], __grand_total__=[$1], __row_number__=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) + LogicalProject(category=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) LogicalAggregate(group=[{0}], __grand_total__=[MAX($1)]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(category=[$0], max(value)=[$2]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_use_other.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_use_other.yaml index be986e25077..0223e718511 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_use_other.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_use_other.yaml @@ -7,7 +7,7 @@ calcite: LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) - LogicalProject(severityText=[$0], __grand_total__=[$1], __row_number__=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) + LogicalProject(severityText=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) LogicalAggregate(group=[{0}], __grand_total__=[MAX($1)]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(severityText=[$1], max(severityNumber)=[$2]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart.yaml index ae966d7eea7..5aa55ca656b 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) - LogicalAggregate(group=[{0, 1}], avg(cpu_usage)=[SUM($2)]) + LogicalAggregate(group=[{0, 1}], avg(cpu_usage)=[AVG($2)]) LogicalProject(@timestamp=[$0], host=[CASE(IS NOT NULL($3), $1, CASE(IS NULL($1), null:NULL, 'OTHER'))], avg(cpu_usage)=[$2]) LogicalJoin(condition=[=($1, $3)], joinType=[left]) LogicalProject(@timestamp=[$1], host=[$0], $f2=[$2]) @@ -10,7 +10,7 @@ calcite: LogicalProject(host=[$4], cpu_usage=[$7], $f3=[SPAN($1, 1, 'm')]) CalciteLogicalIndexScan(table=[[OpenSearch, events]]) LogicalSort(sort0=[$1], dir0=[DESC], fetch=[10]) - LogicalAggregate(group=[{1}], grand_total=[SUM($2)]) + LogicalAggregate(group=[{1}], grand_total=[AVG($2)]) LogicalFilter(condition=[IS NOT NULL($1)]) LogicalProject(@timestamp=[$1], host=[$0], $f2=[$2]) LogicalAggregate(group=[{0, 2}], agg#0=[AVG($1)]) @@ -19,19 +19,21 @@ calcite: physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) - EnumerableAggregate(group=[{0, 1}], avg(cpu_usage)=[SUM($2)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NOT NULL($t3)], expr#6=[IS NULL($t1)], expr#7=[null:NULL], expr#8=['OTHER'], expr#9=[CASE($t6, $t7, $t8)], expr#10=[CASE($t5, $t1, $t9)], @timestamp=[$t0], host=[$t10], avg(cpu_usage)=[$t2]) - EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) - EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], @timestamp=[$t1], host=[$t0], $f2=[$t8]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['m'], expr#18=[SPAN($t1, $t16, $t17)], host=[$t4], cpu_usage=[$t7], $f3=[$t18]) - CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) - EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableLimit(fetch=[10]) - EnumerableSort(sort0=[$1], dir0=[DESC]) - EnumerableAggregate(group=[{0}], grand_total=[SUM($2)]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], expr#9=[IS NOT NULL($t0)], proj#0..1=[{exprs}], $f2=[$t8], $condition=[$t9]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['m'], expr#18=[SPAN($t1, $t16, $t17)], host=[$t4], cpu_usage=[$t7], $f3=[$t18]) - CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(cpu_usage)=[$t8]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NOT NULL($t3)], expr#6=[IS NULL($t1)], expr#7=[null:NULL], expr#8=['OTHER'], expr#9=[CASE($t6, $t7, $t8)], expr#10=[CASE($t5, $t1, $t9)], @timestamp=[$t0], host=[$t10], avg(cpu_usage)=[$t2]) + EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) + EnumerableSort(sort0=[$1], dir0=[ASC]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], @timestamp=[$t1], host=[$t0], $f2=[$t8]) + EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['m'], expr#18=[SPAN($t1, $t16, $t17)], host=[$t4], cpu_usage=[$t7], $f3=[$t18]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableLimit(fetch=[10]) + EnumerableSort(sort0=[$1], dir0=[DESC]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:DOUBLE], expr#6=[CASE($t4, $t5, $t1)], expr#7=[/($t6, $t2)], host=[$t0], grand_total=[$t7]) + EnumerableAggregate(group=[{0}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], expr#9=[IS NOT NULL($t0)], proj#0..1=[{exprs}], $f2=[$t8], $condition=[$t9]) + EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['m'], expr#18=[SPAN($t1, $t16, $t17)], host=[$t4], cpu_usage=[$t7], $f3=[$t18]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLChartTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLChartTest.java index a29dc230949..2619fce64b6 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLChartTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLChartTest.java @@ -104,22 +104,22 @@ public void testChartWithMultipleGroupKeys() { RelNode root = getRelNode(ppl); String expectedSparkSql = - "SELECT `t1`.`gender`, CASE WHEN `t1`.`age` IS NULL THEN 'NULL' WHEN `t7`.`__row_number__`" - + " <= 10 THEN `t1`.`age` ELSE 'OTHER' END `age`, AVG(`t1`.`avg(balance)`)" - + " `avg(balance)`\n" + "SELECT `t1`.`gender`, CASE WHEN `t1`.`age` IS NULL THEN 'NULL' WHEN" + + " `t7`.`_row_number_chart_` <= 10 THEN `t1`.`age` ELSE 'OTHER' END `age`," + + " AVG(`t1`.`avg(balance)`) `avg(balance)`\n" + "FROM (SELECT `gender`, SAFE_CAST(`age` AS STRING) `age`, AVG(`balance`)" + " `avg(balance)`\n" + "FROM `scott`.`bank`\n" + "GROUP BY `gender`, `age`) `t1`\n" + "LEFT JOIN (SELECT `age`, AVG(`avg(balance)`) `__grand_total__`, ROW_NUMBER() OVER" - + " (ORDER BY AVG(`avg(balance)`) DESC) `__row_number__`\n" + + " (ORDER BY AVG(`avg(balance)`) DESC) `_row_number_chart_`\n" + "FROM (SELECT SAFE_CAST(`age` AS STRING) `age`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" + "GROUP BY `gender`, `age`) `t4`\n" + "WHERE `age` IS NOT NULL\n" + "GROUP BY `age`) `t7` ON `t1`.`age` = `t7`.`age`\n" + "GROUP BY `t1`.`gender`, CASE WHEN `t1`.`age` IS NULL THEN 'NULL' WHEN" - + " `t7`.`__row_number__` <= 10 THEN `t1`.`age` ELSE 'OTHER' END"; + + " `t7`.`_row_number_chart_` <= 10 THEN `t1`.`age` ELSE 'OTHER' END"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -129,22 +129,22 @@ public void testChartWithMultipleGroupKeysAlternativeSyntax() { RelNode root = getRelNode(ppl); String expectedSparkSql = - "SELECT `t1`.`gender`, CASE WHEN `t1`.`age` IS NULL THEN 'NULL' WHEN `t7`.`__row_number__`" - + " <= 10 THEN `t1`.`age` ELSE 'OTHER' END `age`, AVG(`t1`.`avg(balance)`)" - + " `avg(balance)`\n" + "SELECT `t1`.`gender`, CASE WHEN `t1`.`age` IS NULL THEN 'NULL' WHEN" + + " `t7`.`_row_number_chart_` <= 10 THEN `t1`.`age` ELSE 'OTHER' END `age`," + + " AVG(`t1`.`avg(balance)`) `avg(balance)`\n" + "FROM (SELECT `gender`, SAFE_CAST(`age` AS STRING) `age`, AVG(`balance`)" + " `avg(balance)`\n" + "FROM `scott`.`bank`\n" + "GROUP BY `gender`, `age`) `t1`\n" + "LEFT JOIN (SELECT `age`, AVG(`avg(balance)`) `__grand_total__`, ROW_NUMBER() OVER" - + " (ORDER BY AVG(`avg(balance)`) DESC) `__row_number__`\n" + + " (ORDER BY AVG(`avg(balance)`) DESC) `_row_number_chart_`\n" + "FROM (SELECT SAFE_CAST(`age` AS STRING) `age`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" + "GROUP BY `gender`, `age`) `t4`\n" + "WHERE `age` IS NOT NULL\n" + "GROUP BY `age`) `t7` ON `t1`.`age` = `t7`.`age`\n" + "GROUP BY `t1`.`gender`, CASE WHEN `t1`.`age` IS NULL THEN 'NULL' WHEN" - + " `t7`.`__row_number__` <= 10 THEN `t1`.`age` ELSE 'OTHER' END"; + + " `t7`.`_row_number_chart_` <= 10 THEN `t1`.`age` ELSE 'OTHER' END"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -191,21 +191,21 @@ public void testChartWithTimeSpan() { RelNode root = getRelNode(ppl); String expectedSparkSql = "SELECT `t1`.`timestamp`, CASE WHEN `t1`.`category` IS NULL THEN 'NULL' WHEN" - + " `t7`.`__row_number__` <= 10 THEN `t1`.`category` ELSE 'OTHER' END `category`," + + " `t7`.`_row_number_chart_` <= 10 THEN `t1`.`category` ELSE 'OTHER' END `category`," + " MAX(`t1`.`max(value)`) `max(value)`\n" + "FROM (SELECT `SPAN`(`timestamp`, 1, 'w') `timestamp`, `category`, MAX(`value`)" + " `max(value)`\n" + "FROM `scott`.`time_data`\n" + "GROUP BY `category`, `SPAN`(`timestamp`, 1, 'w')) `t1`\n" + "LEFT JOIN (SELECT `category`, MAX(`max(value)`) `__grand_total__`, ROW_NUMBER() OVER" - + " (ORDER BY MAX(`max(value)`) DESC) `__row_number__`\n" + + " (ORDER BY MAX(`max(value)`) DESC) `_row_number_chart_`\n" + "FROM (SELECT `category`, MAX(`value`) `max(value)`\n" + "FROM `scott`.`time_data`\n" + "GROUP BY `category`, `SPAN`(`timestamp`, 1, 'w')) `t4`\n" + "WHERE `category` IS NOT NULL\n" + "GROUP BY `category`) `t7` ON `t1`.`category` = `t7`.`category`\n" + "GROUP BY `t1`.`timestamp`, CASE WHEN `t1`.`category` IS NULL THEN 'NULL' WHEN" - + " `t7`.`__row_number__` <= 10 THEN `t1`.`category` ELSE 'OTHER' END"; + + " `t7`.`_row_number_chart_` <= 10 THEN `t1`.`category` ELSE 'OTHER' END"; verifyPPLToSparkSQL(root, expectedSparkSql); } From 285780e791f7423663b4ba59dfd6525286b136ba Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Tue, 4 Nov 2025 14:11:42 +0800 Subject: [PATCH 27/35] Sort by row and col splits on top of chart results Signed-off-by: Yuanchun Shen --- .../sql/calcite/CalciteRelNodeVisitor.java | 9 ++- docs/user/ppl/cmd/chart.rst | 12 ++-- .../explain_chart_multiple_group_keys.yaml | 60 ++++++++-------- .../calcite/explain_chart_null_str.yaml | 68 ++++++++++--------- .../explain_chart_single_group_key.yaml | 11 +-- .../calcite/explain_chart_timestamp_span.yaml | 48 ++++++------- .../calcite/explain_chart_use_other.yaml | 46 +++++++------ .../calcite/explain_chart_with_limit.yaml | 11 +-- .../calcite/explain_chart_with_span.yaml | 13 ++-- .../explain_chart_multiple_group_keys.yaml | 64 ++++++++--------- .../explain_chart_multiple_groups.yaml | 8 ++- .../explain_chart_null_str.yaml | 68 ++++++++++--------- .../explain_chart_single_group.yaml | 10 +-- .../explain_chart_single_group_key.yaml | 16 +++-- .../explain_chart_timestamp_span.yaml | 58 ++++++++-------- .../explain_chart_use_other.yaml | 58 ++++++++-------- .../explain_chart_with_limit.yaml | 16 +++-- .../explain_chart_with_span.yaml | 18 ++--- .../sql/ppl/calcite/CalcitePPLChartTest.java | 39 +++++++---- 19 files changed, 339 insertions(+), 294 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 34be0ed1ead..0512a172469 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -2408,13 +2408,14 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { relBuilder.peek().getRowType().getFieldNames().size() > 2 ? relBuilder.peek().getRowType().getFieldNames().get(1) : null; - RelNode aggregated = context.relBuilder.peek(); // If row or column split does not present or limit equals 0, this is the same as `stats agg // [group by col]` because all truncating is performed on the column split Integer limit = (Integer) argMap.getOrDefault("limit", Chart.DEFAULT_LIMIT).getValue(); if (node.getRowSplit() == null || node.getColumnSplit() == null || Objects.equals(limit, 0)) { - return aggregated; + // The output of chart is expected to be ordered by row split names + relBuilder.sort(relBuilder.field(0)); + return relBuilder.peek(); } String aggFunctionName = getAggFunctionName(node.getAggregationFunction()); @@ -2438,7 +2439,7 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { columSplitName); } relBuilder.project(relBuilder.field(0), colSplit, relBuilder.field(2)); - aggregated = relBuilder.peek(); + RelNode aggregated = relBuilder.peek(); // 1: column-split, 2: agg relBuilder.project(relBuilder.field(1), relBuilder.field(2)); @@ -2516,6 +2517,8 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { relBuilder.aggregate( relBuilder.groupKey(relBuilder.field(0), relBuilder.field(1)), buildAggCall(context.relBuilder, aggFunction, relBuilder.field(2)).as(aggFieldName)); + // The output of chart is expected to be ordered by row and column split names + relBuilder.sort(relBuilder.field(0), relBuilder.field(1)); return relBuilder.peek(); } diff --git a/docs/user/ppl/cmd/chart.rst b/docs/user/ppl/cmd/chart.rst index 5824ef704bc..760843f3d6e 100644 --- a/docs/user/ppl/cmd/chart.rst +++ b/docs/user/ppl/cmd/chart.rst @@ -149,9 +149,9 @@ PPL query:: +--------+-------+---------+ | gender | age | count() | |--------+-------+---------| - | M | OTHER | 2 | - | M | 33 | 1 | | F | OTHER | 1 | + | M | 33 | 1 | + | M | OTHER | 2 | +--------+-------+---------+ Example 5: Using limit with other parameters @@ -166,10 +166,10 @@ PPL query:: +-------+--------------+---------+ | state | gender | count() | |-------+--------------+---------| - | TN | M | 1 | + | IL | M | 1 | | MD | M | 1 | + | TN | M | 1 | | VA | minor_gender | 1 | - | IL | M | 1 | +-------+--------------+---------+ Example 6: Using null parameters @@ -184,10 +184,10 @@ PPL query:: +-----------+------------------------+---------+ | firstname | employer | count() | |-----------+------------------------+---------| - | Nanette | Quility | 1 | | Amber | Pyrami | 1 | | Dale | employer not specified | 1 | | Hattie | Netagy | 1 | + | Nanette | Quility | 1 | +-----------+------------------------+---------+ Example 7: Using chart command with span @@ -202,8 +202,8 @@ PPL query:: +-----+--------+--------------+ | age | gender | max(balance) | |-----+--------+--------------| - | 30 | M | 39225 | | 20 | F | 32838 | + | 30 | M | 39225 | +-----+--------+--------------+ Limitations diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_multiple_group_keys.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_multiple_group_keys.yaml index 30938d7b649..d619f0b89f6 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_multiple_group_keys.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_multiple_group_keys.yaml @@ -1,33 +1,35 @@ calcite: logical: | - LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) - LogicalProject(gender=[$0], age=[CASE(IS NULL($1), 'NULL', <=($5, 10), $1, 'OTHER')], avg(balance)=[$2]) - LogicalJoin(condition=[=($1, $3)], joinType=[left]) - LogicalProject(gender=[$0], age=[SAFE_CAST($1)], avg(balance)=[$2]) - LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) - LogicalProject(gender=[$4], age=[$10], balance=[$7]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - LogicalProject(age=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) - LogicalAggregate(group=[{0}], __grand_total__=[AVG($1)]) - LogicalFilter(condition=[IS NOT NULL($0)]) - LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) - LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) - LogicalProject(gender=[$4], age=[$10], balance=[$7]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(gender=[$0], age=[CASE(IS NULL($1), 'NULL', <=($5, 10), $1, 'OTHER')], avg(balance)=[$2]) + LogicalJoin(condition=[=($1, $3)], joinType=[left]) + LogicalProject(gender=[$0], age=[SAFE_CAST($1)], avg(balance)=[$2]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(gender=[$4], age=[$10], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalProject(age=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) + LogicalAggregate(group=[{0}], __grand_total__=[AVG($1)]) + LogicalFilter(condition=[IS NOT NULL($0)]) + LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(gender=[$4], age=[$10], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(balance)=[$t8]) - EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) - EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) - EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[SAFE_CAST($t1)], gender=[$t0], age=[$t3], avg(balance)=[$t2]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) - EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:DOUBLE], expr#6=[CASE($t4, $t5, $t1)], expr#7=[/($t6, $t2)], age=[$t0], __grand_total__=[$t7]) - EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[SAFE_CAST($t0)], expr#3=[IS NOT NULL($t2)], $f0=[$t2], avg(balance)=[$t1], $condition=[$t3]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2)), PROJECT->[age, avg(balance)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(balance)=[$t8]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) + EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) + EnumerableSort(sort0=[$1], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[SAFE_CAST($t1)], gender=[$t0], age=[$t3], avg(balance)=[$t2]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) + EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:DOUBLE], expr#6=[CASE($t4, $t5, $t1)], expr#7=[/($t6, $t2)], age=[$t0], __grand_total__=[$t7]) + EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[SAFE_CAST($t0)], expr#3=[IS NOT NULL($t2)], $f0=[$t2], avg(balance)=[$t1], $condition=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2)), PROJECT->[age, avg(balance)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_null_str.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_null_str.yaml index 8fc0cfe490f..2fa1a05e515 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_null_str.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_null_str.yaml @@ -1,37 +1,39 @@ calcite: logical: | - LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) - LogicalProject(gender=[$0], age=[CASE(IS NULL($1), 'nil', <=($5, 10), $1, 'OTHER')], avg(balance)=[$2]) - LogicalJoin(condition=[=($1, $3)], joinType=[left]) - LogicalProject(gender=[$0], age=[SAFE_CAST($1)], avg(balance)=[$2]) - LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) - LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) - LogicalProject(age=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) - LogicalAggregate(group=[{0}], __grand_total__=[AVG($1)]) - LogicalFilter(condition=[IS NOT NULL($0)]) - LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) - LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) - LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) + LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(gender=[$0], age=[CASE(IS NULL($1), 'nil', <=($5, 10), $1, 'OTHER')], avg(balance)=[$2]) + LogicalJoin(condition=[=($1, $3)], joinType=[left]) + LogicalProject(gender=[$0], age=[SAFE_CAST($1)], avg(balance)=[$2]) + LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) + LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) + LogicalProject(age=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) + LogicalAggregate(group=[{0}], __grand_total__=[AVG($1)]) + LogicalFilter(condition=[IS NOT NULL($0)]) + LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) + LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) + LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(balance)=[$t8]) - EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['nil'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) - EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) - EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t10]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], proj#0..1=[{exprs}], $f2=[$t5]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[gender, balance, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","balance","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) - EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:DOUBLE], expr#6=[CASE($t4, $t5, $t1)], expr#7=[/($t6, $t2)], age=[$t0], __grand_total__=[$t7]) - EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], expr#11=[IS NOT NULL($t4)], age=[$t4], avg(balance)=[$t10], $condition=[$t11]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], proj#0..1=[{exprs}], $f2=[$t5]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[gender, balance, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","balance","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(balance)=[$t8]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['nil'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) + EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) + EnumerableSort(sort0=[$1], dir0=[ASC]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t10]) + EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], proj#0..1=[{exprs}], $f2=[$t5]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[gender, balance, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","balance","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) + EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:DOUBLE], expr#6=[CASE($t4, $t5, $t1)], expr#7=[/($t6, $t2)], age=[$t0], __grand_total__=[$t7]) + EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], expr#11=[IS NOT NULL($t4)], age=[$t4], avg(balance)=[$t10], $condition=[$t11]) + EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], proj#0..1=[{exprs}], $f2=[$t5]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[gender, balance, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","balance","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_single_group_key.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_single_group_key.yaml index be22ae9c011..8ec7422de37 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_single_group_key.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_single_group_key.yaml @@ -1,8 +1,9 @@ calcite: logical: | - LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)]) - LogicalProject(gender=[$4], balance=[$7]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)]) + LogicalProject(gender=[$4], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg(balance)=AVG($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg(balance)=AVG($1)), SORT->[0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"last","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_timestamp_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_timestamp_span.yaml index 1e22843fd78..1233817e40f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_timestamp_span.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_timestamp_span.yaml @@ -1,27 +1,29 @@ calcite: logical: | - LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalAggregate(group=[{0, 1}], max(value)=[MAX($2)]) - LogicalProject(timestamp=[$0], category=[CASE(IS NULL($1), 'NULL', <=($5, 10), $1, 'OTHER')], max(value)=[$2]) - LogicalJoin(condition=[=($1, $3)], joinType=[left]) - LogicalProject(timestamp=[$1], category=[$0], max(value)=[$2]) - LogicalAggregate(group=[{0, 2}], max(value)=[MAX($1)]) - LogicalProject(category=[$1], value=[$2], timestamp0=[SPAN($3, 1, 'w')]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) - LogicalProject(category=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) - LogicalAggregate(group=[{0}], __grand_total__=[MAX($1)]) - LogicalFilter(condition=[IS NOT NULL($0)]) - LogicalProject(category=[$0], max(value)=[$2]) - LogicalAggregate(group=[{0, 2}], max(value)=[MAX($1)]) - LogicalProject(category=[$1], value=[$2], timestamp0=[SPAN($3, 1, 'w')]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + LogicalAggregate(group=[{0, 1}], max(value)=[MAX($2)]) + LogicalProject(timestamp=[$0], category=[CASE(IS NULL($1), 'NULL', <=($5, 10), $1, 'OTHER')], max(value)=[$2]) + LogicalJoin(condition=[=($1, $3)], joinType=[left]) + LogicalProject(timestamp=[$1], category=[$0], max(value)=[$2]) + LogicalAggregate(group=[{0, 2}], max(value)=[MAX($1)]) + LogicalProject(category=[$1], value=[$2], timestamp0=[SPAN($3, 1, 'w')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + LogicalProject(category=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) + LogicalAggregate(group=[{0}], __grand_total__=[MAX($1)]) + LogicalFilter(condition=[IS NOT NULL($0)]) + LogicalProject(category=[$0], max(value)=[$2]) + LogicalAggregate(group=[{0, 2}], max(value)=[MAX($1)]) + LogicalProject(category=[$1], value=[$2], timestamp0=[SPAN($3, 1, 'w')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableAggregate(group=[{0, 1}], max(value)=[MAX($2)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], timestamp=[$t0], category=[$t10], max(value)=[$t2]) - EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},max(value)=MAX($1)), PROJECT->[timestamp0, category, max(value)], SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"timestamp0":{"date_histogram":{"field":"timestamp","missing_bucket":false,"order":"asc","calendar_interval":"1w"}}}]},"aggregations":{"max(value)":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], category=[$t0], $1=[$t2]) - EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[FILTER->IS NOT NULL($1), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},__grand_total__=MAX($1))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"category","boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"__grand_total__":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + EnumerableAggregate(group=[{0, 1}], max(value)=[MAX($2)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], timestamp=[$t0], category=[$t10], max(value)=[$t2]) + EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},max(value)=MAX($1)), PROJECT->[timestamp0, category, max(value)], SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"timestamp0":{"date_histogram":{"field":"timestamp","missing_bucket":false,"order":"asc","calendar_interval":"1w"}}}]},"aggregations":{"max(value)":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], category=[$t0], $1=[$t2]) + EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[FILTER->IS NOT NULL($1), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},__grand_total__=MAX($1))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"category","boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"__grand_total__":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_use_other.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_use_other.yaml index de71751630e..7916a31a5d8 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_use_other.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_use_other.yaml @@ -1,26 +1,28 @@ calcite: logical: | - LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) - LogicalProject(flags=[$0], severityText=[CASE(IS NULL($1), 'NULL', <=($5, 2), $1, 'max_among_other')], max(severityNumber)=[$2]) - LogicalJoin(condition=[=($1, $3)], joinType=[left]) - LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) - LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) - LogicalProject(severityText=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) - LogicalAggregate(group=[{0}], __grand_total__=[MAX($1)]) - LogicalFilter(condition=[IS NOT NULL($0)]) - LogicalProject(severityText=[$1], max(severityNumber)=[$2]) - LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) - LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) + LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) + LogicalProject(flags=[$0], severityText=[CASE(IS NULL($1), 'NULL', <=($5, 2), $1, 'max_among_other')], max(severityNumber)=[$2]) + LogicalJoin(condition=[=($1, $3)], joinType=[left]) + LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) + LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) + LogicalProject(severityText=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) + LogicalAggregate(group=[{0}], __grand_total__=[MAX($1)]) + LogicalFilter(condition=[IS NOT NULL($0)]) + LogicalProject(severityText=[$1], max(severityNumber)=[$2]) + LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) + LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[2], expr#8=[<=($t4, $t7)], expr#9=['max_among_other'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], flags=[$t0], severityText=[$t10], max(severityNumber)=[$t2]) - EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},max(severityNumber)=MAX($2)), SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"severityText":{"terms":{"field":"severityText","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"flags":{"terms":{"field":"flags","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"max(severityNumber)":{"max":{"field":"severityNumber"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], severityText=[$t0], $1=[$t2]) - EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[FILTER->IS NOT NULL($7), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},__grand_total__=MAX($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"severityText","boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"severityText":{"terms":{"field":"severityText","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"__grand_total__":{"max":{"field":"severityNumber"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + EnumerableAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[2], expr#8=[<=($t4, $t7)], expr#9=['max_among_other'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], flags=[$t0], severityText=[$t10], max(severityNumber)=[$t2]) + EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},max(severityNumber)=MAX($2)), SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"severityText":{"terms":{"field":"severityText","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"flags":{"terms":{"field":"flags","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"max(severityNumber)":{"max":{"field":"severityNumber"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], severityText=[$t0], $1=[$t2]) + EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[FILTER->IS NOT NULL($7), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},__grand_total__=MAX($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"severityText","boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"severityText":{"terms":{"field":"severityText","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"__grand_total__":{"max":{"field":"severityNumber"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_limit.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_limit.yaml index 4c97f6b8506..0b4ea1ad9c8 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_limit.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_limit.yaml @@ -1,8 +1,9 @@ calcite: logical: | - LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) - LogicalProject(state=[$9], gender=[$4], balance=[$7]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(state=[$9], gender=[$4], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2)), SORT->[0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_span.yaml index 5b6b68572c2..73039fe4ed5 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_span.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_span.yaml @@ -1,9 +1,10 @@ calcite: logical: | - LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(age=[$0], max(balance)=[$1]) - LogicalAggregate(group=[{1}], max(balance)=[MAX($0)]) - LogicalProject(balance=[$7], age0=[SPAN($10, 10, null:NULL)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalProject(age=[$0], max(balance)=[$1]) + LogicalAggregate(group=[{1}], max(balance)=[MAX($0)]) + LogicalProject(balance=[$7], age0=[SPAN($10, 10, null:NULL)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},max(balance)=MAX($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age0":{"histogram":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc","interval":10.0}}}]},"aggregations":{"max(balance)":{"max":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},max(balance)=MAX($0)), SORT->[0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age0":{"histogram":{"field":"age","missing_bucket":true,"missing_order":"last","order":"asc","interval":10.0}}}]},"aggregations":{"max(balance)":{"max":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_group_keys.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_group_keys.yaml index ec4c159bece..c9a7377f629 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_group_keys.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_group_keys.yaml @@ -1,35 +1,37 @@ calcite: logical: | - LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) - LogicalProject(gender=[$0], age=[CASE(IS NULL($1), 'NULL', <=($5, 10), $1, 'OTHER')], avg(balance)=[$2]) - LogicalJoin(condition=[=($1, $3)], joinType=[left]) - LogicalProject(gender=[$0], age=[SAFE_CAST($1)], avg(balance)=[$2]) - LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) - LogicalProject(gender=[$4], age=[$10], balance=[$7]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - LogicalProject(age=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) - LogicalAggregate(group=[{0}], __grand_total__=[AVG($1)]) - LogicalFilter(condition=[IS NOT NULL($0)]) - LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) - LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) - LogicalProject(gender=[$4], age=[$10], balance=[$7]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(gender=[$0], age=[CASE(IS NULL($1), 'NULL', <=($5, 10), $1, 'OTHER')], avg(balance)=[$2]) + LogicalJoin(condition=[=($1, $3)], joinType=[left]) + LogicalProject(gender=[$0], age=[SAFE_CAST($1)], avg(balance)=[$2]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(gender=[$4], age=[$10], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalProject(age=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) + LogicalAggregate(group=[{0}], __grand_total__=[AVG($1)]) + LogicalFilter(condition=[IS NOT NULL($0)]) + LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(gender=[$4], age=[$10], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(balance)=[$t8]) - EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) - EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) - EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t10]) - EnumerableAggregate(group=[{4, 10}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) - EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:DOUBLE], expr#6=[CASE($t4, $t5, $t1)], expr#7=[/($t6, $t2)], age=[$t0], __grand_total__=[$t7]) - EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], expr#11=[IS NOT NULL($t4)], age=[$t4], avg(balance)=[$t10], $condition=[$t11]) - EnumerableAggregate(group=[{4, 10}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(balance)=[$t8]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) + EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) + EnumerableSort(sort0=[$1], dir0=[ASC]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t10]) + EnumerableAggregate(group=[{4, 10}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) + EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:DOUBLE], expr#6=[CASE($t4, $t5, $t1)], expr#7=[/($t6, $t2)], age=[$t0], __grand_total__=[$t7]) + EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], expr#11=[IS NOT NULL($t4)], age=[$t4], avg(balance)=[$t10], $condition=[$t11]) + EnumerableAggregate(group=[{4, 10}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_groups.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_groups.yaml index 8409ddf0604..df3fd8391d5 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_groups.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_groups.yaml @@ -1,6 +1,7 @@ logical: | - LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalAggregate(group=[{1, 2}], avg(balance)=[AVG($0)]) + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT], sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + LogicalAggregate(group=[{1, 2}], avg(balance)=[AVG($0)]) LogicalProject(avg(balance)=[$0], gender=[$1], age=[CASE(IS NULL($2), 'NULL', <=($5, 10), $2, 'OTHER')]) LogicalJoin(condition=[=($2, $3)], joinType=[left]) LogicalProject(avg(balance)=[$2], gender=[$0], age=[SAFE_CAST($1)]) @@ -15,7 +16,8 @@ logical: | CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(balance)=[$t8]) EnumerableAggregate(group=[{1, 2}], agg#0=[$SUM0($0)], agg#1=[COUNT($0)]) EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t2)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t2, $t9)], proj#0..1=[{exprs}], age=[$t10]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_null_str.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_null_str.yaml index f7e95b6a716..72d29bd3691 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_null_str.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_null_str.yaml @@ -1,37 +1,39 @@ calcite: logical: | - LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) - LogicalProject(gender=[$0], age=[CASE(IS NULL($1), 'nil', <=($5, 10), $1, 'OTHER')], avg(balance)=[$2]) - LogicalJoin(condition=[=($1, $3)], joinType=[left]) - LogicalProject(gender=[$0], age=[SAFE_CAST($1)], avg(balance)=[$2]) - LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) - LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) - LogicalProject(age=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) - LogicalAggregate(group=[{0}], __grand_total__=[AVG($1)]) - LogicalFilter(condition=[IS NOT NULL($0)]) - LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) - LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) - LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) + LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(gender=[$0], age=[CASE(IS NULL($1), 'nil', <=($5, 10), $1, 'OTHER')], avg(balance)=[$2]) + LogicalJoin(condition=[=($1, $3)], joinType=[left]) + LogicalProject(gender=[$0], age=[SAFE_CAST($1)], avg(balance)=[$2]) + LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) + LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) + LogicalProject(age=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) + LogicalAggregate(group=[{0}], __grand_total__=[AVG($1)]) + LogicalFilter(condition=[IS NOT NULL($0)]) + LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) + LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) + LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(balance)=[$t8]) - EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['nil'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) - EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) - EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t10]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..12=[{inputs}], expr#13=[10], expr#14=[null:NULL], expr#15=[SPAN($t5, $t13, $t14)], gender=[$t4], balance=[$t3], age0=[$t15]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) - EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) - EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:DOUBLE], expr#6=[CASE($t4, $t5, $t1)], expr#7=[/($t6, $t2)], age=[$t0], __grand_total__=[$t7]) - EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], expr#11=[IS NOT NULL($t4)], age=[$t4], avg(balance)=[$t10], $condition=[$t11]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..12=[{inputs}], expr#13=[10], expr#14=[null:NULL], expr#15=[SPAN($t5, $t13, $t14)], gender=[$t4], balance=[$t3], age0=[$t15]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) + EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(balance)=[$t8]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['nil'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) + EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) + EnumerableSort(sort0=[$1], dir0=[ASC]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t10]) + EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[10], expr#14=[null:NULL], expr#15=[SPAN($t5, $t13, $t14)], gender=[$t4], balance=[$t3], age0=[$t15]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) + EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:DOUBLE], expr#6=[CASE($t4, $t5, $t1)], expr#7=[/($t6, $t2)], age=[$t0], __grand_total__=[$t7]) + EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], expr#11=[IS NOT NULL($t4)], age=[$t4], avg(balance)=[$t10], $condition=[$t11]) + EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[10], expr#14=[null:NULL], expr#15=[SPAN($t5, $t13, $t14)], gender=[$t4], balance=[$t3], age0=[$t15]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_single_group.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_single_group.yaml index 0cf28205b97..208fdf99935 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_single_group.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_single_group.yaml @@ -1,12 +1,14 @@ logical: | - LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(avg(balance)=[$1], gender=[$0]) - LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)]) + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT], sort0=[$0], dir0=[ASC]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalProject(avg(balance)=[$1], gender=[$0]) + LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)]) LogicalProject(gender=[$4], balance=[$7]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], avg(balance)=[$t8], gender=[$t0]) EnumerableAggregate(group=[{4}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_single_group_key.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_single_group_key.yaml index 5af244df172..9370110fa0e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_single_group_key.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_single_group_key.yaml @@ -1,11 +1,13 @@ calcite: logical: | - LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)]) - LogicalProject(gender=[$4], balance=[$7]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)]) + LogicalProject(gender=[$4], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], gender=[$t0], avg(balance)=[$t8]) - EnumerableAggregate(group=[{4}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], gender=[$t0], avg(balance)=[$t8]) + EnumerableAggregate(group=[{4}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_timestamp_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_timestamp_span.yaml index d5ce3f2febe..e0826fddcea 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_timestamp_span.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_timestamp_span.yaml @@ -1,33 +1,35 @@ calcite: logical: | - LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalAggregate(group=[{0, 1}], max(value)=[MAX($2)]) - LogicalProject(timestamp=[$0], category=[CASE(IS NULL($1), 'NULL', <=($5, 10), $1, 'OTHER')], max(value)=[$2]) - LogicalJoin(condition=[=($1, $3)], joinType=[left]) - LogicalProject(timestamp=[$1], category=[$0], max(value)=[$2]) - LogicalAggregate(group=[{0, 2}], max(value)=[MAX($1)]) - LogicalProject(category=[$1], value=[$2], timestamp0=[SPAN($3, 1, 'w')]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) - LogicalProject(category=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) - LogicalAggregate(group=[{0}], __grand_total__=[MAX($1)]) - LogicalFilter(condition=[IS NOT NULL($0)]) - LogicalProject(category=[$0], max(value)=[$2]) - LogicalAggregate(group=[{0, 2}], max(value)=[MAX($1)]) - LogicalProject(category=[$1], value=[$2], timestamp0=[SPAN($3, 1, 'w')]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + LogicalAggregate(group=[{0, 1}], max(value)=[MAX($2)]) + LogicalProject(timestamp=[$0], category=[CASE(IS NULL($1), 'NULL', <=($5, 10), $1, 'OTHER')], max(value)=[$2]) + LogicalJoin(condition=[=($1, $3)], joinType=[left]) + LogicalProject(timestamp=[$1], category=[$0], max(value)=[$2]) + LogicalAggregate(group=[{0, 2}], max(value)=[MAX($1)]) + LogicalProject(category=[$1], value=[$2], timestamp0=[SPAN($3, 1, 'w')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + LogicalProject(category=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) + LogicalAggregate(group=[{0}], __grand_total__=[MAX($1)]) + LogicalFilter(condition=[IS NOT NULL($0)]) + LogicalProject(category=[$0], max(value)=[$2]) + LogicalAggregate(group=[{0, 2}], max(value)=[MAX($1)]) + LogicalProject(category=[$1], value=[$2], timestamp0=[SPAN($3, 1, 'w')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableAggregate(group=[{0, 1}], max(value)=[MAX($2)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], timestamp=[$t0], category=[$t10], max(value)=[$t2]) - EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) - EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], timestamp=[$t1], category=[$t0], max(value)=[$t2]) - EnumerableAggregate(group=[{0, 2}], max(value)=[MAX($1)]) - EnumerableCalc(expr#0..9=[{inputs}], expr#10=[1], expr#11=['w'], expr#12=[SPAN($t3, $t10, $t11)], category=[$t1], value=[$t2], timestamp0=[$t12]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) - EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], category=[$t0], $1=[$t2]) - EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - EnumerableAggregate(group=[{1}], __grand_total__=[MAX($2)]) - EnumerableCalc(expr#0..9=[{inputs}], expr#10=[IS NOT NULL($t1)], proj#0..9=[{exprs}], $condition=[$t10]) + EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + EnumerableAggregate(group=[{0, 1}], max(value)=[MAX($2)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], timestamp=[$t0], category=[$t10], max(value)=[$t2]) + EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) + EnumerableSort(sort0=[$1], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], timestamp=[$t1], category=[$t0], max(value)=[$t2]) + EnumerableAggregate(group=[{0, 2}], max(value)=[MAX($1)]) + EnumerableCalc(expr#0..9=[{inputs}], expr#10=[1], expr#11=['w'], expr#12=[SPAN($t3, $t10, $t11)], category=[$t1], value=[$t2], timestamp0=[$t12]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], category=[$t0], $1=[$t2]) + EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableAggregate(group=[{1}], __grand_total__=[MAX($2)]) + EnumerableCalc(expr#0..9=[{inputs}], expr#10=[IS NOT NULL($t1)], proj#0..9=[{exprs}], $condition=[$t10]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_use_other.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_use_other.yaml index 0223e718511..8e36c39627f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_use_other.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_use_other.yaml @@ -1,32 +1,34 @@ calcite: logical: | - LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) - LogicalProject(flags=[$0], severityText=[CASE(IS NULL($1), 'NULL', <=($5, 2), $1, 'max_among_other')], max(severityNumber)=[$2]) - LogicalJoin(condition=[=($1, $3)], joinType=[left]) - LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) - LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) - LogicalProject(severityText=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) - LogicalAggregate(group=[{0}], __grand_total__=[MAX($1)]) - LogicalFilter(condition=[IS NOT NULL($0)]) - LogicalProject(severityText=[$1], max(severityNumber)=[$2]) - LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) - LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) + LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) + LogicalProject(flags=[$0], severityText=[CASE(IS NULL($1), 'NULL', <=($5, 2), $1, 'max_among_other')], max(severityNumber)=[$2]) + LogicalJoin(condition=[=($1, $3)], joinType=[left]) + LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) + LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) + LogicalProject(severityText=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) + LogicalAggregate(group=[{0}], __grand_total__=[MAX($1)]) + LogicalFilter(condition=[IS NOT NULL($0)]) + LogicalProject(severityText=[$1], max(severityNumber)=[$2]) + LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) + LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[2], expr#8=[<=($t4, $t7)], expr#9=['max_among_other'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], flags=[$t0], severityText=[$t10], max(severityNumber)=[$t2]) - EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) - EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], flags=[$t1], severityText=[$t0], max(severityNumber)=[$t2]) - EnumerableAggregate(group=[{7, 23}], max(severityNumber)=[MAX($163)]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) - EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], severityText=[$t0], $1=[$t2]) - EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - EnumerableAggregate(group=[{0}], __grand_total__=[MAX($2)]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t0)], proj#0..2=[{exprs}], $condition=[$t3]) - EnumerableAggregate(group=[{7, 23}], max(severityNumber)=[MAX($163)]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) + EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + EnumerableAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[2], expr#8=[<=($t4, $t7)], expr#9=['max_among_other'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], flags=[$t0], severityText=[$t10], max(severityNumber)=[$t2]) + EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) + EnumerableSort(sort0=[$1], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], flags=[$t1], severityText=[$t0], max(severityNumber)=[$t2]) + EnumerableAggregate(group=[{7, 23}], max(severityNumber)=[MAX($163)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], severityText=[$t0], $1=[$t2]) + EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableAggregate(group=[{0}], __grand_total__=[MAX($2)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t0)], proj#0..2=[{exprs}], $condition=[$t3]) + EnumerableAggregate(group=[{7, 23}], max(severityNumber)=[MAX($163)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_limit.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_limit.yaml index 0c23fbaf627..b51f7589cf3 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_limit.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_limit.yaml @@ -1,11 +1,13 @@ calcite: logical: | - LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) - LogicalProject(state=[$9], gender=[$4], balance=[$7]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(state=[$9], gender=[$4], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], state=[$t1], gender=[$t0], avg(balance)=[$t9]) - EnumerableAggregate(group=[{4, 9}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], state=[$t1], gender=[$t0], avg(balance)=[$t9]) + EnumerableAggregate(group=[{4, 9}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_span.yaml index 8708b777a13..77129b1530d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_span.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_span.yaml @@ -1,12 +1,14 @@ calcite: logical: | - LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(age=[$0], max(balance)=[$1]) - LogicalAggregate(group=[{1}], max(balance)=[MAX($0)]) - LogicalProject(balance=[$7], age0=[SPAN($10, 10, null:NULL)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalProject(age=[$0], max(balance)=[$1]) + LogicalAggregate(group=[{1}], max(balance)=[MAX($0)]) + LogicalProject(balance=[$7], age0=[SPAN($10, 10, null:NULL)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableAggregate(group=[{1}], max(balance)=[MAX($0)]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[10], expr#20=[null:NULL], expr#21=[SPAN($t10, $t19, $t20)], balance=[$t7], age0=[$t21]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableAggregate(group=[{1}], max(balance)=[MAX($0)]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[10], expr#20=[null:NULL], expr#21=[SPAN($t10, $t19, $t20)], balance=[$t7], age0=[$t21]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLChartTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLChartTest.java index 2619fce64b6..8f7e09a55b1 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLChartTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLChartTest.java @@ -82,7 +82,8 @@ public void testChartWithSingleGroupKey() { String expectedSparkSql = "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" - + "GROUP BY `gender`"; + + "GROUP BY `gender`\n" + + "ORDER BY `gender` NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -94,7 +95,8 @@ public void testChartWithOverSyntax() { String expectedSparkSql = "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" - + "GROUP BY `gender`"; + + "GROUP BY `gender`\n" + + "ORDER BY `gender` NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -119,7 +121,8 @@ public void testChartWithMultipleGroupKeys() { + "WHERE `age` IS NOT NULL\n" + "GROUP BY `age`) `t7` ON `t1`.`age` = `t7`.`age`\n" + "GROUP BY `t1`.`gender`, CASE WHEN `t1`.`age` IS NULL THEN 'NULL' WHEN" - + " `t7`.`_row_number_chart_` <= 10 THEN `t1`.`age` ELSE 'OTHER' END"; + + " `t7`.`_row_number_chart_` <= 10 THEN `t1`.`age` ELSE 'OTHER' END\n" + + "ORDER BY `t1`.`gender` NULLS LAST, 2 NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -144,7 +147,8 @@ public void testChartWithMultipleGroupKeysAlternativeSyntax() { + "WHERE `age` IS NOT NULL\n" + "GROUP BY `age`) `t7` ON `t1`.`age` = `t7`.`age`\n" + "GROUP BY `t1`.`gender`, CASE WHEN `t1`.`age` IS NULL THEN 'NULL' WHEN" - + " `t7`.`_row_number_chart_` <= 10 THEN `t1`.`age` ELSE 'OTHER' END"; + + " `t7`.`_row_number_chart_` <= 10 THEN `t1`.`age` ELSE 'OTHER' END\n" + + "ORDER BY `t1`.`gender` NULLS LAST, 2 NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -156,7 +160,8 @@ public void testChartWithLimit() { String expectedSparkSql = "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" - + "GROUP BY `gender`"; + + "GROUP BY `gender`\n" + + "ORDER BY `gender` NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -168,7 +173,8 @@ public void testChartWithLimitZero() { String expectedSparkSql = "SELECT `state`, `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" - + "GROUP BY `state`, `gender`"; + + "GROUP BY `state`, `gender`\n" + + "ORDER BY `state` NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -180,7 +186,8 @@ public void testChartWithSpan() { String expectedSparkSql = "SELECT `SPAN`(`age`, 10, NULL) `age`, MAX(`balance`) `max(balance)`\n" + "FROM `scott`.`bank`\n" - + "GROUP BY `SPAN`(`age`, 10, NULL)"; + + "GROUP BY `SPAN`(`age`, 10, NULL)\n" + + "ORDER BY 1 NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -205,7 +212,8 @@ public void testChartWithTimeSpan() { + "WHERE `category` IS NOT NULL\n" + "GROUP BY `category`) `t7` ON `t1`.`category` = `t7`.`category`\n" + "GROUP BY `t1`.`timestamp`, CASE WHEN `t1`.`category` IS NULL THEN 'NULL' WHEN" - + " `t7`.`_row_number_chart_` <= 10 THEN `t1`.`category` ELSE 'OTHER' END"; + + " `t7`.`_row_number_chart_` <= 10 THEN `t1`.`category` ELSE 'OTHER' END\n" + + "ORDER BY `t1`.`timestamp` NULLS LAST, 2 NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -217,7 +225,8 @@ public void testChartWithUseOtherTrue() { String expectedSparkSql = "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" - + "GROUP BY `gender`"; + + "GROUP BY `gender`\n" + + "ORDER BY `gender` NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -229,7 +238,8 @@ public void testChartWithUseOtherFalse() { String expectedSparkSql = "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" - + "GROUP BY `gender`"; + + "GROUP BY `gender`\n" + + "ORDER BY `gender` NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -241,7 +251,8 @@ public void testChartWithOtherStr() { String expectedSparkSql = "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" - + "GROUP BY `gender`"; + + "GROUP BY `gender`\n" + + "ORDER BY `gender` NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -253,7 +264,8 @@ public void testChartWithNullStr() { String expectedSparkSql = "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" - + "GROUP BY `gender`"; + + "GROUP BY `gender`\n" + + "ORDER BY `gender` NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -266,7 +278,8 @@ public void testChartWithUseNull() { "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" + "WHERE `gender` IS NOT NULL\n" - + "GROUP BY `gender`"; + + "GROUP BY `gender`\n" + + "ORDER BY `gender` NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); } From 96e5303a0cffdb138536bd1c783aebc0237f5940 Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Wed, 5 Nov 2025 17:17:08 +0800 Subject: [PATCH 28/35] Ignore rows without a row split in chart command Signed-off-by: Yuanchun Shen --- .../sql/calcite/CalciteRelNodeVisitor.java | 75 +++++++++++-------- docs/user/ppl/cmd/chart.rst | 21 +++--- .../calcite/remote/CalciteChartCommandIT.java | 17 +++++ .../explain_chart_multiple_group_keys.yaml | 12 +-- .../calcite/explain_chart_null_str.yaml | 14 ++-- .../explain_chart_single_group_key.yaml | 5 +- .../calcite/explain_chart_timestamp_span.yaml | 10 ++- .../calcite/explain_chart_use_other.yaml | 10 ++- .../calcite/explain_chart_with_limit.yaml | 5 +- .../calcite/explain_chart_with_span.yaml | 5 +- 10 files changed, 106 insertions(+), 68 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 0512a172469..77f60521496 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -30,6 +30,7 @@ import com.google.common.collect.Streams; import java.util.ArrayList; import java.util.Arrays; +import java.util.BitSet; import java.util.Collections; import java.util.Comparator; import java.util.HashSet; @@ -39,6 +40,7 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.IntStream; import java.util.stream.Stream; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.plan.RelOptTable; @@ -1097,7 +1099,15 @@ private Pair, List> resolveAttributesForAggregation( @Override public RelNode visitAggregation(Aggregation node, CalcitePlanContext context) { - visitAggregation(node, context, true); + Argument.ArgumentMap statsArgs = Argument.ArgumentMap.of(node.getArgExprList()); + Boolean bucketNullable = + (Boolean) statsArgs.getOrDefault(Argument.BUCKET_NULLABLE, Literal.TRUE).getValue(); + int nGroup = node.getGroupExprList().size() + (Objects.nonNull(node.getSpan()) ? 1 : 0); + BitSet nonNullGroupMask = new BitSet(nGroup); + if (!bucketNullable) { + nonNullGroupMask.set(0, nGroup); + } + visitAggregation(node, context, true, nonNullGroupMask); return context.relBuilder.peek(); } @@ -1108,8 +1118,10 @@ public RelNode visitAggregation(Aggregation node, CalcitePlanContext context) { * @param context the Calcite plan context for building RelNodes * @param aggFirst if true, aggregation results (metrics) appear first in output schema (agg, * group-by fields); if false, group expressions appear first (group-by fields, agg). + * @param nonNullGroupMask bit set indicating group by fields that need to be non-null */ - private void visitAggregation(Aggregation node, CalcitePlanContext context, boolean aggFirst) { + private void visitAggregation( + Aggregation node, CalcitePlanContext context, boolean aggFirst, BitSet nonNullGroupMask) { visitChildren(node, context); List aggExprList = node.getAggExprList(); @@ -1119,36 +1131,30 @@ private void visitAggregation(Aggregation node, CalcitePlanContext context, bool UnresolvedExpression span = node.getSpan(); if (Objects.nonNull(span)) { groupExprList.add(span); - List timeSpanFilters = - getTimeSpanField(span).stream() - .map(f -> rexVisitor.analyze(f, context)) - .map(context.relBuilder::isNotNull) - .toList(); - if (!timeSpanFilters.isEmpty()) { - // add isNotNull filter before aggregation for time span - context.relBuilder.filter(timeSpanFilters); + if (getTimeSpanField(span).isPresent()){ + nonNullGroupMask.set(0); } } groupExprList.addAll(node.getGroupExprList()); // add stats hint to LogicalAggregation - Argument.ArgumentMap statsArgs = Argument.ArgumentMap.of(node.getArgExprList()); - Boolean bucketNullable = - (Boolean) statsArgs.getOrDefault(Argument.BUCKET_NULLABLE, Literal.TRUE).getValue(); - boolean toAddHintsOnAggregate = false; - if (!bucketNullable - && !groupExprList.isEmpty() - && !(groupExprList.size() == 1 && getTimeSpanField(span).isPresent())) { - toAddHintsOnAggregate = true; - // add isNotNull filter before aggregation for non-nullable buckets - List groupByList = - groupExprList.stream().map(expr -> rexVisitor.analyze(expr, context)).toList(); - context.relBuilder.filter( - PlanUtils.getSelectColumns(groupByList).stream() - .map(context.relBuilder::field) - .map(context.relBuilder::isNotNull) - .toList()); - } + boolean toAddHintsOnAggregate = + nonNullGroupMask.nextClearBit(0) >= groupExprList.size() // This checks if all group-bys are nonnull + && !groupExprList.isEmpty() + && !(groupExprList.size() == 1 && getTimeSpanField(span).isPresent()); + // add isNotNull filter before aggregation for non-nullable buckets + List groupByList = + groupExprList.stream().map(expr -> rexVisitor.analyze(expr, context)).toList(); + List nonNullGroupBys = + IntStream.range(0, groupByList.size()) + .filter(nonNullGroupMask::get) + .mapToObj(groupByList::get) + .toList(); + context.relBuilder.filter( + PlanUtils.getSelectColumns(nonNullGroupBys).stream() + .map(context.relBuilder::field) + .map(context.relBuilder::isNotNull) + .toList()); Pair, List> aggregationAttributes = aggregateWithTrimming(groupExprList, aggExprList, context, toAddHintsOnAggregate); @@ -2397,12 +2403,15 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { ChartConfig config = ChartConfig.fromArguments(argMap); Aggregation aggregation = new Aggregation( - List.of(node.getAggregationFunction()), - List.of(), - groupExprList, - null, - List.of(new Argument(Argument.BUCKET_NULLABLE, AstDSL.booleanLiteral(config.useNull)))); - visitAggregation(aggregation, context, false); + List.of(node.getAggregationFunction()), List.of(), groupExprList, null, List.of()); + BitSet nonNullGroupMask = new BitSet(groupExprList.size()); + // Rows without a row-split are always ignored + if (config.useNull) { + nonNullGroupMask.set(0); + } else { + nonNullGroupMask.set(0, groupExprList.size()); + } + visitAggregation(aggregation, context, false, nonNullGroupMask); RelBuilder relBuilder = context.relBuilder; String columnSplitName = relBuilder.peek().getRowType().getFieldNames().size() > 2 diff --git a/docs/user/ppl/cmd/chart.rst b/docs/user/ppl/cmd/chart.rst index 760843f3d6e..27ef2140c25 100644 --- a/docs/user/ppl/cmd/chart.rst +++ b/docs/user/ppl/cmd/chart.rst @@ -30,28 +30,29 @@ Syntax **Parameters:** -* **limit**: optional. Specifies the number of distinct values to display when using column split. +* **limit**: optional. Specifies the number of categories to display when using column split. Each unique value in the column split field represents a category. * Default: top10 * Syntax: ``limit=(top|bottom)`` or ``limit=`` (defaults to top) - * When ``limit=K`` is set, the top or bottom K distinct column split values are retained; the additional values are grouped into an "OTHER" category if ``useother`` is not set to false. - * Set limit to 0 to show all distinct values without any limit. - * Use ``limit=topK`` or ``limit=bottomK`` to specify whether to retain the top or bottom K column categories. The ranking is based on the aggregated values for each distinct column-split value. For example, ``chart limit=top3 count() by a b`` retains the 3 most common b categories; ``chart limit=top5 min(value) by a b`` selects the 5 b categories that contains smallest aggregated values. If not specified, top is used by default. - * Only applies when column split presents (by 2 fields or over...by... coexists). + * When ``limit=K`` is set, the top or bottom K categories from the column split field are retained; the remaining categories are grouped into an "OTHER" category if ``useother`` is not set to false. + * Set limit to 0 to show all categories without any limit. + * Use ``limit=topK`` or ``limit=bottomK`` to specify whether to retain the top or bottom K column categories. The ranking is based on the aggregated values for each category. For example, ``chart limit=top3 count() by a b`` retains the 3 most common b categories; ``chart limit=top5 min(value) by a b`` selects the 5 b categories that contain the smallest aggregated values. If not specified, top is used by default. + * Only applies when column split is present (by 2 fields or over...by... coexists). -* **useother**: optional. Controls whether to create an "OTHER" category for distinct column-split values beyond the limit. +* **useother**: optional. Controls whether to create an "OTHER" category for categories beyond the limit. * Default: true - * When set to false, only the top/bottom N distinct column-split values (based on limit) are shown without an "OTHER" category. - * When set to true, distinct values beyond the limit are grouped into an "OTHER" category. - * Only applies when using column split and when there are more distinct column-split values than the limit. + * When set to false, only the top/bottom N categories (based on limit) are shown without an "OTHER" category. + * When set to true, categories beyond the limit are grouped into an "OTHER" category. + * Only applies when using column split and when there are more categories than the limit. * **usenull**: optional. Controls whether to group events without a column split (i.e. whose column split is null) into a separate "NULL" category. * Default: true + * ``usenull`` only applies to column split. + * Row split should always be non-null value. Events with null values in row split will be ignored. * When ``usenull=false``, events with a null column split are excluded from results. * When ``usenull=true``, events with a null column split are grouped into a separate "NULL" category. - * ``usenull`` only applies to column split. Null values in the row split are handled in the same way as normal aggregations. * **nullstr**: optional. Specifies the category name for rows that do not contain the column split value. diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java index 46b067ac9d7..c66891b2631 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java @@ -314,4 +314,21 @@ public void testChartUseNullFalseWithNullStr() throws IOException { result, schema("gender", "string"), schema("age", "string"), schema("count()", "bigint")); verifyDataRows(result, rows("M", "30", 4), rows("F", "30", 1), rows("F", "20", 1)); } + + @Test + public void testChartNullsInRowSplitShouldBeIgnored() throws IOException { + JSONObject result = + executeQuery( + "source=events_null | chart min(cpu_usage) by host region"); + verifySchema( + result, + schema("host", "string"), + schema("region", "string"), + schema("min(cpu_usage)", "double")); + verifyDataRows( + result, + rows("db-01", "eu-west", 42.1), + rows("web-01", "us-east", 45.2), + rows("web-02", "us-west", 38.7)); + } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_multiple_group_keys.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_multiple_group_keys.yaml index d619f0b89f6..191881f3021 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_multiple_group_keys.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_multiple_group_keys.yaml @@ -8,14 +8,16 @@ calcite: LogicalProject(gender=[$0], age=[SAFE_CAST($1)], avg(balance)=[$2]) LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) LogicalProject(gender=[$4], age=[$10], balance=[$7]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalFilter(condition=[IS NOT NULL($4)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) LogicalProject(age=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) LogicalAggregate(group=[{0}], __grand_total__=[AVG($1)]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) LogicalProject(gender=[$4], age=[$10], balance=[$7]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalFilter(condition=[IS NOT NULL($4)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) @@ -25,11 +27,11 @@ calcite: EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) EnumerableSort(sort0=[$1], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[SAFE_CAST($t1)], gender=[$t0], age=[$t3], avg(balance)=[$t2]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[gender, balance, age], FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"gender","boost":1.0}},"_source":{"includes":["gender","balance","age"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:DOUBLE], expr#6=[CASE($t4, $t5, $t1)], expr#7=[/($t6, $t2)], age=[$t0], __grand_total__=[$t7]) EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[SAFE_CAST($t0)], expr#3=[IS NOT NULL($t2)], $f0=[$t2], avg(balance)=[$t1], $condition=[$t3]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2)), PROJECT->[age, avg(balance)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[SAFE_CAST($t0)], expr#3=[IS NOT NULL($t2)], age=[$t2], avg(balance)=[$t1], $condition=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[gender, balance, age], FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2)), PROJECT->[age, avg(balance)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"gender","boost":1.0}},"_source":{"includes":["gender","balance","age"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_null_str.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_null_str.yaml index 2fa1a05e515..4e1e51b74a6 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_null_str.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_null_str.yaml @@ -8,14 +8,16 @@ calcite: LogicalProject(gender=[$0], age=[SAFE_CAST($1)], avg(balance)=[$2]) LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) + LogicalFilter(condition=[IS NOT NULL($4)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) LogicalProject(age=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) LogicalAggregate(group=[{0}], __grand_total__=[AVG($1)]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) + LogicalFilter(condition=[IS NOT NULL($4)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) @@ -26,8 +28,8 @@ calcite: EnumerableSort(sort0=[$1], dir0=[ASC]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t10]) EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], proj#0..1=[{exprs}], $f2=[$t5]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[gender, balance, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","balance","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], gender=[$t1], balance=[$t0], age0=[$t5]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->IS NOT NULL($1)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"exists":{"field":"gender","boost":1.0}},"_source":{"includes":["balance","gender","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) @@ -35,5 +37,5 @@ calcite: EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], expr#11=[IS NOT NULL($t4)], age=[$t4], avg(balance)=[$t10], $condition=[$t11]) EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], proj#0..1=[{exprs}], $f2=[$t5]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[gender, balance, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","balance","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], gender=[$t1], balance=[$t0], age0=[$t5]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->IS NOT NULL($1)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"exists":{"field":"gender","boost":1.0}},"_source":{"includes":["balance","gender","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_single_group_key.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_single_group_key.yaml index 8ec7422de37..ca725256ece 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_single_group_key.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_single_group_key.yaml @@ -4,6 +4,7 @@ calcite: LogicalSort(sort0=[$0], dir0=[ASC]) LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)]) LogicalProject(gender=[$4], balance=[$7]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalFilter(condition=[IS NOT NULL($4)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg(balance)=AVG($1)), SORT->[0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"last","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg(balance)=AVG($1)), SORT->[0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_timestamp_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_timestamp_span.yaml index 1233817e40f..1608f8cbb6e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_timestamp_span.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_timestamp_span.yaml @@ -8,22 +8,24 @@ calcite: LogicalProject(timestamp=[$1], category=[$0], max(value)=[$2]) LogicalAggregate(group=[{0, 2}], max(value)=[MAX($1)]) LogicalProject(category=[$1], value=[$2], timestamp0=[SPAN($3, 1, 'w')]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + LogicalFilter(condition=[IS NOT NULL($3)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) LogicalProject(category=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) LogicalAggregate(group=[{0}], __grand_total__=[MAX($1)]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(category=[$0], max(value)=[$2]) LogicalAggregate(group=[{0, 2}], max(value)=[MAX($1)]) LogicalProject(category=[$1], value=[$2], timestamp0=[SPAN($3, 1, 'w')]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + LogicalFilter(condition=[IS NOT NULL($3)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) EnumerableAggregate(group=[{0, 1}], max(value)=[MAX($2)]) EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], timestamp=[$t0], category=[$t10], max(value)=[$t2]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},max(value)=MAX($1)), PROJECT->[timestamp0, category, max(value)], SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"timestamp0":{"date_histogram":{"field":"timestamp","missing_bucket":false,"order":"asc","calendar_interval":"1w"}}}]},"aggregations":{"max(value)":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[PROJECT->[category, value, timestamp], FILTER->IS NOT NULL($2), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},max(value)=MAX($1)), PROJECT->[timestamp0, category, max(value)], SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"timestamp","boost":1.0}},"_source":{"includes":["category","value","timestamp"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"timestamp0":{"date_histogram":{"field":"timestamp","missing_bucket":false,"order":"asc","calendar_interval":"1w"}}}]},"aggregations":{"max(value)":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], category=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[FILTER->IS NOT NULL($1), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},__grand_total__=MAX($1))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"category","boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"__grand_total__":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[PROJECT->[category, value, timestamp], FILTER->IS NOT NULL($2), FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},__grand_total__=MAX($1))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"filter":[{"exists":{"field":"timestamp","boost":1.0}},{"exists":{"field":"category","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["category","value","timestamp"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"__grand_total__":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_use_other.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_use_other.yaml index 7916a31a5d8..d60fd950950 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_use_other.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_use_other.yaml @@ -7,22 +7,24 @@ calcite: LogicalJoin(condition=[=($1, $3)], joinType=[left]) LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) + LogicalFilter(condition=[IS NOT NULL($23)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) LogicalProject(severityText=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) LogicalAggregate(group=[{0}], __grand_total__=[MAX($1)]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(severityText=[$1], max(severityNumber)=[$2]) LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) + LogicalFilter(condition=[IS NOT NULL($23)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) EnumerableAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[2], expr#8=[<=($t4, $t7)], expr#9=['max_among_other'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], flags=[$t0], severityText=[$t10], max(severityNumber)=[$t2]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},max(severityNumber)=MAX($2)), SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"severityText":{"terms":{"field":"severityText","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"flags":{"terms":{"field":"flags","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"max(severityNumber)":{"max":{"field":"severityNumber"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[PROJECT->[severityText, flags, severityNumber], FILTER->IS NOT NULL($1), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},max(severityNumber)=MAX($2)), SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"flags","boost":1.0}},"_source":{"includes":["severityText","flags","severityNumber"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"severityText":{"terms":{"field":"severityText","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"flags":{"terms":{"field":"flags","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"max(severityNumber)":{"max":{"field":"severityNumber"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], severityText=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[FILTER->IS NOT NULL($7), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},__grand_total__=MAX($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"severityText","boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"severityText":{"terms":{"field":"severityText","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"__grand_total__":{"max":{"field":"severityNumber"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[PROJECT->[severityText, flags, severityNumber], FILTER->IS NOT NULL($1), FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},__grand_total__=MAX($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"filter":[{"exists":{"field":"flags","boost":1.0}},{"exists":{"field":"severityText","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["severityText","flags","severityNumber"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"severityText":{"terms":{"field":"severityText","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"__grand_total__":{"max":{"field":"severityNumber"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_limit.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_limit.yaml index 0b4ea1ad9c8..533e9d89a5e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_limit.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_limit.yaml @@ -4,6 +4,7 @@ calcite: LogicalSort(sort0=[$0], dir0=[ASC]) LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) LogicalProject(state=[$9], gender=[$4], balance=[$7]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalFilter(condition=[IS NOT NULL($9)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2)), SORT->[0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[gender, balance, state], FILTER->IS NOT NULL($2), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2)), SORT->[0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"state","boost":1.0}},"_source":{"includes":["gender","balance","state"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_span.yaml index 73039fe4ed5..fce35663e64 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_span.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_span.yaml @@ -5,6 +5,7 @@ calcite: LogicalProject(age=[$0], max(balance)=[$1]) LogicalAggregate(group=[{1}], max(balance)=[MAX($0)]) LogicalProject(balance=[$7], age0=[SPAN($10, 10, null:NULL)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalFilter(condition=[IS NOT NULL($10)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},max(balance)=MAX($0)), SORT->[0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age0":{"histogram":{"field":"age","missing_bucket":true,"missing_order":"last","order":"asc","interval":10.0}}}]},"aggregations":{"max(balance)":{"max":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[balance, age], FILTER->IS NOT NULL($1), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},max(balance)=MAX($0)), SORT->[0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"age","boost":1.0}},"_source":{"includes":["balance","age"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age0":{"histogram":{"field":"age","missing_bucket":false,"order":"asc","interval":10.0}}}]},"aggregations":{"max(balance)":{"max":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) From 21667c7bb4894e7cbd1d08c90ac8b7f5366cd58b Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Wed, 5 Nov 2025 17:34:19 +0800 Subject: [PATCH 29/35] Keep categories with max summed values when top k is set Signed-off-by: Yuanchun Shen --- .../sql/calcite/CalciteRelNodeVisitor.java | 23 +++--- .../calcite/remote/CalciteChartCommandIT.java | 47 +++++------ .../explain_chart_multiple_group_keys.yaml | 9 +-- .../calcite/explain_chart_null_str.yaml | 13 ++- .../calcite/explain_chart_timestamp_span.yaml | 5 +- .../calcite/explain_chart_use_other.yaml | 5 +- .../explain_chart_multiple_group_keys.yaml | 19 +++-- .../explain_chart_null_str.yaml | 21 ++--- .../explain_chart_single_group_key.yaml | 6 +- .../explain_chart_timestamp_span.yaml | 17 ++-- .../explain_chart_use_other.yaml | 17 ++-- .../explain_chart_with_limit.yaml | 6 +- .../explain_chart_with_span.yaml | 5 +- .../sql/ppl/calcite/CalcitePPLChartTest.java | 81 +++++++++++-------- 14 files changed, 150 insertions(+), 124 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 77f60521496..05d0e22508b 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -1131,15 +1131,16 @@ private void visitAggregation( UnresolvedExpression span = node.getSpan(); if (Objects.nonNull(span)) { groupExprList.add(span); - if (getTimeSpanField(span).isPresent()){ - nonNullGroupMask.set(0); + if (getTimeSpanField(span).isPresent()) { + nonNullGroupMask.set(0); } } groupExprList.addAll(node.getGroupExprList()); // add stats hint to LogicalAggregation boolean toAddHintsOnAggregate = - nonNullGroupMask.nextClearBit(0) >= groupExprList.size() // This checks if all group-bys are nonnull + nonNullGroupMask.nextClearBit(0) + >= groupExprList.size() // This checks if all group-bys should be nonnull && !groupExprList.isEmpty() && !(groupExprList.size() == 1 && getTimeSpanField(span).isPresent()); // add isNotNull filter before aggregation for non-nullable buckets @@ -2413,10 +2414,6 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { } visitAggregation(aggregation, context, false, nonNullGroupMask); RelBuilder relBuilder = context.relBuilder; - String columnSplitName = - relBuilder.peek().getRowType().getFieldNames().size() > 2 - ? relBuilder.peek().getRowType().getFieldNames().get(1) - : null; // If row or column split does not present or limit equals 0, this is the same as `stats agg // [group by col]` because all truncating is performed on the column split @@ -2457,16 +2454,13 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { final String GRAND_TOTAL_COL = "__grand_total__"; relBuilder.aggregate( relBuilder.groupKey(relBuilder.field(0)), - buildAggCall(context.relBuilder, aggFunction, relBuilder.field(1)) - .as(GRAND_TOTAL_COL)); // results: group key, agg calls + // Top-K semantic: Retain categories whose summed values are among the greatest + relBuilder.sum(relBuilder.field(1)).as(GRAND_TOTAL_COL)); // results: group key, agg calls RexNode grandTotal = relBuilder.field(GRAND_TOTAL_COL); - // Apply sorting: for MIN/EARLIEST, reverse the top/bottom logic - boolean smallestFirst = - aggFunction == BuiltinFunctionName.MIN || aggFunction == BuiltinFunctionName.EARLIEST; - if (config.top != smallestFirst) { + // Apply sorting: keep the max values if top is set + if (config.top) { grandTotal = relBuilder.desc(grandTotal); } - // Always set it to null last so that it does not interfere with top / bottom calculation grandTotal = relBuilder.nullsLast(grandTotal); RexNode rowNum = @@ -2518,6 +2512,7 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { relBuilder.literal(config.otherStr)); } + String columnSplitName = ((Alias) node.getColumnSplit()).getName(); String aggFieldName = relBuilder.peek().getRowType().getFieldNames().get(2); relBuilder.project( relBuilder.field(0), diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java index c66891b2631..dee2d4c7776 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java @@ -13,6 +13,7 @@ import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRowsInOrder; import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; @@ -221,10 +222,11 @@ public void testChartLimitTopWithUseOther() throws IOException { schema("max(severityNumber)", "bigint")); verifyDataRows( result, - rows(1, "max_among_other", 17), - rows(0, "max_among_other", 22), - rows(0, "FATAL3", 23), - rows(0, "FATAL4", 24)); + rows(0, "ERROR", 17), + rows(0, "FATAL4", 24), + rows(0, "max_among_other", 23), + rows(1, "ERROR", 17), + rows(1, "max_among_other", 9)); } @Test @@ -255,12 +257,13 @@ public void testChartLimitTopWithMinAgg() throws IOException { schema("flags", "bigint"), schema("severityText", "string"), schema("min(severityNumber)", "bigint")); - verifyDataRows( + verifyDataRowsInOrder( result, - rows(1, "OTHER", 9), - rows(1, "TRACE", 1), - rows(0, "OTHER", 3), - rows(0, "TRACE2", 2)); + rows(0, "ERROR", 17), + rows(0, "FATAL4", 24), + rows(0, "OTHER", 2), + rows(1, "ERROR", 17), + rows(1, "OTHER", 1)); } @Test @@ -316,19 +319,17 @@ public void testChartUseNullFalseWithNullStr() throws IOException { } @Test - public void testChartNullsInRowSplitShouldBeIgnored() throws IOException { - JSONObject result = - executeQuery( - "source=events_null | chart min(cpu_usage) by host region"); - verifySchema( - result, - schema("host", "string"), - schema("region", "string"), - schema("min(cpu_usage)", "double")); - verifyDataRows( - result, - rows("db-01", "eu-west", 42.1), - rows("web-01", "us-east", 45.2), - rows("web-02", "us-west", 38.7)); + public void testChartNullsInRowSplitShouldBeIgnored() throws IOException { + JSONObject result = executeQuery("source=events_null | chart min(cpu_usage) by host region"); + verifySchema( + result, + schema("host", "string"), + schema("region", "string"), + schema("min(cpu_usage)", "double")); + verifyDataRows( + result, + rows("db-01", "eu-west", 42.1), + rows("web-01", "us-east", 45.2), + rows("web-02", "us-west", 38.7)); } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_multiple_group_keys.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_multiple_group_keys.yaml index 191881f3021..5f7db3b2fdd 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_multiple_group_keys.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_multiple_group_keys.yaml @@ -11,7 +11,7 @@ calcite: LogicalFilter(condition=[IS NOT NULL($4)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) LogicalProject(age=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) - LogicalAggregate(group=[{0}], __grand_total__=[AVG($1)]) + LogicalAggregate(group=[{0}], __grand_total__=[SUM($1)]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) @@ -31,7 +31,6 @@ calcite: EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:DOUBLE], expr#6=[CASE($t4, $t5, $t1)], expr#7=[/($t6, $t2)], age=[$t0], __grand_total__=[$t7]) - EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[SAFE_CAST($t0)], expr#3=[IS NOT NULL($t2)], age=[$t2], avg(balance)=[$t1], $condition=[$t3]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[gender, balance, age], FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2)), PROJECT->[age, avg(balance)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"gender","boost":1.0}},"_source":{"includes":["gender","balance","age"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[SAFE_CAST($t0)], expr#3=[IS NOT NULL($t2)], age=[$t2], avg(balance)=[$t1], $condition=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[gender, balance, age], FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2)), PROJECT->[age, avg(balance)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"gender","boost":1.0}},"_source":{"includes":["gender","balance","age"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_null_str.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_null_str.yaml index 4e1e51b74a6..a750eea8287 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_null_str.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_null_str.yaml @@ -11,7 +11,7 @@ calcite: LogicalFilter(condition=[IS NOT NULL($4)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) LogicalProject(age=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) - LogicalAggregate(group=[{0}], __grand_total__=[AVG($1)]) + LogicalAggregate(group=[{0}], __grand_total__=[SUM($1)]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) @@ -33,9 +33,8 @@ calcite: EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:DOUBLE], expr#6=[CASE($t4, $t5, $t1)], expr#7=[/($t6, $t2)], age=[$t0], __grand_total__=[$t7]) - EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], expr#11=[IS NOT NULL($t4)], age=[$t4], avg(balance)=[$t10], $condition=[$t11]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], gender=[$t1], balance=[$t0], age0=[$t5]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->IS NOT NULL($1)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"exists":{"field":"gender","boost":1.0}},"_source":{"includes":["balance","gender","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], expr#11=[IS NOT NULL($t4)], age=[$t4], avg(balance)=[$t10], $condition=[$t11]) + EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], gender=[$t1], balance=[$t0], age0=[$t5]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->IS NOT NULL($1)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"exists":{"field":"gender","boost":1.0}},"_source":{"includes":["balance","gender","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_timestamp_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_timestamp_span.yaml index 1608f8cbb6e..a592dfc4e22 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_timestamp_span.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_timestamp_span.yaml @@ -11,7 +11,7 @@ calcite: LogicalFilter(condition=[IS NOT NULL($3)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) LogicalProject(category=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) - LogicalAggregate(group=[{0}], __grand_total__=[MAX($1)]) + LogicalAggregate(group=[{0}], __grand_total__=[SUM($1)]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(category=[$0], max(value)=[$2]) LogicalAggregate(group=[{0, 2}], max(value)=[MAX($1)]) @@ -28,4 +28,5 @@ calcite: EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], category=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[PROJECT->[category, value, timestamp], FILTER->IS NOT NULL($2), FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},__grand_total__=MAX($1))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"filter":[{"exists":{"field":"timestamp","boost":1.0}},{"exists":{"field":"category","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["category","value","timestamp"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"__grand_total__":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[PROJECT->[category, value, timestamp], FILTER->IS NOT NULL($2), FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},max(value)=MAX($1)), PROJECT->[category, max(value)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"filter":[{"exists":{"field":"timestamp","boost":1.0}},{"exists":{"field":"category","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["category","value","timestamp"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"timestamp0":{"date_histogram":{"field":"timestamp","missing_bucket":false,"order":"asc","calendar_interval":"1w"}}}]},"aggregations":{"max(value)":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_use_other.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_use_other.yaml index d60fd950950..67fdea2fc97 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_use_other.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_use_other.yaml @@ -10,7 +10,7 @@ calcite: LogicalFilter(condition=[IS NOT NULL($23)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) LogicalProject(severityText=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) - LogicalAggregate(group=[{0}], __grand_total__=[MAX($1)]) + LogicalAggregate(group=[{0}], __grand_total__=[SUM($1)]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(severityText=[$1], max(severityNumber)=[$2]) LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) @@ -27,4 +27,5 @@ calcite: EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], severityText=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[PROJECT->[severityText, flags, severityNumber], FILTER->IS NOT NULL($1), FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},__grand_total__=MAX($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"filter":[{"exists":{"field":"flags","boost":1.0}},{"exists":{"field":"severityText","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["severityText","flags","severityNumber"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"severityText":{"terms":{"field":"severityText","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"__grand_total__":{"max":{"field":"severityNumber"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[PROJECT->[severityText, flags, severityNumber], FILTER->IS NOT NULL($1), FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},max(severityNumber)=MAX($2)), PROJECT->[severityText, max(severityNumber)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"filter":[{"exists":{"field":"flags","boost":1.0}},{"exists":{"field":"severityText","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["severityText","flags","severityNumber"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"flags":{"terms":{"field":"flags","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"severityText":{"terms":{"field":"severityText","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"max(severityNumber)":{"max":{"field":"severityNumber"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_group_keys.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_group_keys.yaml index c9a7377f629..21d605b815a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_group_keys.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_group_keys.yaml @@ -8,14 +8,16 @@ calcite: LogicalProject(gender=[$0], age=[SAFE_CAST($1)], avg(balance)=[$2]) LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) LogicalProject(gender=[$4], age=[$10], balance=[$7]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalFilter(condition=[IS NOT NULL($4)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) LogicalProject(age=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) - LogicalAggregate(group=[{0}], __grand_total__=[AVG($1)]) + LogicalAggregate(group=[{0}], __grand_total__=[SUM($1)]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) LogicalProject(gender=[$4], age=[$10], balance=[$7]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalFilter(condition=[IS NOT NULL($4)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) @@ -26,12 +28,13 @@ calcite: EnumerableSort(sort0=[$1], dir0=[ASC]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t10]) EnumerableAggregate(group=[{4, 10}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t4)], proj#0..18=[{exprs}], $condition=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:DOUBLE], expr#6=[CASE($t4, $t5, $t1)], expr#7=[/($t6, $t2)], age=[$t0], __grand_total__=[$t7]) - EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], expr#11=[IS NOT NULL($t4)], age=[$t4], avg(balance)=[$t10], $condition=[$t11]) - EnumerableAggregate(group=[{4, 10}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) + EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], age=[$t4], avg(balance)=[$t10]) + EnumerableAggregate(group=[{4, 10}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t4)], expr#20=[SAFE_CAST($t10)], expr#21=[IS NOT NULL($t20)], expr#22=[AND($t19, $t21)], proj#0..18=[{exprs}], $condition=[$t22]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_null_str.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_null_str.yaml index 72d29bd3691..a0a499e805f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_null_str.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_null_str.yaml @@ -8,14 +8,16 @@ calcite: LogicalProject(gender=[$0], age=[SAFE_CAST($1)], avg(balance)=[$2]) LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) + LogicalFilter(condition=[IS NOT NULL($4)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) LogicalProject(age=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) - LogicalAggregate(group=[{0}], __grand_total__=[AVG($1)]) + LogicalAggregate(group=[{0}], __grand_total__=[SUM($1)]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) + LogicalFilter(condition=[IS NOT NULL($4)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) @@ -26,14 +28,13 @@ calcite: EnumerableSort(sort0=[$1], dir0=[ASC]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t10]) EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..12=[{inputs}], expr#13=[10], expr#14=[null:NULL], expr#15=[SPAN($t5, $t13, $t14)], gender=[$t4], balance=[$t3], age0=[$t15]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[10], expr#14=[null:NULL], expr#15=[SPAN($t5, $t13, $t14)], expr#16=[IS NOT NULL($t4)], gender=[$t4], balance=[$t3], age0=[$t15], $condition=[$t16]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:DOUBLE], expr#6=[CASE($t4, $t5, $t1)], expr#7=[/($t6, $t2)], age=[$t0], __grand_total__=[$t7]) - EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], expr#11=[IS NOT NULL($t4)], age=[$t4], avg(balance)=[$t10], $condition=[$t11]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..12=[{inputs}], expr#13=[10], expr#14=[null:NULL], expr#15=[SPAN($t5, $t13, $t14)], gender=[$t4], balance=[$t3], age0=[$t15]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) + EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], age=[$t4], avg(balance)=[$t10]) + EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[10], expr#14=[null:NULL], expr#15=[SPAN($t5, $t13, $t14)], expr#16=[IS NOT NULL($t4)], expr#17=[SAFE_CAST($t15)], expr#18=[IS NOT NULL($t17)], expr#19=[AND($t16, $t18)], gender=[$t4], balance=[$t3], age0=[$t15], $condition=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_single_group_key.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_single_group_key.yaml index 9370110fa0e..b711ae1feed 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_single_group_key.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_single_group_key.yaml @@ -4,10 +4,12 @@ calcite: LogicalSort(sort0=[$0], dir0=[ASC]) LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)]) LogicalProject(gender=[$4], balance=[$7]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalFilter(condition=[IS NOT NULL($4)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], gender=[$t0], avg(balance)=[$t8]) EnumerableAggregate(group=[{4}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t4)], proj#0..18=[{exprs}], $condition=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_timestamp_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_timestamp_span.yaml index e0826fddcea..7e968b31dbb 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_timestamp_span.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_timestamp_span.yaml @@ -8,14 +8,16 @@ calcite: LogicalProject(timestamp=[$1], category=[$0], max(value)=[$2]) LogicalAggregate(group=[{0, 2}], max(value)=[MAX($1)]) LogicalProject(category=[$1], value=[$2], timestamp0=[SPAN($3, 1, 'w')]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + LogicalFilter(condition=[IS NOT NULL($3)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) LogicalProject(category=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) - LogicalAggregate(group=[{0}], __grand_total__=[MAX($1)]) + LogicalAggregate(group=[{0}], __grand_total__=[SUM($1)]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(category=[$0], max(value)=[$2]) LogicalAggregate(group=[{0, 2}], max(value)=[MAX($1)]) LogicalProject(category=[$1], value=[$2], timestamp0=[SPAN($3, 1, 'w')]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + LogicalFilter(condition=[IS NOT NULL($3)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) @@ -25,11 +27,12 @@ calcite: EnumerableSort(sort0=[$1], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], timestamp=[$t1], category=[$t0], max(value)=[$t2]) EnumerableAggregate(group=[{0, 2}], max(value)=[MAX($1)]) - EnumerableCalc(expr#0..9=[{inputs}], expr#10=[1], expr#11=['w'], expr#12=[SPAN($t3, $t10, $t11)], category=[$t1], value=[$t2], timestamp0=[$t12]) + EnumerableCalc(expr#0..9=[{inputs}], expr#10=[1], expr#11=['w'], expr#12=[SPAN($t3, $t10, $t11)], expr#13=[IS NOT NULL($t3)], category=[$t1], value=[$t2], timestamp0=[$t12], $condition=[$t13]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], category=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - EnumerableAggregate(group=[{1}], __grand_total__=[MAX($2)]) - EnumerableCalc(expr#0..9=[{inputs}], expr#10=[IS NOT NULL($t1)], proj#0..9=[{exprs}], $condition=[$t10]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + EnumerableAggregate(group=[{0}], __grand_total__=[SUM($2)]) + EnumerableAggregate(group=[{0, 2}], max(value)=[MAX($1)]) + EnumerableCalc(expr#0..9=[{inputs}], expr#10=[1], expr#11=['w'], expr#12=[SPAN($t3, $t10, $t11)], expr#13=[IS NOT NULL($t3)], expr#14=[IS NOT NULL($t1)], expr#15=[AND($t13, $t14)], category=[$t1], value=[$t2], timestamp0=[$t12], $condition=[$t15]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_use_other.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_use_other.yaml index 8e36c39627f..c5fe280b32d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_use_other.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_use_other.yaml @@ -7,14 +7,16 @@ calcite: LogicalJoin(condition=[=($1, $3)], joinType=[left]) LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) + LogicalFilter(condition=[IS NOT NULL($23)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) LogicalProject(severityText=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) - LogicalAggregate(group=[{0}], __grand_total__=[MAX($1)]) + LogicalAggregate(group=[{0}], __grand_total__=[SUM($1)]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(severityText=[$1], max(severityNumber)=[$2]) LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) + LogicalFilter(condition=[IS NOT NULL($23)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) @@ -24,11 +26,12 @@ calcite: EnumerableSort(sort0=[$1], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], flags=[$t1], severityText=[$t0], max(severityNumber)=[$t2]) EnumerableAggregate(group=[{7, 23}], max(severityNumber)=[MAX($163)]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) + EnumerableCalc(expr#0..171=[{inputs}], expr#172=[IS NOT NULL($t23)], proj#0..171=[{exprs}], $condition=[$t172]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], severityText=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - EnumerableAggregate(group=[{0}], __grand_total__=[MAX($2)]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t0)], proj#0..2=[{exprs}], $condition=[$t3]) - EnumerableAggregate(group=[{7, 23}], max(severityNumber)=[MAX($163)]) + EnumerableAggregate(group=[{0}], __grand_total__=[SUM($2)]) + EnumerableAggregate(group=[{7, 23}], max(severityNumber)=[MAX($163)]) + EnumerableCalc(expr#0..171=[{inputs}], expr#172=[IS NOT NULL($t23)], expr#173=[IS NOT NULL($t7)], expr#174=[AND($t172, $t173)], proj#0..171=[{exprs}], $condition=[$t174]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_limit.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_limit.yaml index b51f7589cf3..84ba13a3217 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_limit.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_limit.yaml @@ -4,10 +4,12 @@ calcite: LogicalSort(sort0=[$0], dir0=[ASC]) LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) LogicalProject(state=[$9], gender=[$4], balance=[$7]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalFilter(condition=[IS NOT NULL($9)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], state=[$t1], gender=[$t0], avg(balance)=[$t9]) EnumerableAggregate(group=[{4, 9}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t9)], proj#0..18=[{exprs}], $condition=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_span.yaml index 77129b1530d..651934bf035 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_span.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_span.yaml @@ -5,10 +5,11 @@ calcite: LogicalProject(age=[$0], max(balance)=[$1]) LogicalAggregate(group=[{1}], max(balance)=[MAX($0)]) LogicalProject(balance=[$7], age0=[SPAN($10, 10, null:NULL)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalFilter(condition=[IS NOT NULL($10)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableAggregate(group=[{1}], max(balance)=[MAX($0)]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[10], expr#20=[null:NULL], expr#21=[SPAN($t10, $t19, $t20)], balance=[$t7], age0=[$t21]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[10], expr#20=[null:NULL], expr#21=[SPAN($t10, $t19, $t20)], expr#22=[IS NOT NULL($t10)], balance=[$t7], age0=[$t21], $condition=[$t22]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLChartTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLChartTest.java index 8f7e09a55b1..6f8873571d5 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLChartTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLChartTest.java @@ -82,6 +82,7 @@ public void testChartWithSingleGroupKey() { String expectedSparkSql = "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" + + "WHERE `gender` IS NOT NULL\n" + "GROUP BY `gender`\n" + "ORDER BY `gender` NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -95,6 +96,7 @@ public void testChartWithOverSyntax() { String expectedSparkSql = "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" + + "WHERE `gender` IS NOT NULL\n" + "GROUP BY `gender`\n" + "ORDER BY `gender` NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -106,23 +108,25 @@ public void testChartWithMultipleGroupKeys() { RelNode root = getRelNode(ppl); String expectedSparkSql = - "SELECT `t1`.`gender`, CASE WHEN `t1`.`age` IS NULL THEN 'NULL' WHEN" - + " `t7`.`_row_number_chart_` <= 10 THEN `t1`.`age` ELSE 'OTHER' END `age`," - + " AVG(`t1`.`avg(balance)`) `avg(balance)`\n" + "SELECT `t2`.`gender`, CASE WHEN `t2`.`age` IS NULL THEN 'NULL' WHEN" + + " `t9`.`_row_number_chart_` <= 10 THEN `t2`.`age` ELSE 'OTHER' END `age`," + + " AVG(`t2`.`avg(balance)`) `avg(balance)`\n" + "FROM (SELECT `gender`, SAFE_CAST(`age` AS STRING) `age`, AVG(`balance`)" + " `avg(balance)`\n" + "FROM `scott`.`bank`\n" - + "GROUP BY `gender`, `age`) `t1`\n" - + "LEFT JOIN (SELECT `age`, AVG(`avg(balance)`) `__grand_total__`, ROW_NUMBER() OVER" - + " (ORDER BY AVG(`avg(balance)`) DESC) `_row_number_chart_`\n" + + "WHERE `gender` IS NOT NULL\n" + + "GROUP BY `gender`, `age`) `t2`\n" + + "LEFT JOIN (SELECT `age`, SUM(`avg(balance)`) `__grand_total__`, ROW_NUMBER() OVER" + + " (ORDER BY SUM(`avg(balance)`) DESC) `_row_number_chart_`\n" + "FROM (SELECT SAFE_CAST(`age` AS STRING) `age`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" - + "GROUP BY `gender`, `age`) `t4`\n" + + "WHERE `gender` IS NOT NULL\n" + + "GROUP BY `gender`, `age`) `t6`\n" + "WHERE `age` IS NOT NULL\n" - + "GROUP BY `age`) `t7` ON `t1`.`age` = `t7`.`age`\n" - + "GROUP BY `t1`.`gender`, CASE WHEN `t1`.`age` IS NULL THEN 'NULL' WHEN" - + " `t7`.`_row_number_chart_` <= 10 THEN `t1`.`age` ELSE 'OTHER' END\n" - + "ORDER BY `t1`.`gender` NULLS LAST, 2 NULLS LAST"; + + "GROUP BY `age`) `t9` ON `t2`.`age` = `t9`.`age`\n" + + "GROUP BY `t2`.`gender`, CASE WHEN `t2`.`age` IS NULL THEN 'NULL' WHEN" + + " `t9`.`_row_number_chart_` <= 10 THEN `t2`.`age` ELSE 'OTHER' END\n" + + "ORDER BY `t2`.`gender` NULLS LAST, 2 NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -132,23 +136,25 @@ public void testChartWithMultipleGroupKeysAlternativeSyntax() { RelNode root = getRelNode(ppl); String expectedSparkSql = - "SELECT `t1`.`gender`, CASE WHEN `t1`.`age` IS NULL THEN 'NULL' WHEN" - + " `t7`.`_row_number_chart_` <= 10 THEN `t1`.`age` ELSE 'OTHER' END `age`," - + " AVG(`t1`.`avg(balance)`) `avg(balance)`\n" + "SELECT `t2`.`gender`, CASE WHEN `t2`.`age` IS NULL THEN 'NULL' WHEN" + + " `t9`.`_row_number_chart_` <= 10 THEN `t2`.`age` ELSE 'OTHER' END `age`," + + " AVG(`t2`.`avg(balance)`) `avg(balance)`\n" + "FROM (SELECT `gender`, SAFE_CAST(`age` AS STRING) `age`, AVG(`balance`)" + " `avg(balance)`\n" + "FROM `scott`.`bank`\n" - + "GROUP BY `gender`, `age`) `t1`\n" - + "LEFT JOIN (SELECT `age`, AVG(`avg(balance)`) `__grand_total__`, ROW_NUMBER() OVER" - + " (ORDER BY AVG(`avg(balance)`) DESC) `_row_number_chart_`\n" + + "WHERE `gender` IS NOT NULL\n" + + "GROUP BY `gender`, `age`) `t2`\n" + + "LEFT JOIN (SELECT `age`, SUM(`avg(balance)`) `__grand_total__`, ROW_NUMBER() OVER" + + " (ORDER BY SUM(`avg(balance)`) DESC) `_row_number_chart_`\n" + "FROM (SELECT SAFE_CAST(`age` AS STRING) `age`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" - + "GROUP BY `gender`, `age`) `t4`\n" + + "WHERE `gender` IS NOT NULL\n" + + "GROUP BY `gender`, `age`) `t6`\n" + "WHERE `age` IS NOT NULL\n" - + "GROUP BY `age`) `t7` ON `t1`.`age` = `t7`.`age`\n" - + "GROUP BY `t1`.`gender`, CASE WHEN `t1`.`age` IS NULL THEN 'NULL' WHEN" - + " `t7`.`_row_number_chart_` <= 10 THEN `t1`.`age` ELSE 'OTHER' END\n" - + "ORDER BY `t1`.`gender` NULLS LAST, 2 NULLS LAST"; + + "GROUP BY `age`) `t9` ON `t2`.`age` = `t9`.`age`\n" + + "GROUP BY `t2`.`gender`, CASE WHEN `t2`.`age` IS NULL THEN 'NULL' WHEN" + + " `t9`.`_row_number_chart_` <= 10 THEN `t2`.`age` ELSE 'OTHER' END\n" + + "ORDER BY `t2`.`gender` NULLS LAST, 2 NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -160,6 +166,7 @@ public void testChartWithLimit() { String expectedSparkSql = "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" + + "WHERE `gender` IS NOT NULL\n" + "GROUP BY `gender`\n" + "ORDER BY `gender` NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -173,6 +180,7 @@ public void testChartWithLimitZero() { String expectedSparkSql = "SELECT `state`, `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" + + "WHERE `state` IS NOT NULL\n" + "GROUP BY `state`, `gender`\n" + "ORDER BY `state` NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -186,6 +194,7 @@ public void testChartWithSpan() { String expectedSparkSql = "SELECT `SPAN`(`age`, 10, NULL) `age`, MAX(`balance`) `max(balance)`\n" + "FROM `scott`.`bank`\n" + + "WHERE `age` IS NOT NULL\n" + "GROUP BY `SPAN`(`age`, 10, NULL)\n" + "ORDER BY 1 NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -197,23 +206,25 @@ public void testChartWithTimeSpan() { RelNode root = getRelNode(ppl); String expectedSparkSql = - "SELECT `t1`.`timestamp`, CASE WHEN `t1`.`category` IS NULL THEN 'NULL' WHEN" - + " `t7`.`_row_number_chart_` <= 10 THEN `t1`.`category` ELSE 'OTHER' END `category`," - + " MAX(`t1`.`max(value)`) `max(value)`\n" + "SELECT `t2`.`timestamp`, CASE WHEN `t2`.`category` IS NULL THEN 'NULL' WHEN" + + " `t9`.`_row_number_chart_` <= 10 THEN `t2`.`category` ELSE 'OTHER' END `category`," + + " MAX(`t2`.`max(value)`) `max(value)`\n" + "FROM (SELECT `SPAN`(`timestamp`, 1, 'w') `timestamp`, `category`, MAX(`value`)" + " `max(value)`\n" + "FROM `scott`.`time_data`\n" - + "GROUP BY `category`, `SPAN`(`timestamp`, 1, 'w')) `t1`\n" - + "LEFT JOIN (SELECT `category`, MAX(`max(value)`) `__grand_total__`, ROW_NUMBER() OVER" - + " (ORDER BY MAX(`max(value)`) DESC) `_row_number_chart_`\n" + + "WHERE `timestamp` IS NOT NULL\n" + + "GROUP BY `category`, `SPAN`(`timestamp`, 1, 'w')) `t2`\n" + + "LEFT JOIN (SELECT `category`, SUM(`max(value)`) `__grand_total__`, ROW_NUMBER() OVER" + + " (ORDER BY SUM(`max(value)`) DESC) `_row_number_chart_`\n" + "FROM (SELECT `category`, MAX(`value`) `max(value)`\n" + "FROM `scott`.`time_data`\n" - + "GROUP BY `category`, `SPAN`(`timestamp`, 1, 'w')) `t4`\n" + + "WHERE `timestamp` IS NOT NULL\n" + + "GROUP BY `category`, `SPAN`(`timestamp`, 1, 'w')) `t6`\n" + "WHERE `category` IS NOT NULL\n" - + "GROUP BY `category`) `t7` ON `t1`.`category` = `t7`.`category`\n" - + "GROUP BY `t1`.`timestamp`, CASE WHEN `t1`.`category` IS NULL THEN 'NULL' WHEN" - + " `t7`.`_row_number_chart_` <= 10 THEN `t1`.`category` ELSE 'OTHER' END\n" - + "ORDER BY `t1`.`timestamp` NULLS LAST, 2 NULLS LAST"; + + "GROUP BY `category`) `t9` ON `t2`.`category` = `t9`.`category`\n" + + "GROUP BY `t2`.`timestamp`, CASE WHEN `t2`.`category` IS NULL THEN 'NULL' WHEN" + + " `t9`.`_row_number_chart_` <= 10 THEN `t2`.`category` ELSE 'OTHER' END\n" + + "ORDER BY `t2`.`timestamp` NULLS LAST, 2 NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -225,6 +236,7 @@ public void testChartWithUseOtherTrue() { String expectedSparkSql = "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" + + "WHERE `gender` IS NOT NULL\n" + "GROUP BY `gender`\n" + "ORDER BY `gender` NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -238,6 +250,7 @@ public void testChartWithUseOtherFalse() { String expectedSparkSql = "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" + + "WHERE `gender` IS NOT NULL\n" + "GROUP BY `gender`\n" + "ORDER BY `gender` NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -251,6 +264,7 @@ public void testChartWithOtherStr() { String expectedSparkSql = "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" + + "WHERE `gender` IS NOT NULL\n" + "GROUP BY `gender`\n" + "ORDER BY `gender` NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -264,6 +278,7 @@ public void testChartWithNullStr() { String expectedSparkSql = "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" + + "WHERE `gender` IS NOT NULL\n" + "GROUP BY `gender`\n" + "ORDER BY `gender` NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); From 72c2021fe4f873c5a706d6c136fa7b84e7468983 Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Wed, 5 Nov 2025 18:11:18 +0800 Subject: [PATCH 30/35] Simplify toAddHintsOnAggregate condition Signed-off-by: Yuanchun Shen --- .../opensearch/sql/calcite/CalciteRelNodeVisitor.java | 11 +++++------ .../calcite/explain_agg_script_udt_arg_push.yaml | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 05d0e22508b..0177fe39f83 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -1137,13 +1137,12 @@ private void visitAggregation( } groupExprList.addAll(node.getGroupExprList()); - // add stats hint to LogicalAggregation + // Add stats hint to LogicalAggregation. boolean toAddHintsOnAggregate = - nonNullGroupMask.nextClearBit(0) - >= groupExprList.size() // This checks if all group-bys should be nonnull - && !groupExprList.isEmpty() - && !(groupExprList.size() == 1 && getTimeSpanField(span).isPresent()); - // add isNotNull filter before aggregation for non-nullable buckets + !groupExprList.isEmpty() + // This checks if all group-bys should be nonnull + && nonNullGroupMask.nextClearBit(0) >= groupExprList.size(); + // Add isNotNull filter before aggregation for non-nullable buckets List groupByList = groupExprList.stream().map(expr -> rexVisitor.analyze(expr, context)).toList(); List nonNullGroupBys = diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_udt_arg_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_udt_arg_push.yaml index e0d54375e15..0c34a565cdc 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_udt_arg_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_udt_arg_push.yaml @@ -8,4 +8,4 @@ calcite: LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], t=[DATE_ADD($3, 1:INTERVAL DAY)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[SCRIPT->IS NOT NULL(DATE_ADD($3, 1:INTERVAL DAY)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(t,1d)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAt3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AyV7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiREFURV9BREQiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImlucHV0IjogMCwKICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVSVkFMX0RBWSIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogMTAsCiAgICAgICAgICAgICJzY2FsZSI6IDYKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInVkdCI6ICJFWFBSX1RJTUVTVEFNUCIsCiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfSwKICAgICAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAAJYmlydGhkYXRlc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0ZVR5cGWeLVKuEH3KrwIAAUwAB2Zvcm1hdHN0ABBMamF2YS91dGlsL0xpc3Q7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXN0AA9MamF2YS91dGlsL01hcDt4cH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQACVRJTUVTVEFNUH5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+ABJ0AAREYXRlc3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAZeHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAAdXEAfgAbAAAAAHNxAH4AAAAAAAF3BAAAAAB4eHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(t,1d)":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAt3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BQN7CiAgIm9wIjogewogICAgIm5hbWUiOiAiU1BBTiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiREFURV9BREQiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImlucHV0IjogMCwKICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVSVkFMX0RBWSIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogMTAsCiAgICAgICAgICAgICJzY2FsZSI6IDYKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInVkdCI6ICJFWFBSX1RJTUVTVEFNUCIsCiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfSwKICAgICAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogImQiLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogMQogICAgICB9CiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICJwcmVjaXNpb24iOiAtMQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQACWJpcnRoZGF0ZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGVUeXBlni1SrhB9yq8CAAFMAAdmb3JtYXRzdAAQTGphdmEvdXRpbC9MaXN0O3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzdAAPTGphdmEvdXRpbC9NYXA7eHB+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAlUSU1FU1RBTVB+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgASdAAERGF0ZXNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGXhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGwAAAABzcQB+AAAAAAABdwQAAAAAeHh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"missing_bucket":true,"value_type":"long","missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[SCRIPT->IS NOT NULL(DATE_ADD($3, 1:INTERVAL DAY)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(t,1d)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAt3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AyV7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiREFURV9BREQiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImlucHV0IjogMCwKICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVSVkFMX0RBWSIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogMTAsCiAgICAgICAgICAgICJzY2FsZSI6IDYKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInVkdCI6ICJFWFBSX1RJTUVTVEFNUCIsCiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfSwKICAgICAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAAJYmlydGhkYXRlc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0ZVR5cGWeLVKuEH3KrwIAAUwAB2Zvcm1hdHN0ABBMamF2YS91dGlsL0xpc3Q7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXN0AA9MamF2YS91dGlsL01hcDt4cH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQACVRJTUVTVEFNUH5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+ABJ0AAREYXRlc3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAZeHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAAdXEAfgAbAAAAAHNxAH4AAAAAAAF3BAAAAAB4eHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(t,1d)":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAt3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BQN7CiAgIm9wIjogewogICAgIm5hbWUiOiAiU1BBTiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiREFURV9BREQiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImlucHV0IjogMCwKICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVSVkFMX0RBWSIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogMTAsCiAgICAgICAgICAgICJzY2FsZSI6IDYKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInVkdCI6ICJFWFBSX1RJTUVTVEFNUCIsCiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfSwKICAgICAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogImQiLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogMQogICAgICB9CiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICJwcmVjaXNpb24iOiAtMQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQACWJpcnRoZGF0ZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGVUeXBlni1SrhB9yq8CAAFMAAdmb3JtYXRzdAAQTGphdmEvdXRpbC9MaXN0O3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzdAAPTGphdmEvdXRpbC9NYXA7eHB+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAlUSU1FU1RBTVB+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgASdAAERGF0ZXNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGXhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGwAAAABzcQB+AAAAAAABdwQAAAAAeHh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"missing_bucket":false,"value_type":"long","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) From 718135e790f244b4563cd1fe173e4876bcdfa088 Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Wed, 5 Nov 2025 22:58:42 +0800 Subject: [PATCH 31/35] Chores: eliminate unnecessary variables Signed-off-by: Yuanchun Shen --- .../sql/calcite/CalciteRelNodeVisitor.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 0177fe39f83..8c32d6e1ce4 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -2416,8 +2416,9 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { // If row or column split does not present or limit equals 0, this is the same as `stats agg // [group by col]` because all truncating is performed on the column split - Integer limit = (Integer) argMap.getOrDefault("limit", Chart.DEFAULT_LIMIT).getValue(); - if (node.getRowSplit() == null || node.getColumnSplit() == null || Objects.equals(limit, 0)) { + if (node.getRowSplit() == null + || node.getColumnSplit() == null + || Objects.equals(config.limit, 0)) { // The output of chart is expected to be ordered by row split names relBuilder.sort(relBuilder.field(0)); return relBuilder.peek(); @@ -2435,13 +2436,13 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { // Convert the column split to string if necessary: column split was supposed to be pivoted to // column names. This guarantees that its type compatibility with useother and usenull RexNode colSplit = relBuilder.field(1); - String columSplitName = relBuilder.peek().getRowType().getFieldNames().get(1); + String columnSplitName = relBuilder.peek().getRowType().getFieldNames().get(1); if (!SqlTypeUtil.isCharacter(colSplit.getType())) { colSplit = relBuilder.alias( context.rexBuilder.makeCast( UserDefinedFunctionUtils.NULLABLE_STRING, colSplit, true, true), - columSplitName); + columnSplitName); } relBuilder.project(relBuilder.field(0), colSplit, relBuilder.field(2)); RelNode aggregated = relBuilder.peek(); @@ -2460,7 +2461,7 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { if (config.top) { grandTotal = relBuilder.desc(grandTotal); } - // Always set it to null last so that it does not interfere with top / bottom calculation + // Always set it to null last so that nulls don't interfere with top / bottom calculation grandTotal = relBuilder.nullsLast(grandTotal); RexNode rowNum = PlanUtils.makeOver( @@ -2486,7 +2487,7 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { relBuilder.call( SqlStdOperatorTable.LESS_THAN_OR_EQUAL, relBuilder.field(PlanUtils.ROW_NUMBER_COLUMN_FOR_CHART), - relBuilder.literal(limit)); + relBuilder.literal(config.limit)); if (!config.useOther) { relBuilder.filter(lteCondition); } @@ -2511,7 +2512,6 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { relBuilder.literal(config.otherStr)); } - String columnSplitName = ((Alias) node.getColumnSplit()).getName(); String aggFieldName = relBuilder.peek().getRowType().getFieldNames().get(2); relBuilder.project( relBuilder.field(0), From 6b7b915c5f31a87c084d75ab67ca362fde69db30 Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Thu, 6 Nov 2025 13:53:57 +0800 Subject: [PATCH 32/35] Apply a non-null filter on fields referred by aggregations Signed-off-by: Yuanchun Shen --- .../sql/calcite/CalciteRelNodeVisitor.java | 44 +++++++++++++------ docs/user/ppl/cmd/chart.rst | 7 +-- .../calcite/remote/CalciteChartCommandIT.java | 6 +-- .../sql/calcite/remote/CalciteExplainIT.java | 25 ++++++----- .../calcite/chart_multiple_group_keys.yaml | 36 +++++++++++++++ .../calcite/chart_null_str.yaml | 40 +++++++++++++++++ .../calcite/chart_single_group_key.yaml | 10 +++++ ...=> chart_timestamp_span_and_category.yaml} | 8 ++-- .../calcite/chart_use_other.yaml | 31 +++++++++++++ .../calcite/chart_with_integer_span.yaml | 11 +++++ .../calcite/chart_with_limit.yaml | 10 +++++ .../calcite/chart_with_timestamp_span.yaml | 11 +++++ .../explain_chart_multiple_group_keys.yaml | 36 --------------- .../calcite/explain_chart_null_str.yaml | 40 ----------------- .../explain_chart_single_group_key.yaml | 10 ----- .../calcite/explain_chart_use_other.yaml | 31 ------------- .../calcite/explain_chart_with_limit.yaml | 10 ----- .../calcite/explain_chart_with_span.yaml | 11 ----- ...ys.yaml => chart_multiple_group_keys.yaml} | 8 ++-- ...groups.yaml => chart_multiple_groups.yaml} | 0 ...hart_null_str.yaml => chart_null_str.yaml} | 8 ++-- ...gle_group.yaml => chart_single_group.yaml} | 0 ...p_key.yaml => chart_single_group_key.yaml} | 4 +- ...=> chart_timestamp_span_and_category.yaml} | 8 ++-- ...rt_use_other.yaml => chart_use_other.yaml} | 8 ++-- ...span.yaml => chart_with_integer_span.yaml} | 4 +- ..._with_limit.yaml => chart_with_limit.yaml} | 4 +- .../chart_with_timestamp_span.yaml | 15 +++++++ .../sql/ppl/calcite/CalcitePPLChartTest.java | 32 +++++++------- 29 files changed, 256 insertions(+), 212 deletions(-) create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/chart_multiple_group_keys.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/chart_single_group_key.yaml rename integ-test/src/test/resources/expectedOutput/calcite/{explain_chart_timestamp_span.yaml => chart_timestamp_span_and_category.yaml} (51%) create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/chart_use_other.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/chart_with_integer_span.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/chart_with_limit.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/chart_with_timestamp_span.yaml delete mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_chart_multiple_group_keys.yaml delete mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_chart_null_str.yaml delete mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_chart_single_group_key.yaml delete mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_chart_use_other.yaml delete mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_limit.yaml delete mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_span.yaml rename integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/{explain_chart_multiple_group_keys.yaml => chart_multiple_group_keys.yaml} (88%) rename integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/{explain_chart_multiple_groups.yaml => chart_multiple_groups.yaml} (100%) rename integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/{explain_chart_null_str.yaml => chart_null_str.yaml} (87%) rename integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/{explain_chart_single_group.yaml => chart_single_group.yaml} (100%) rename integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/{explain_chart_single_group_key.yaml => chart_single_group_key.yaml} (82%) rename integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/{explain_chart_timestamp_span.yaml => chart_timestamp_span_and_category.yaml} (86%) rename integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/{explain_chart_use_other.yaml => chart_use_other.yaml} (86%) rename integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/{explain_chart_with_span.yaml => chart_with_integer_span.yaml} (80%) rename integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/{explain_chart_with_limit.yaml => chart_with_limit.yaml} (83%) create mode 100644 integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_timestamp_span.yaml diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 8c32d6e1ce4..1142d2d125c 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -1097,6 +1097,7 @@ private Pair, List> resolveAttributesForAggregation( return Pair.of(groupByList, aggCallList); } + /** Visits an aggregation for stats command */ @Override public RelNode visitAggregation(Aggregation node, CalcitePlanContext context) { Argument.ArgumentMap statsArgs = Argument.ArgumentMap.of(node.getArgExprList()); @@ -1107,7 +1108,7 @@ public RelNode visitAggregation(Aggregation node, CalcitePlanContext context) { if (!bucketNullable) { nonNullGroupMask.set(0, nGroup); } - visitAggregation(node, context, true, nonNullGroupMask); + visitAggregation(node, context, nonNullGroupMask, true, false); return context.relBuilder.peek(); } @@ -1116,12 +1117,19 @@ public RelNode visitAggregation(Aggregation node, CalcitePlanContext context) { * * @param node the aggregation node containing group expressions and aggregation functions * @param context the Calcite plan context for building RelNodes - * @param aggFirst if true, aggregation results (metrics) appear first in output schema (agg, - * group-by fields); if false, group expressions appear first (group-by fields, agg). * @param nonNullGroupMask bit set indicating group by fields that need to be non-null + * @param metricsFirst if true, aggregation results (metrics) appear first in output schema + * (metrics, group-by fields); if false, group expressions appear first (group-by fields, + * metrics). + * @param includeAggFieldsInNullFilter if true, also applies non-null filters to aggregation input + * fields in addition to group-by fields */ private void visitAggregation( - Aggregation node, CalcitePlanContext context, boolean aggFirst, BitSet nonNullGroupMask) { + Aggregation node, + CalcitePlanContext context, + BitSet nonNullGroupMask, + boolean metricsFirst, + boolean includeAggFieldsInNullFilter) { visitChildren(node, context); List aggExprList = node.getAggExprList(); @@ -1143,15 +1151,23 @@ private void visitAggregation( // This checks if all group-bys should be nonnull && nonNullGroupMask.nextClearBit(0) >= groupExprList.size(); // Add isNotNull filter before aggregation for non-nullable buckets - List groupByList = - groupExprList.stream().map(expr -> rexVisitor.analyze(expr, context)).toList(); - List nonNullGroupBys = - IntStream.range(0, groupByList.size()) + List nonNullCandidates = + groupExprList.stream() + .map(expr -> rexVisitor.analyze(expr, context)) + .collect(Collectors.toCollection(ArrayList::new)); + if (includeAggFieldsInNullFilter) { + nonNullCandidates.addAll( + PlanUtils.getInputRefsFromAggCall( + aggExprList.stream().map(expr -> aggVisitor.analyze(expr, context)).toList())); + nonNullGroupMask.set(groupExprList.size(), nonNullCandidates.size()); + } + List nonNullFields = + IntStream.range(0, nonNullCandidates.size()) .filter(nonNullGroupMask::get) - .mapToObj(groupByList::get) + .mapToObj(nonNullCandidates::get) .toList(); context.relBuilder.filter( - PlanUtils.getSelectColumns(nonNullGroupBys).stream() + PlanUtils.getSelectColumns(nonNullFields).stream() .map(context.relBuilder::field) .map(context.relBuilder::isNotNull) .toList()); @@ -1175,7 +1191,7 @@ private void visitAggregation( .map(context.relBuilder::field) .map(f -> (RexNode) f) .toList(); - if (aggFirst) { + if (metricsFirst) { // As an example, in command `stats count() by colA, colB`, // the sequence of output schema is "count, colA, colB". reordered.addAll(aggRexList); @@ -2411,11 +2427,11 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { } else { nonNullGroupMask.set(0, groupExprList.size()); } - visitAggregation(aggregation, context, false, nonNullGroupMask); + visitAggregation(aggregation, context, nonNullGroupMask, false, true); RelBuilder relBuilder = context.relBuilder; - // If row or column split does not present or limit equals 0, this is the same as `stats agg - // [group by col]` because all truncating is performed on the column split + // If a second split does not present or limit equals 0, we go no further for limit, nullstr, + // otherstr parameters because all truncating & renaming is performed on the column split if (node.getRowSplit() == null || node.getColumnSplit() == null || Objects.equals(config.limit, 0)) { diff --git a/docs/user/ppl/cmd/chart.rst b/docs/user/ppl/cmd/chart.rst index 27ef2140c25..fe8aac4a3e0 100644 --- a/docs/user/ppl/cmd/chart.rst +++ b/docs/user/ppl/cmd/chart.rst @@ -36,7 +36,7 @@ Syntax * Syntax: ``limit=(top|bottom)`` or ``limit=`` (defaults to top) * When ``limit=K`` is set, the top or bottom K categories from the column split field are retained; the remaining categories are grouped into an "OTHER" category if ``useother`` is not set to false. * Set limit to 0 to show all categories without any limit. - * Use ``limit=topK`` or ``limit=bottomK`` to specify whether to retain the top or bottom K column categories. The ranking is based on the aggregated values for each category. For example, ``chart limit=top3 count() by a b`` retains the 3 most common b categories; ``chart limit=top5 min(value) by a b`` selects the 5 b categories that contain the smallest aggregated values. If not specified, top is used by default. + * Use ``limit=topK`` or ``limit=bottomK`` to specify whether to retain the top or bottom K column categories. The ranking is based on the sum of aggregated values for each column category. For example, ``chart limit=top3 count() by region, product`` keeps the 3 products with the highest total counts across all regions. If not specified, top is used by default. * Only applies when column split is present (by 2 fields or over...by... coexists). * **useother**: optional. Controls whether to create an "OTHER" category for categories beyond the limit. @@ -50,7 +50,7 @@ Syntax * Default: true * ``usenull`` only applies to column split. - * Row split should always be non-null value. Events with null values in row split will be ignored. + * Row split should always be non-null value. Documents with null values in row split will be ignored. * When ``usenull=false``, events with a null column split are excluded from results. * When ``usenull=true``, events with a null column split are grouped into a separate "NULL" category. @@ -84,7 +84,8 @@ Notes ===== * The fields generated by column splitting are converted to strings so that they are compatible with ``nullstr`` and ``otherstr`` and can be used as column names once pivoted. -* The aggregation metric appears as the last column in the result. Result columns are ordered as: [row-split] [column-split] [aggregation-metrics] +* Documents with null values in fields used by the aggregation function are excluded from aggregation. For example, in ``chart avg(balance) over deptno, group``, documents where ``balance`` is null are excluded from the average calculation. +* The aggregation metric appears as the last column in the result. Result columns are ordered as: [row-split] [column-split] [aggregation-metrics]. Examples ======== diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java index dee2d4c7776..b325912e7d7 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java @@ -279,11 +279,7 @@ public void testChartUseNullTrueWithNullStr() throws IOException { schema("age", "string"), schema("avg(balance)", "double")); verifyDataRows( - result, - rows("M", "30", 21702.5), - rows("F", "30", 48086.0), - rows("F", "20", 32838.0), - rows("F", "nil", null)); + result, rows("M", "30", 21702.5), rows("F", "30", 48086.0), rows("F", "20", 32838.0)); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index f975bcdc2fe..3787540825a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -1409,35 +1409,40 @@ public void testPushDownMinOrMaxAggOnDerivedField() throws IOException { @Test public void testExplainChartWithSingleGroupKey() throws IOException { assertYamlEqualsIgnoreId( - loadExpectedPlan("explain_chart_single_group_key.yaml"), + loadExpectedPlan("chart_single_group_key.yaml"), explainQueryYaml( String.format("source=%s | chart avg(balance) by gender", TEST_INDEX_BANK))); assertYamlEqualsIgnoreId( - loadExpectedPlan("explain_chart_with_span.yaml"), + loadExpectedPlan("chart_with_integer_span.yaml"), explainQueryYaml( String.format("source=%s | chart max(balance) by age span=10", TEST_INDEX_BANK))); assertYamlEqualsIgnoreId( - loadExpectedPlan("explain_chart_timestamp_span.yaml"), + loadExpectedPlan("chart_with_timestamp_span.yaml"), explainQueryYaml( String.format( - "source=%s | chart max(value) over timestamp span=1week by category", - TEST_INDEX_TIME_DATA))); + "source=%s | chart count by @timestamp span=1day", TEST_INDEX_TIME_DATA))); } @Test public void testExplainChartWithMultipleGroupKeys() throws IOException { - String expected = loadExpectedPlan("explain_chart_multiple_group_keys.yaml"); assertYamlEqualsIgnoreId( - expected, + loadExpectedPlan("chart_multiple_group_keys.yaml"), explainQueryYaml( String.format("source=%s | chart avg(balance) over gender by age", TEST_INDEX_BANK))); + + assertYamlEqualsIgnoreId( + loadExpectedPlan("chart_timestamp_span_and_category.yaml"), + explainQueryYaml( + String.format( + "source=%s | chart max(value) over timestamp span=1week by category", + TEST_INDEX_TIME_DATA))); } @Test public void testExplainChartWithLimits() throws IOException { - String expected = loadExpectedPlan("explain_chart_with_limit.yaml"); + String expected = loadExpectedPlan("chart_with_limit.yaml"); assertYamlEqualsIgnoreId( expected, explainQueryYaml( @@ -1445,7 +1450,7 @@ public void testExplainChartWithLimits() throws IOException { "source=%s | chart limit=0 avg(balance) over state by gender", TEST_INDEX_BANK))); assertYamlEqualsIgnoreId( - loadExpectedPlan("explain_chart_use_other.yaml"), + loadExpectedPlan("chart_use_other.yaml"), explainQueryYaml( String.format( "source=%s | chart limit=2 useother=true otherstr='max_among_other'" @@ -1455,7 +1460,7 @@ public void testExplainChartWithLimits() throws IOException { @Test public void testExplainChartWithNullStr() throws IOException { - String expected = loadExpectedPlan("explain_chart_null_str.yaml"); + String expected = loadExpectedPlan("chart_null_str.yaml"); assertYamlEqualsIgnoreId( expected, explainQueryYaml( diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_multiple_group_keys.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_multiple_group_keys.yaml new file mode 100644 index 00000000000..4b7b86e69f9 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_multiple_group_keys.yaml @@ -0,0 +1,36 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(gender=[$0], age=[CASE(IS NULL($1), 'NULL', <=($5, 10), $1, 'OTHER')], avg(balance)=[$2]) + LogicalJoin(condition=[=($1, $3)], joinType=[left]) + LogicalProject(gender=[$0], age=[SAFE_CAST($1)], avg(balance)=[$2]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(gender=[$4], age=[$10], balance=[$7]) + LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($7))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalProject(age=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) + LogicalAggregate(group=[{0}], __grand_total__=[SUM($1)]) + LogicalFilter(condition=[IS NOT NULL($0)]) + LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(gender=[$4], age=[$10], balance=[$7]) + LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($7))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(balance)=[$t8]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) + EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) + EnumerableSort(sort0=[$1], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[SAFE_CAST($t1)], gender=[$t0], age=[$t3], avg(balance)=[$t2]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[gender, balance, age], FILTER->AND(IS NOT NULL($0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"filter":[{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["gender","balance","age"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) + EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[SAFE_CAST($t0)], expr#3=[IS NOT NULL($t2)], age=[$t2], avg(balance)=[$t1], $condition=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[gender, balance, age], FILTER->AND(IS NOT NULL($0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2)), PROJECT->[age, avg(balance)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"filter":[{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["gender","balance","age"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml new file mode 100644 index 00000000000..78eb63241dc --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml @@ -0,0 +1,40 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(gender=[$0], age=[CASE(IS NULL($1), 'nil', <=($5, 10), $1, 'OTHER')], avg(balance)=[$2]) + LogicalJoin(condition=[=($1, $3)], joinType=[left]) + LogicalProject(gender=[$0], age=[SAFE_CAST($1)], avg(balance)=[$2]) + LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) + LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) + LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($3))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) + LogicalProject(age=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) + LogicalAggregate(group=[{0}], __grand_total__=[SUM($1)]) + LogicalFilter(condition=[IS NOT NULL($0)]) + LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) + LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) + LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) + LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($3))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(balance)=[$t8]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['nil'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) + EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) + EnumerableSort(sort0=[$1], dir0=[ASC]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t10]) + EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], gender=[$t1], balance=[$t0], age0=[$t5]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->AND(IS NOT NULL($1), IS NOT NULL($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"filter":[{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["balance","gender","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) + EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], expr#11=[IS NOT NULL($t4)], age=[$t4], avg(balance)=[$t10], $condition=[$t11]) + EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], gender=[$t1], balance=[$t0], age0=[$t5]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->AND(IS NOT NULL($1), IS NOT NULL($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"filter":[{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["balance","gender","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_single_group_key.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_single_group_key.yaml new file mode 100644 index 00000000000..b011edc42f5 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_single_group_key.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)]) + LogicalProject(gender=[$4], balance=[$7]) + LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($7))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[FILTER->AND(IS NOT NULL($4), IS NOT NULL($7)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg(balance)=AVG($1)), SORT->[0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_timestamp_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_timestamp_span_and_category.yaml similarity index 51% rename from integ-test/src/test/resources/expectedOutput/calcite/explain_chart_timestamp_span.yaml rename to integ-test/src/test/resources/expectedOutput/calcite/chart_timestamp_span_and_category.yaml index a592dfc4e22..696edc9ea35 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_timestamp_span.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_timestamp_span_and_category.yaml @@ -8,7 +8,7 @@ calcite: LogicalProject(timestamp=[$1], category=[$0], max(value)=[$2]) LogicalAggregate(group=[{0, 2}], max(value)=[MAX($1)]) LogicalProject(category=[$1], value=[$2], timestamp0=[SPAN($3, 1, 'w')]) - LogicalFilter(condition=[IS NOT NULL($3)]) + LogicalFilter(condition=[AND(IS NOT NULL($3), IS NOT NULL($2))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) LogicalProject(category=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) LogicalAggregate(group=[{0}], __grand_total__=[SUM($1)]) @@ -16,7 +16,7 @@ calcite: LogicalProject(category=[$0], max(value)=[$2]) LogicalAggregate(group=[{0, 2}], max(value)=[MAX($1)]) LogicalProject(category=[$1], value=[$2], timestamp0=[SPAN($3, 1, 'w')]) - LogicalFilter(condition=[IS NOT NULL($3)]) + LogicalFilter(condition=[AND(IS NOT NULL($3), IS NOT NULL($2))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) physical: | EnumerableLimit(fetch=[10000]) @@ -24,9 +24,9 @@ calcite: EnumerableAggregate(group=[{0, 1}], max(value)=[MAX($2)]) EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], timestamp=[$t0], category=[$t10], max(value)=[$t2]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[PROJECT->[category, value, timestamp], FILTER->IS NOT NULL($2), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},max(value)=MAX($1)), PROJECT->[timestamp0, category, max(value)], SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"timestamp","boost":1.0}},"_source":{"includes":["category","value","timestamp"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"timestamp0":{"date_histogram":{"field":"timestamp","missing_bucket":false,"order":"asc","calendar_interval":"1w"}}}]},"aggregations":{"max(value)":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[PROJECT->[category, value, timestamp], FILTER->AND(IS NOT NULL($2), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},max(value)=MAX($1)), PROJECT->[timestamp0, category, max(value)], SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"timestamp","boost":1.0}},{"exists":{"field":"value","boost":1.0}}],"filter":[{"exists":{"field":"category","boost":1.0}},{"exists":{"field":"category","boost":1.0}},{"exists":{"field":"category","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["category","value","timestamp"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"timestamp0":{"date_histogram":{"field":"timestamp","missing_bucket":false,"order":"asc","calendar_interval":"1w"}}}]},"aggregations":{"max(value)":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], category=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[PROJECT->[category, value, timestamp], FILTER->IS NOT NULL($2), FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},max(value)=MAX($1)), PROJECT->[category, max(value)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"filter":[{"exists":{"field":"timestamp","boost":1.0}},{"exists":{"field":"category","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["category","value","timestamp"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"timestamp0":{"date_histogram":{"field":"timestamp","missing_bucket":false,"order":"asc","calendar_interval":"1w"}}}]},"aggregations":{"max(value)":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[PROJECT->[category, value, timestamp], FILTER->AND(IS NOT NULL($2), IS NOT NULL($1)), FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},max(value)=MAX($1)), PROJECT->[category, max(value)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"timestamp","boost":1.0}},{"exists":{"field":"value","boost":1.0}}],"filter":[{"exists":{"field":"category","boost":1.0}},{"exists":{"field":"category","boost":1.0}},{"exists":{"field":"category","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["category","value","timestamp"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"timestamp0":{"date_histogram":{"field":"timestamp","missing_bucket":false,"order":"asc","calendar_interval":"1w"}}}]},"aggregations":{"max(value)":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_use_other.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_use_other.yaml new file mode 100644 index 00000000000..33fbbdb4c86 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_use_other.yaml @@ -0,0 +1,31 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) + LogicalProject(flags=[$0], severityText=[CASE(IS NULL($1), 'NULL', <=($5, 2), $1, 'max_among_other')], max(severityNumber)=[$2]) + LogicalJoin(condition=[=($1, $3)], joinType=[left]) + LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) + LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) + LogicalFilter(condition=[AND(IS NOT NULL($23), IS NOT NULL($163))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) + LogicalProject(severityText=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) + LogicalAggregate(group=[{0}], __grand_total__=[SUM($1)]) + LogicalFilter(condition=[IS NOT NULL($0)]) + LogicalProject(severityText=[$1], max(severityNumber)=[$2]) + LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) + LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) + LogicalFilter(condition=[AND(IS NOT NULL($23), IS NOT NULL($163))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + EnumerableAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[2], expr#8=[<=($t4, $t7)], expr#9=['max_among_other'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], flags=[$t0], severityText=[$t10], max(severityNumber)=[$t2]) + EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[PROJECT->[severityText, flags, severityNumber], FILTER->AND(IS NOT NULL($1), IS NOT NULL($2)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},max(severityNumber)=MAX($2)), SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"flags","boost":1.0}},{"exists":{"field":"severityNumber","boost":1.0}}],"filter":[{"exists":{"field":"severityText","boost":1.0}},{"exists":{"field":"severityText","boost":1.0}},{"exists":{"field":"severityText","boost":1.0}},{"exists":{"field":"severityText","boost":1.0}},{"exists":{"field":"severityText","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["severityText","flags","severityNumber"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"severityText":{"terms":{"field":"severityText","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"flags":{"terms":{"field":"flags","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"max(severityNumber)":{"max":{"field":"severityNumber"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], severityText=[$t0], $1=[$t2]) + EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[PROJECT->[severityText, flags, severityNumber], FILTER->AND(IS NOT NULL($1), IS NOT NULL($2)), FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},max(severityNumber)=MAX($2)), PROJECT->[severityText, max(severityNumber)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"flags","boost":1.0}},{"exists":{"field":"severityNumber","boost":1.0}}],"filter":[{"exists":{"field":"severityText","boost":1.0}},{"exists":{"field":"severityText","boost":1.0}},{"exists":{"field":"severityText","boost":1.0}},{"exists":{"field":"severityText","boost":1.0}},{"exists":{"field":"severityText","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["severityText","flags","severityNumber"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"flags":{"terms":{"field":"flags","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"severityText":{"terms":{"field":"severityText","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"max(severityNumber)":{"max":{"field":"severityNumber"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_with_integer_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_with_integer_span.yaml new file mode 100644 index 00000000000..c47fb9dd438 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_with_integer_span.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalProject(age=[$0], max(balance)=[$1]) + LogicalAggregate(group=[{1}], max(balance)=[MAX($0)]) + LogicalProject(balance=[$7], age0=[SPAN($10, 10, null:NULL)]) + LogicalFilter(condition=[AND(IS NOT NULL($10), IS NOT NULL($7))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[balance, age], FILTER->AND(IS NOT NULL($1), IS NOT NULL($0)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},max(balance)=MAX($0)), SORT->[0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"age","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["balance","age"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age0":{"histogram":{"field":"age","missing_bucket":false,"order":"asc","interval":10.0}}}]},"aggregations":{"max(balance)":{"max":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_with_limit.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_with_limit.yaml new file mode 100644 index 00000000000..389825459df --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_with_limit.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(state=[$9], gender=[$4], balance=[$7]) + LogicalFilter(condition=[AND(IS NOT NULL($9), IS NOT NULL($7))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[gender, balance, state], FILTER->AND(IS NOT NULL($2), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2)), SORT->[0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"state","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["gender","balance","state"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_with_timestamp_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_with_timestamp_span.yaml new file mode 100644 index 00000000000..cdce175e83a --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_with_timestamp_span.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalProject(@timestamp=[$0], count=[$1]) + LogicalAggregate(group=[{0}], count=[COUNT()]) + LogicalProject(@timestamp0=[SPAN($0, 1, 'd')]) + LogicalFilter(condition=[IS NOT NULL($0)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[PROJECT->[@timestamp], FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count=COUNT()), SORT->[0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"@timestamp","boost":1.0}},"_source":{"includes":["@timestamp"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"@timestamp0":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1d"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_multiple_group_keys.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_multiple_group_keys.yaml deleted file mode 100644 index 5f7db3b2fdd..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_multiple_group_keys.yaml +++ /dev/null @@ -1,36 +0,0 @@ -calcite: - logical: | - LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) - LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) - LogicalProject(gender=[$0], age=[CASE(IS NULL($1), 'NULL', <=($5, 10), $1, 'OTHER')], avg(balance)=[$2]) - LogicalJoin(condition=[=($1, $3)], joinType=[left]) - LogicalProject(gender=[$0], age=[SAFE_CAST($1)], avg(balance)=[$2]) - LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) - LogicalProject(gender=[$4], age=[$10], balance=[$7]) - LogicalFilter(condition=[IS NOT NULL($4)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - LogicalProject(age=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) - LogicalAggregate(group=[{0}], __grand_total__=[SUM($1)]) - LogicalFilter(condition=[IS NOT NULL($0)]) - LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) - LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) - LogicalProject(gender=[$4], age=[$10], balance=[$7]) - LogicalFilter(condition=[IS NOT NULL($4)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - physical: | - EnumerableLimit(fetch=[10000]) - EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(balance)=[$t8]) - EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) - EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) - EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[SAFE_CAST($t1)], gender=[$t0], age=[$t3], avg(balance)=[$t2]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[gender, balance, age], FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"gender","boost":1.0}},"_source":{"includes":["gender","balance","age"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) - EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[SAFE_CAST($t0)], expr#3=[IS NOT NULL($t2)], age=[$t2], avg(balance)=[$t1], $condition=[$t3]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[gender, balance, age], FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2)), PROJECT->[age, avg(balance)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"gender","boost":1.0}},"_source":{"includes":["gender","balance","age"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_null_str.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_null_str.yaml deleted file mode 100644 index a750eea8287..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_null_str.yaml +++ /dev/null @@ -1,40 +0,0 @@ -calcite: - logical: | - LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) - LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) - LogicalProject(gender=[$0], age=[CASE(IS NULL($1), 'nil', <=($5, 10), $1, 'OTHER')], avg(balance)=[$2]) - LogicalJoin(condition=[=($1, $3)], joinType=[left]) - LogicalProject(gender=[$0], age=[SAFE_CAST($1)], avg(balance)=[$2]) - LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) - LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) - LogicalFilter(condition=[IS NOT NULL($4)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) - LogicalProject(age=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) - LogicalAggregate(group=[{0}], __grand_total__=[SUM($1)]) - LogicalFilter(condition=[IS NOT NULL($0)]) - LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) - LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) - LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) - LogicalFilter(condition=[IS NOT NULL($4)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) - physical: | - EnumerableLimit(fetch=[10000]) - EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(balance)=[$t8]) - EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['nil'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) - EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) - EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t10]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], gender=[$t1], balance=[$t0], age0=[$t5]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->IS NOT NULL($1)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"exists":{"field":"gender","boost":1.0}},"_source":{"includes":["balance","gender","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) - EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], expr#11=[IS NOT NULL($t4)], age=[$t4], avg(balance)=[$t10], $condition=[$t11]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], gender=[$t1], balance=[$t0], age0=[$t5]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->IS NOT NULL($1)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"exists":{"field":"gender","boost":1.0}},"_source":{"includes":["balance","gender","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_single_group_key.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_single_group_key.yaml deleted file mode 100644 index ca725256ece..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_single_group_key.yaml +++ /dev/null @@ -1,10 +0,0 @@ -calcite: - logical: | - LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalSort(sort0=[$0], dir0=[ASC]) - LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)]) - LogicalProject(gender=[$4], balance=[$7]) - LogicalFilter(condition=[IS NOT NULL($4)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg(balance)=AVG($1)), SORT->[0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_use_other.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_use_other.yaml deleted file mode 100644 index 67fdea2fc97..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_use_other.yaml +++ /dev/null @@ -1,31 +0,0 @@ -calcite: - logical: | - LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) - LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) - LogicalProject(flags=[$0], severityText=[CASE(IS NULL($1), 'NULL', <=($5, 2), $1, 'max_among_other')], max(severityNumber)=[$2]) - LogicalJoin(condition=[=($1, $3)], joinType=[left]) - LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) - LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) - LogicalFilter(condition=[IS NOT NULL($23)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) - LogicalProject(severityText=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) - LogicalAggregate(group=[{0}], __grand_total__=[SUM($1)]) - LogicalFilter(condition=[IS NOT NULL($0)]) - LogicalProject(severityText=[$1], max(severityNumber)=[$2]) - LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) - LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) - LogicalFilter(condition=[IS NOT NULL($23)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) - physical: | - EnumerableLimit(fetch=[10000]) - EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) - EnumerableAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[2], expr#8=[<=($t4, $t7)], expr#9=['max_among_other'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], flags=[$t0], severityText=[$t10], max(severityNumber)=[$t2]) - EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[PROJECT->[severityText, flags, severityNumber], FILTER->IS NOT NULL($1), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},max(severityNumber)=MAX($2)), SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"flags","boost":1.0}},"_source":{"includes":["severityText","flags","severityNumber"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"severityText":{"terms":{"field":"severityText","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"flags":{"terms":{"field":"flags","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"max(severityNumber)":{"max":{"field":"severityNumber"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], severityText=[$t0], $1=[$t2]) - EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[PROJECT->[severityText, flags, severityNumber], FILTER->IS NOT NULL($1), FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},max(severityNumber)=MAX($2)), PROJECT->[severityText, max(severityNumber)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"filter":[{"exists":{"field":"flags","boost":1.0}},{"exists":{"field":"severityText","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["severityText","flags","severityNumber"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"flags":{"terms":{"field":"flags","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"severityText":{"terms":{"field":"severityText","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"max(severityNumber)":{"max":{"field":"severityNumber"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_limit.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_limit.yaml deleted file mode 100644 index 533e9d89a5e..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_limit.yaml +++ /dev/null @@ -1,10 +0,0 @@ -calcite: - logical: | - LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalSort(sort0=[$0], dir0=[ASC]) - LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) - LogicalProject(state=[$9], gender=[$4], balance=[$7]) - LogicalFilter(condition=[IS NOT NULL($9)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[gender, balance, state], FILTER->IS NOT NULL($2), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2)), SORT->[0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"state","boost":1.0}},"_source":{"includes":["gender","balance","state"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_span.yaml deleted file mode 100644 index fce35663e64..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_chart_with_span.yaml +++ /dev/null @@ -1,11 +0,0 @@ -calcite: - logical: | - LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalSort(sort0=[$0], dir0=[ASC]) - LogicalProject(age=[$0], max(balance)=[$1]) - LogicalAggregate(group=[{1}], max(balance)=[MAX($0)]) - LogicalProject(balance=[$7], age0=[SPAN($10, 10, null:NULL)]) - LogicalFilter(condition=[IS NOT NULL($10)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[balance, age], FILTER->IS NOT NULL($1), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},max(balance)=MAX($0)), SORT->[0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"age","boost":1.0}},"_source":{"includes":["balance","age"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age0":{"histogram":{"field":"age","missing_bucket":false,"order":"asc","interval":10.0}}}]},"aggregations":{"max(balance)":{"max":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_group_keys.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_group_keys.yaml similarity index 88% rename from integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_group_keys.yaml rename to integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_group_keys.yaml index 21d605b815a..95e83cdcd19 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_group_keys.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_group_keys.yaml @@ -8,7 +8,7 @@ calcite: LogicalProject(gender=[$0], age=[SAFE_CAST($1)], avg(balance)=[$2]) LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) LogicalProject(gender=[$4], age=[$10], balance=[$7]) - LogicalFilter(condition=[IS NOT NULL($4)]) + LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) LogicalProject(age=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) LogicalAggregate(group=[{0}], __grand_total__=[SUM($1)]) @@ -16,7 +16,7 @@ calcite: LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) LogicalProject(gender=[$4], age=[$10], balance=[$7]) - LogicalFilter(condition=[IS NOT NULL($4)]) + LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) @@ -28,7 +28,7 @@ calcite: EnumerableSort(sort0=[$1], dir0=[ASC]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t10]) EnumerableAggregate(group=[{4, 10}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t4)], proj#0..18=[{exprs}], $condition=[$t19]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t4)], expr#20=[IS NOT NULL($t7)], expr#21=[AND($t19, $t20)], proj#0..18=[{exprs}], $condition=[$t21]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) @@ -36,5 +36,5 @@ calcite: EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], age=[$t4], avg(balance)=[$t10]) EnumerableAggregate(group=[{4, 10}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t4)], expr#20=[SAFE_CAST($t10)], expr#21=[IS NOT NULL($t20)], expr#22=[AND($t19, $t21)], proj#0..18=[{exprs}], $condition=[$t22]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t4)], expr#20=[IS NOT NULL($t7)], expr#21=[SAFE_CAST($t10)], expr#22=[IS NOT NULL($t21)], expr#23=[AND($t19, $t20, $t22)], proj#0..18=[{exprs}], $condition=[$t23]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_groups.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_groups.yaml similarity index 100% rename from integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_multiple_groups.yaml rename to integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_groups.yaml diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_null_str.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_null_str.yaml similarity index 87% rename from integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_null_str.yaml rename to integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_null_str.yaml index a0a499e805f..274186e377e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_null_str.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_null_str.yaml @@ -8,7 +8,7 @@ calcite: LogicalProject(gender=[$0], age=[SAFE_CAST($1)], avg(balance)=[$2]) LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) - LogicalFilter(condition=[IS NOT NULL($4)]) + LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($3))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) LogicalProject(age=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) LogicalAggregate(group=[{0}], __grand_total__=[SUM($1)]) @@ -16,7 +16,7 @@ calcite: LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) - LogicalFilter(condition=[IS NOT NULL($4)]) + LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($3))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) physical: | EnumerableLimit(fetch=[10000]) @@ -28,7 +28,7 @@ calcite: EnumerableSort(sort0=[$1], dir0=[ASC]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t10]) EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..12=[{inputs}], expr#13=[10], expr#14=[null:NULL], expr#15=[SPAN($t5, $t13, $t14)], expr#16=[IS NOT NULL($t4)], gender=[$t4], balance=[$t3], age0=[$t15], $condition=[$t16]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[10], expr#14=[null:NULL], expr#15=[SPAN($t5, $t13, $t14)], expr#16=[IS NOT NULL($t4)], expr#17=[IS NOT NULL($t3)], expr#18=[AND($t16, $t17)], gender=[$t4], balance=[$t3], age0=[$t15], $condition=[$t18]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) @@ -36,5 +36,5 @@ calcite: EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], age=[$t4], avg(balance)=[$t10]) EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..12=[{inputs}], expr#13=[10], expr#14=[null:NULL], expr#15=[SPAN($t5, $t13, $t14)], expr#16=[IS NOT NULL($t4)], expr#17=[SAFE_CAST($t15)], expr#18=[IS NOT NULL($t17)], expr#19=[AND($t16, $t18)], gender=[$t4], balance=[$t3], age0=[$t15], $condition=[$t19]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[10], expr#14=[null:NULL], expr#15=[SPAN($t5, $t13, $t14)], expr#16=[IS NOT NULL($t4)], expr#17=[IS NOT NULL($t3)], expr#18=[SAFE_CAST($t15)], expr#19=[IS NOT NULL($t18)], expr#20=[AND($t16, $t17, $t19)], gender=[$t4], balance=[$t3], age0=[$t15], $condition=[$t20]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_single_group.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_single_group.yaml similarity index 100% rename from integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_single_group.yaml rename to integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_single_group.yaml diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_single_group_key.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_single_group_key.yaml similarity index 82% rename from integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_single_group_key.yaml rename to integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_single_group_key.yaml index b711ae1feed..8224f075819 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_single_group_key.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_single_group_key.yaml @@ -4,12 +4,12 @@ calcite: LogicalSort(sort0=[$0], dir0=[ASC]) LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)]) LogicalProject(gender=[$4], balance=[$7]) - LogicalFilter(condition=[IS NOT NULL($4)]) + LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], gender=[$t0], avg(balance)=[$t8]) EnumerableAggregate(group=[{4}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t4)], proj#0..18=[{exprs}], $condition=[$t19]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t4)], expr#20=[IS NOT NULL($t7)], expr#21=[AND($t19, $t20)], proj#0..18=[{exprs}], $condition=[$t21]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_timestamp_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_timestamp_span_and_category.yaml similarity index 86% rename from integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_timestamp_span.yaml rename to integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_timestamp_span_and_category.yaml index 7e968b31dbb..76b833ce3f1 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_timestamp_span.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_timestamp_span_and_category.yaml @@ -8,7 +8,7 @@ calcite: LogicalProject(timestamp=[$1], category=[$0], max(value)=[$2]) LogicalAggregate(group=[{0, 2}], max(value)=[MAX($1)]) LogicalProject(category=[$1], value=[$2], timestamp0=[SPAN($3, 1, 'w')]) - LogicalFilter(condition=[IS NOT NULL($3)]) + LogicalFilter(condition=[AND(IS NOT NULL($3), IS NOT NULL($2))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) LogicalProject(category=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) LogicalAggregate(group=[{0}], __grand_total__=[SUM($1)]) @@ -16,7 +16,7 @@ calcite: LogicalProject(category=[$0], max(value)=[$2]) LogicalAggregate(group=[{0, 2}], max(value)=[MAX($1)]) LogicalProject(category=[$1], value=[$2], timestamp0=[SPAN($3, 1, 'w')]) - LogicalFilter(condition=[IS NOT NULL($3)]) + LogicalFilter(condition=[AND(IS NOT NULL($3), IS NOT NULL($2))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) physical: | EnumerableLimit(fetch=[10000]) @@ -27,12 +27,12 @@ calcite: EnumerableSort(sort0=[$1], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], timestamp=[$t1], category=[$t0], max(value)=[$t2]) EnumerableAggregate(group=[{0, 2}], max(value)=[MAX($1)]) - EnumerableCalc(expr#0..9=[{inputs}], expr#10=[1], expr#11=['w'], expr#12=[SPAN($t3, $t10, $t11)], expr#13=[IS NOT NULL($t3)], category=[$t1], value=[$t2], timestamp0=[$t12], $condition=[$t13]) + EnumerableCalc(expr#0..9=[{inputs}], expr#10=[1], expr#11=['w'], expr#12=[SPAN($t3, $t10, $t11)], expr#13=[IS NOT NULL($t3)], expr#14=[IS NOT NULL($t2)], expr#15=[AND($t13, $t14)], category=[$t1], value=[$t2], timestamp0=[$t12], $condition=[$t15]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], category=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($2)]) EnumerableAggregate(group=[{0, 2}], max(value)=[MAX($1)]) - EnumerableCalc(expr#0..9=[{inputs}], expr#10=[1], expr#11=['w'], expr#12=[SPAN($t3, $t10, $t11)], expr#13=[IS NOT NULL($t3)], expr#14=[IS NOT NULL($t1)], expr#15=[AND($t13, $t14)], category=[$t1], value=[$t2], timestamp0=[$t12], $condition=[$t15]) + EnumerableCalc(expr#0..9=[{inputs}], expr#10=[1], expr#11=['w'], expr#12=[SPAN($t3, $t10, $t11)], expr#13=[IS NOT NULL($t3)], expr#14=[IS NOT NULL($t2)], expr#15=[IS NOT NULL($t1)], expr#16=[AND($t13, $t14, $t15)], category=[$t1], value=[$t2], timestamp0=[$t12], $condition=[$t16]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_use_other.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_use_other.yaml similarity index 86% rename from integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_use_other.yaml rename to integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_use_other.yaml index c5fe280b32d..027d0e30124 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_use_other.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_use_other.yaml @@ -7,7 +7,7 @@ calcite: LogicalJoin(condition=[=($1, $3)], joinType=[left]) LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) - LogicalFilter(condition=[IS NOT NULL($23)]) + LogicalFilter(condition=[AND(IS NOT NULL($23), IS NOT NULL($163))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) LogicalProject(severityText=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) LogicalAggregate(group=[{0}], __grand_total__=[SUM($1)]) @@ -15,7 +15,7 @@ calcite: LogicalProject(severityText=[$1], max(severityNumber)=[$2]) LogicalAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) LogicalProject(flags=[$23], severityText=[$7], severityNumber=[$163]) - LogicalFilter(condition=[IS NOT NULL($23)]) + LogicalFilter(condition=[AND(IS NOT NULL($23), IS NOT NULL($163))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) physical: | EnumerableLimit(fetch=[10000]) @@ -26,12 +26,12 @@ calcite: EnumerableSort(sort0=[$1], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], flags=[$t1], severityText=[$t0], max(severityNumber)=[$t2]) EnumerableAggregate(group=[{7, 23}], max(severityNumber)=[MAX($163)]) - EnumerableCalc(expr#0..171=[{inputs}], expr#172=[IS NOT NULL($t23)], proj#0..171=[{exprs}], $condition=[$t172]) + EnumerableCalc(expr#0..171=[{inputs}], expr#172=[IS NOT NULL($t23)], expr#173=[IS NOT NULL($t163)], expr#174=[AND($t172, $t173)], proj#0..171=[{exprs}], $condition=[$t174]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], severityText=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($2)]) EnumerableAggregate(group=[{7, 23}], max(severityNumber)=[MAX($163)]) - EnumerableCalc(expr#0..171=[{inputs}], expr#172=[IS NOT NULL($t23)], expr#173=[IS NOT NULL($t7)], expr#174=[AND($t172, $t173)], proj#0..171=[{exprs}], $condition=[$t174]) + EnumerableCalc(expr#0..171=[{inputs}], expr#172=[IS NOT NULL($t23)], expr#173=[IS NOT NULL($t163)], expr#174=[IS NOT NULL($t7)], expr#175=[AND($t172, $t173, $t174)], proj#0..171=[{exprs}], $condition=[$t175]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_integer_span.yaml similarity index 80% rename from integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_span.yaml rename to integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_integer_span.yaml index 651934bf035..5e3a5d0ba33 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_span.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_integer_span.yaml @@ -5,11 +5,11 @@ calcite: LogicalProject(age=[$0], max(balance)=[$1]) LogicalAggregate(group=[{1}], max(balance)=[MAX($0)]) LogicalProject(balance=[$7], age0=[SPAN($10, 10, null:NULL)]) - LogicalFilter(condition=[IS NOT NULL($10)]) + LogicalFilter(condition=[AND(IS NOT NULL($10), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableAggregate(group=[{1}], max(balance)=[MAX($0)]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[10], expr#20=[null:NULL], expr#21=[SPAN($t10, $t19, $t20)], expr#22=[IS NOT NULL($t10)], balance=[$t7], age0=[$t21], $condition=[$t22]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[10], expr#20=[null:NULL], expr#21=[SPAN($t10, $t19, $t20)], expr#22=[IS NOT NULL($t10)], expr#23=[IS NOT NULL($t7)], expr#24=[AND($t22, $t23)], balance=[$t7], age0=[$t21], $condition=[$t24]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_limit.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_limit.yaml similarity index 83% rename from integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_limit.yaml rename to integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_limit.yaml index 84ba13a3217..16aa3871687 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_chart_with_limit.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_limit.yaml @@ -4,12 +4,12 @@ calcite: LogicalSort(sort0=[$0], dir0=[ASC]) LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) LogicalProject(state=[$9], gender=[$4], balance=[$7]) - LogicalFilter(condition=[IS NOT NULL($9)]) + LogicalFilter(condition=[AND(IS NOT NULL($9), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], state=[$t1], gender=[$t0], avg(balance)=[$t9]) EnumerableAggregate(group=[{4, 9}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t9)], proj#0..18=[{exprs}], $condition=[$t19]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t9)], expr#20=[IS NOT NULL($t7)], expr#21=[AND($t19, $t20)], proj#0..18=[{exprs}], $condition=[$t21]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_timestamp_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_timestamp_span.yaml new file mode 100644 index 00000000000..a8bd9d61f77 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_timestamp_span.yaml @@ -0,0 +1,15 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalProject(@timestamp=[$0], count=[$1]) + LogicalAggregate(group=[{0}], count=[COUNT()]) + LogicalProject(@timestamp0=[SPAN($0, 1, 'd')]) + LogicalFilter(condition=[IS NOT NULL($0)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableAggregate(group=[{0}], count=[COUNT()]) + EnumerableCalc(expr#0..9=[{inputs}], expr#10=[1], expr#11=['d'], expr#12=[SPAN($t0, $t10, $t11)], expr#13=[IS NOT NULL($t0)], @timestamp0=[$t12], $condition=[$t13]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLChartTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLChartTest.java index 6f8873571d5..338b586ba29 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLChartTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLChartTest.java @@ -82,7 +82,7 @@ public void testChartWithSingleGroupKey() { String expectedSparkSql = "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" - + "WHERE `gender` IS NOT NULL\n" + + "WHERE `gender` IS NOT NULL AND `balance` IS NOT NULL\n" + "GROUP BY `gender`\n" + "ORDER BY `gender` NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -96,7 +96,7 @@ public void testChartWithOverSyntax() { String expectedSparkSql = "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" - + "WHERE `gender` IS NOT NULL\n" + + "WHERE `gender` IS NOT NULL AND `balance` IS NOT NULL\n" + "GROUP BY `gender`\n" + "ORDER BY `gender` NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -114,13 +114,13 @@ public void testChartWithMultipleGroupKeys() { + "FROM (SELECT `gender`, SAFE_CAST(`age` AS STRING) `age`, AVG(`balance`)" + " `avg(balance)`\n" + "FROM `scott`.`bank`\n" - + "WHERE `gender` IS NOT NULL\n" + + "WHERE `gender` IS NOT NULL AND `balance` IS NOT NULL\n" + "GROUP BY `gender`, `age`) `t2`\n" + "LEFT JOIN (SELECT `age`, SUM(`avg(balance)`) `__grand_total__`, ROW_NUMBER() OVER" + " (ORDER BY SUM(`avg(balance)`) DESC) `_row_number_chart_`\n" + "FROM (SELECT SAFE_CAST(`age` AS STRING) `age`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" - + "WHERE `gender` IS NOT NULL\n" + + "WHERE `gender` IS NOT NULL AND `balance` IS NOT NULL\n" + "GROUP BY `gender`, `age`) `t6`\n" + "WHERE `age` IS NOT NULL\n" + "GROUP BY `age`) `t9` ON `t2`.`age` = `t9`.`age`\n" @@ -142,13 +142,13 @@ public void testChartWithMultipleGroupKeysAlternativeSyntax() { + "FROM (SELECT `gender`, SAFE_CAST(`age` AS STRING) `age`, AVG(`balance`)" + " `avg(balance)`\n" + "FROM `scott`.`bank`\n" - + "WHERE `gender` IS NOT NULL\n" + + "WHERE `gender` IS NOT NULL AND `balance` IS NOT NULL\n" + "GROUP BY `gender`, `age`) `t2`\n" + "LEFT JOIN (SELECT `age`, SUM(`avg(balance)`) `__grand_total__`, ROW_NUMBER() OVER" + " (ORDER BY SUM(`avg(balance)`) DESC) `_row_number_chart_`\n" + "FROM (SELECT SAFE_CAST(`age` AS STRING) `age`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" - + "WHERE `gender` IS NOT NULL\n" + + "WHERE `gender` IS NOT NULL AND `balance` IS NOT NULL\n" + "GROUP BY `gender`, `age`) `t6`\n" + "WHERE `age` IS NOT NULL\n" + "GROUP BY `age`) `t9` ON `t2`.`age` = `t9`.`age`\n" @@ -166,7 +166,7 @@ public void testChartWithLimit() { String expectedSparkSql = "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" - + "WHERE `gender` IS NOT NULL\n" + + "WHERE `gender` IS NOT NULL AND `balance` IS NOT NULL\n" + "GROUP BY `gender`\n" + "ORDER BY `gender` NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -180,7 +180,7 @@ public void testChartWithLimitZero() { String expectedSparkSql = "SELECT `state`, `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" - + "WHERE `state` IS NOT NULL\n" + + "WHERE `state` IS NOT NULL AND `balance` IS NOT NULL\n" + "GROUP BY `state`, `gender`\n" + "ORDER BY `state` NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -194,7 +194,7 @@ public void testChartWithSpan() { String expectedSparkSql = "SELECT `SPAN`(`age`, 10, NULL) `age`, MAX(`balance`) `max(balance)`\n" + "FROM `scott`.`bank`\n" - + "WHERE `age` IS NOT NULL\n" + + "WHERE `age` IS NOT NULL AND `balance` IS NOT NULL\n" + "GROUP BY `SPAN`(`age`, 10, NULL)\n" + "ORDER BY 1 NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -212,13 +212,13 @@ public void testChartWithTimeSpan() { + "FROM (SELECT `SPAN`(`timestamp`, 1, 'w') `timestamp`, `category`, MAX(`value`)" + " `max(value)`\n" + "FROM `scott`.`time_data`\n" - + "WHERE `timestamp` IS NOT NULL\n" + + "WHERE `timestamp` IS NOT NULL AND `value` IS NOT NULL\n" + "GROUP BY `category`, `SPAN`(`timestamp`, 1, 'w')) `t2`\n" + "LEFT JOIN (SELECT `category`, SUM(`max(value)`) `__grand_total__`, ROW_NUMBER() OVER" + " (ORDER BY SUM(`max(value)`) DESC) `_row_number_chart_`\n" + "FROM (SELECT `category`, MAX(`value`) `max(value)`\n" + "FROM `scott`.`time_data`\n" - + "WHERE `timestamp` IS NOT NULL\n" + + "WHERE `timestamp` IS NOT NULL AND `value` IS NOT NULL\n" + "GROUP BY `category`, `SPAN`(`timestamp`, 1, 'w')) `t6`\n" + "WHERE `category` IS NOT NULL\n" + "GROUP BY `category`) `t9` ON `t2`.`category` = `t9`.`category`\n" @@ -236,7 +236,7 @@ public void testChartWithUseOtherTrue() { String expectedSparkSql = "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" - + "WHERE `gender` IS NOT NULL\n" + + "WHERE `gender` IS NOT NULL AND `balance` IS NOT NULL\n" + "GROUP BY `gender`\n" + "ORDER BY `gender` NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -250,7 +250,7 @@ public void testChartWithUseOtherFalse() { String expectedSparkSql = "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" - + "WHERE `gender` IS NOT NULL\n" + + "WHERE `gender` IS NOT NULL AND `balance` IS NOT NULL\n" + "GROUP BY `gender`\n" + "ORDER BY `gender` NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -264,7 +264,7 @@ public void testChartWithOtherStr() { String expectedSparkSql = "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" - + "WHERE `gender` IS NOT NULL\n" + + "WHERE `gender` IS NOT NULL AND `balance` IS NOT NULL\n" + "GROUP BY `gender`\n" + "ORDER BY `gender` NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -278,7 +278,7 @@ public void testChartWithNullStr() { String expectedSparkSql = "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" - + "WHERE `gender` IS NOT NULL\n" + + "WHERE `gender` IS NOT NULL AND `balance` IS NOT NULL\n" + "GROUP BY `gender`\n" + "ORDER BY `gender` NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -292,7 +292,7 @@ public void testChartWithUseNull() { String expectedSparkSql = "SELECT `gender`, AVG(`balance`) `avg(balance)`\n" + "FROM `scott`.`bank`\n" - + "WHERE `gender` IS NOT NULL\n" + + "WHERE `gender` IS NOT NULL AND `balance` IS NOT NULL\n" + "GROUP BY `gender`\n" + "ORDER BY `gender` NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); From 42fde26f6caba4979b26ffdb4c70c2d022a94262 Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Thu, 6 Nov 2025 15:16:46 +0800 Subject: [PATCH 33/35] Fix chart plans Signed-off-by: Yuanchun Shen --- .../expectedOutput/calcite/chart_multiple_group_keys.yaml | 4 ++-- .../test/resources/expectedOutput/calcite/chart_null_str.yaml | 4 ++-- .../calcite/chart_timestamp_span_and_category.yaml | 4 ++-- .../resources/expectedOutput/calcite/chart_use_other.yaml | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_multiple_group_keys.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_multiple_group_keys.yaml index 4b7b86e69f9..d692ba70f69 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/chart_multiple_group_keys.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_multiple_group_keys.yaml @@ -27,10 +27,10 @@ calcite: EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) EnumerableSort(sort0=[$1], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[SAFE_CAST($t1)], gender=[$t0], age=[$t3], avg(balance)=[$t2]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[gender, balance, age], FILTER->AND(IS NOT NULL($0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"filter":[{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["gender","balance","age"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[gender, balance, age], FILTER->AND(IS NOT NULL($0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["gender","balance","age"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) EnumerableCalc(expr#0..1=[{inputs}], expr#2=[SAFE_CAST($t0)], expr#3=[IS NOT NULL($t2)], age=[$t2], avg(balance)=[$t1], $condition=[$t3]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[gender, balance, age], FILTER->AND(IS NOT NULL($0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2)), PROJECT->[age, avg(balance)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"filter":[{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["gender","balance","age"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[gender, balance, age], FILTER->AND(IS NOT NULL($0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2)), PROJECT->[age, avg(balance)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["gender","balance","age"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml index 78eb63241dc..70ca93b7ddf 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml @@ -29,7 +29,7 @@ calcite: EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t10]) EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], gender=[$t1], balance=[$t0], age0=[$t5]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->AND(IS NOT NULL($1), IS NOT NULL($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"filter":[{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["balance","gender","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->AND(IS NOT NULL($1), IS NOT NULL($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["balance","gender","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) @@ -37,4 +37,4 @@ calcite: EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], expr#11=[IS NOT NULL($t4)], age=[$t4], avg(balance)=[$t10], $condition=[$t11]) EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], gender=[$t1], balance=[$t0], age0=[$t5]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->AND(IS NOT NULL($1), IS NOT NULL($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"filter":[{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BLp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0FGRV9DQVNUIiwKICAgICAgICAia2luZCI6ICJTQUZFX0NBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiU1BBTiIsCiAgICAgICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IG51bGwsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0sCiAgICAgICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0sCiAgICAgICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["balance","gender","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->AND(IS NOT NULL($1), IS NOT NULL($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["balance","gender","age"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_timestamp_span_and_category.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_timestamp_span_and_category.yaml index 696edc9ea35..ef2320d13c5 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/chart_timestamp_span_and_category.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_timestamp_span_and_category.yaml @@ -24,9 +24,9 @@ calcite: EnumerableAggregate(group=[{0, 1}], max(value)=[MAX($2)]) EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], timestamp=[$t0], category=[$t10], max(value)=[$t2]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[PROJECT->[category, value, timestamp], FILTER->AND(IS NOT NULL($2), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},max(value)=MAX($1)), PROJECT->[timestamp0, category, max(value)], SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"timestamp","boost":1.0}},{"exists":{"field":"value","boost":1.0}}],"filter":[{"exists":{"field":"category","boost":1.0}},{"exists":{"field":"category","boost":1.0}},{"exists":{"field":"category","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["category","value","timestamp"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"timestamp0":{"date_histogram":{"field":"timestamp","missing_bucket":false,"order":"asc","calendar_interval":"1w"}}}]},"aggregations":{"max(value)":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[PROJECT->[category, value, timestamp], FILTER->AND(IS NOT NULL($2), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},max(value)=MAX($1)), PROJECT->[timestamp0, category, max(value)], SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"timestamp","boost":1.0}},{"exists":{"field":"value","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["category","value","timestamp"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"timestamp0":{"date_histogram":{"field":"timestamp","missing_bucket":false,"order":"asc","calendar_interval":"1w"}}}]},"aggregations":{"max(value)":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], category=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[PROJECT->[category, value, timestamp], FILTER->AND(IS NOT NULL($2), IS NOT NULL($1)), FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},max(value)=MAX($1)), PROJECT->[category, max(value)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"timestamp","boost":1.0}},{"exists":{"field":"value","boost":1.0}}],"filter":[{"exists":{"field":"category","boost":1.0}},{"exists":{"field":"category","boost":1.0}},{"exists":{"field":"category","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["category","value","timestamp"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"timestamp0":{"date_histogram":{"field":"timestamp","missing_bucket":false,"order":"asc","calendar_interval":"1w"}}}]},"aggregations":{"max(value)":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[PROJECT->[category, value, timestamp], FILTER->AND(IS NOT NULL($2), IS NOT NULL($1)), FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},max(value)=MAX($1)), PROJECT->[category, max(value)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"filter":[{"bool":{"must":[{"exists":{"field":"timestamp","boost":1.0}},{"exists":{"field":"value","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"category","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["category","value","timestamp"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"timestamp0":{"date_histogram":{"field":"timestamp","missing_bucket":false,"order":"asc","calendar_interval":"1w"}}}]},"aggregations":{"max(value)":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_use_other.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_use_other.yaml index 33fbbdb4c86..4ee0759468f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/chart_use_other.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_use_other.yaml @@ -23,9 +23,9 @@ calcite: EnumerableAggregate(group=[{0, 1}], max(severityNumber)=[MAX($2)]) EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[2], expr#8=[<=($t4, $t7)], expr#9=['max_among_other'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], flags=[$t0], severityText=[$t10], max(severityNumber)=[$t2]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[PROJECT->[severityText, flags, severityNumber], FILTER->AND(IS NOT NULL($1), IS NOT NULL($2)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},max(severityNumber)=MAX($2)), SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"flags","boost":1.0}},{"exists":{"field":"severityNumber","boost":1.0}}],"filter":[{"exists":{"field":"severityText","boost":1.0}},{"exists":{"field":"severityText","boost":1.0}},{"exists":{"field":"severityText","boost":1.0}},{"exists":{"field":"severityText","boost":1.0}},{"exists":{"field":"severityText","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["severityText","flags","severityNumber"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"severityText":{"terms":{"field":"severityText","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"flags":{"terms":{"field":"flags","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"max(severityNumber)":{"max":{"field":"severityNumber"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[PROJECT->[severityText, flags, severityNumber], FILTER->AND(IS NOT NULL($1), IS NOT NULL($2)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},max(severityNumber)=MAX($2)), SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"flags","boost":1.0}},{"exists":{"field":"severityNumber","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["severityText","flags","severityNumber"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"severityText":{"terms":{"field":"severityText","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"flags":{"terms":{"field":"flags","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"max(severityNumber)":{"max":{"field":"severityNumber"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], severityText=[$t0], $1=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[PROJECT->[severityText, flags, severityNumber], FILTER->AND(IS NOT NULL($1), IS NOT NULL($2)), FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},max(severityNumber)=MAX($2)), PROJECT->[severityText, max(severityNumber)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"flags","boost":1.0}},{"exists":{"field":"severityNumber","boost":1.0}}],"filter":[{"exists":{"field":"severityText","boost":1.0}},{"exists":{"field":"severityText","boost":1.0}},{"exists":{"field":"severityText","boost":1.0}},{"exists":{"field":"severityText","boost":1.0}},{"exists":{"field":"severityText","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["severityText","flags","severityNumber"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"flags":{"terms":{"field":"flags","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"severityText":{"terms":{"field":"severityText","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"max(severityNumber)":{"max":{"field":"severityNumber"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[PROJECT->[severityText, flags, severityNumber], FILTER->AND(IS NOT NULL($1), IS NOT NULL($2)), FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},max(severityNumber)=MAX($2)), PROJECT->[severityText, max(severityNumber)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"filter":[{"bool":{"must":[{"exists":{"field":"flags","boost":1.0}},{"exists":{"field":"severityNumber","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"severityText","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["severityText","flags","severityNumber"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"flags":{"terms":{"field":"flags","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"severityText":{"terms":{"field":"severityText","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"max(severityNumber)":{"max":{"field":"severityNumber"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) From ed159ba34acaf387cf1f604d3c91477c834c7bb7 Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Thu, 6 Nov 2025 15:23:59 +0800 Subject: [PATCH 34/35] Get rid of record class Signed-off-by: Yuanchun Shen --- .../sql/calcite/CalciteRelNodeVisitor.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index a8c884f2bb9..1777ea67155 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -42,6 +42,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; +import lombok.AllArgsConstructor; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.plan.ViewExpanders; @@ -2541,8 +2542,15 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { return relBuilder.peek(); } - private record ChartConfig( - int limit, boolean top, boolean useOther, boolean useNull, String otherStr, String nullStr) { + @AllArgsConstructor + private static class ChartConfig { + private final int limit; + private final boolean top; + private final boolean useOther; + private final boolean useNull; + private final String otherStr; + private final String nullStr; + static ChartConfig fromArguments(ArgumentMap argMap) { int limit = (Integer) argMap.getOrDefault("limit", Chart.DEFAULT_LIMIT).getValue(); boolean top = (Boolean) argMap.getOrDefault("top", Chart.DEFAULT_TOP).getValue(); From 48eee8d3f57dd62333fed8a2144aefe894038515 Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Thu, 6 Nov 2025 17:37:59 +0800 Subject: [PATCH 35/35] Move ranking by column split to a helper function Signed-off-by: Yuanchun Shen --- .../sql/calcite/CalciteRelNodeVisitor.java | 83 ++++++++++--------- 1 file changed, 46 insertions(+), 37 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 1777ea67155..94086952610 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -2415,9 +2415,9 @@ private String getAggFieldAlias(UnresolvedExpression aggregateFunction) { public RelNode visitChart(Chart node, CalcitePlanContext context) { visitChildren(node, context); ArgumentMap argMap = ArgumentMap.of(node.getArguments()); + ChartConfig config = ChartConfig.fromArguments(argMap); List groupExprList = Stream.of(node.getRowSplit(), node.getColumnSplit()).filter(Objects::nonNull).toList(); - ChartConfig config = ChartConfig.fromArguments(argMap); Aggregation aggregation = new Aggregation( List.of(node.getAggregationFunction()), List.of(), groupExprList, null, List.of()); @@ -2441,15 +2441,6 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { return relBuilder.peek(); } - String aggFunctionName = getAggFunctionName(node.getAggregationFunction()); - BuiltinFunctionName aggFunction = - BuiltinFunctionName.of(aggFunctionName) - .orElseThrow( - () -> - new IllegalArgumentException( - StringUtils.format( - "Unrecognized aggregation function: %s", aggFunctionName))); - // Convert the column split to string if necessary: column split was supposed to be pivoted to // column names. This guarantees that its type compatibility with useother and usenull RexNode colSplit = relBuilder.field(1); @@ -2463,34 +2454,8 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { } relBuilder.project(relBuilder.field(0), colSplit, relBuilder.field(2)); RelNode aggregated = relBuilder.peek(); - // 1: column-split, 2: agg - relBuilder.project(relBuilder.field(1), relBuilder.field(2)); - // Make sure that rows who don't have a column split not interfere grand total calculation - relBuilder.filter(relBuilder.isNotNull(relBuilder.field(0))); - final String GRAND_TOTAL_COL = "__grand_total__"; - relBuilder.aggregate( - relBuilder.groupKey(relBuilder.field(0)), - // Top-K semantic: Retain categories whose summed values are among the greatest - relBuilder.sum(relBuilder.field(1)).as(GRAND_TOTAL_COL)); // results: group key, agg calls - RexNode grandTotal = relBuilder.field(GRAND_TOTAL_COL); - // Apply sorting: keep the max values if top is set - if (config.top) { - grandTotal = relBuilder.desc(grandTotal); - } - // Always set it to null last so that nulls don't interfere with top / bottom calculation - grandTotal = relBuilder.nullsLast(grandTotal); - RexNode rowNum = - PlanUtils.makeOver( - context, - BuiltinFunctionName.ROW_NUMBER, - relBuilder.literal(1), // dummy expression for row number calculation - List.of(), - List.of(), - List.of(grandTotal), - WindowFrame.toCurrentRow()); - relBuilder.projectPlus(relBuilder.alias(rowNum, PlanUtils.ROW_NUMBER_COLUMN_FOR_CHART)); - RelNode ranked = relBuilder.build(); + RelNode ranked = rankByColumnSplit(context, 1, 2, config.top); relBuilder.push(aggregated); relBuilder.push(ranked); @@ -2534,6 +2499,14 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { relBuilder.field(0), relBuilder.alias(columnSplitExpr, columnSplitName), relBuilder.field(2)); + String aggFunctionName = getAggFunctionName(node.getAggregationFunction()); + BuiltinFunctionName aggFunction = + BuiltinFunctionName.of(aggFunctionName) + .orElseThrow( + () -> + new IllegalArgumentException( + StringUtils.format( + "Unrecognized aggregation function: %s", aggFunctionName))); relBuilder.aggregate( relBuilder.groupKey(relBuilder.field(0), relBuilder.field(1)), buildAggCall(context.relBuilder, aggFunction, relBuilder.field(2)).as(aggFieldName)); @@ -2542,6 +2515,42 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { return relBuilder.peek(); } + /** + * Aggregate by column split then rank by grand total (summed value of each category). The output + * is [col-split, grand-total, row-number] + */ + private RelNode rankByColumnSplit( + CalcitePlanContext context, int columnSplitOrdinal, int aggOrdinal, boolean top) { + RelBuilder relBuilder = context.relBuilder; + + relBuilder.project(relBuilder.field(columnSplitOrdinal), relBuilder.field(aggOrdinal)); + // Make sure that rows who don't have a column split not interfere grand total calculation + relBuilder.filter(relBuilder.isNotNull(relBuilder.field(0))); + final String GRAND_TOTAL_COL = "__grand_total__"; + relBuilder.aggregate( + relBuilder.groupKey(relBuilder.field(0)), + // Top-K semantic: Retain categories whose summed values are among the greatest + relBuilder.sum(relBuilder.field(1)).as(GRAND_TOTAL_COL)); // results: group key, agg calls + RexNode grandTotal = relBuilder.field(GRAND_TOTAL_COL); + // Apply sorting: keep the max values if top is set + if (top) { + grandTotal = relBuilder.desc(grandTotal); + } + // Always set it to null last so that nulls don't interfere with top / bottom calculation + grandTotal = relBuilder.nullsLast(grandTotal); + RexNode rowNum = + PlanUtils.makeOver( + context, + BuiltinFunctionName.ROW_NUMBER, + relBuilder.literal(1), // dummy expression for row number calculation + List.of(), + List.of(), + List.of(grandTotal), + WindowFrame.toCurrentRow()); + relBuilder.projectPlus(relBuilder.alias(rowNum, PlanUtils.ROW_NUMBER_COLUMN_FOR_CHART)); + return relBuilder.build(); + } + @AllArgsConstructor private static class ChartConfig { private final int limit;