Skip to content

Commit 788da98

Browse files
songkant-awssongkant-aws
authored
Revert stream pattern method in V2 and implement SIMPLE_PATTERN in Calcite (#3553)
* Revert simple_pattern window function change to recover pushdown ability Signed-off-by: Songkan Tang <[email protected]> * Add SIMPLE_PATTERN patterns command support based on parse command Signed-off-by: Songkan Tang <[email protected]> * Address minor comments Signed-off-by: Songkan Tang <[email protected]> * Address comments part 2 Signed-off-by: Songkan Tang <[email protected]> * Make allowCast for pattern VARCHAR literal Signed-off-by: Songkan Tang <[email protected]> * Fix spotless Signed-off-by: Songkan Tang <[email protected]> * Minor ut failure fix Signed-off-by: Songkan Tang <[email protected]> * Add patterns command push down cases in ExplainIT Signed-off-by: Songkan Tang <[email protected]> * Fix patterns ExplainIT failure Signed-off-by: Songkan Tang <[email protected]> * Rename visitWindow to visitPatterns Signed-off-by: Songkan Tang <[email protected]> * Correct ExplainIT expected plan for patterns Signed-off-by: Songkan Tang <[email protected]> * Add patterns anonymizer test for SIMPLE_PATTERN Signed-off-by: Songkan Tang <[email protected]> --------- Signed-off-by: Songkan Tang <[email protected]>
1 parent a10ccd2 commit 788da98

File tree

30 files changed

+461
-299
lines changed

30 files changed

+461
-299
lines changed

‎core/src/main/java/org/opensearch/sql/analysis/Analyzer.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@
6161
import org.opensearch.sql.ast.tree.ML;
6262
import org.opensearch.sql.ast.tree.Paginate;
6363
import org.opensearch.sql.ast.tree.Parse;
64+
import org.opensearch.sql.ast.tree.Patterns;
6465
import org.opensearch.sql.ast.tree.Project;
6566
import org.opensearch.sql.ast.tree.RareTopN;
6667
import org.opensearch.sql.ast.tree.Relation;
@@ -73,7 +74,6 @@
7374
import org.opensearch.sql.ast.tree.Trendline;
7475
import org.opensearch.sql.ast.tree.UnresolvedPlan;
7576
import org.opensearch.sql.ast.tree.Values;
76-
import org.opensearch.sql.ast.tree.Window;
7777
import org.opensearch.sql.common.antlr.SyntaxCheckException;
7878
import org.opensearch.sql.data.model.ExprMissingValue;
7979
import org.opensearch.sql.data.type.ExprCoreType;
@@ -493,7 +493,7 @@ public LogicalPlan visitParse(Parse node, AnalysisContext context) {
493493
}
494494

495495
@Override
496-
public LogicalPlan visitWindow(Window node, AnalysisContext context) {
496+
public LogicalPlan visitPatterns(Patterns node, AnalysisContext context) {
497497
LogicalPlan child = node.getChild().get(0).accept(this, context);
498498
WindowExpressionAnalyzer windowAnalyzer =
499499
new WindowExpressionAnalyzer(expressionAnalyzer, child);

‎core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@
5959
import org.opensearch.sql.ast.tree.ML;
6060
import org.opensearch.sql.ast.tree.Paginate;
6161
import org.opensearch.sql.ast.tree.Parse;
62+
import org.opensearch.sql.ast.tree.Patterns;
6263
import org.opensearch.sql.ast.tree.Project;
6364
import org.opensearch.sql.ast.tree.RareTopN;
6465
import org.opensearch.sql.ast.tree.Relation;
@@ -69,7 +70,6 @@
6970
import org.opensearch.sql.ast.tree.TableFunction;
7071
import org.opensearch.sql.ast.tree.Trendline;
7172
import org.opensearch.sql.ast.tree.Values;
72-
import org.opensearch.sql.ast.tree.Window;
7373

7474
/** AST nodes visitor Defines the traverse path. */
7575
public abstract class AbstractNodeVisitor<T, C> {
@@ -343,8 +343,8 @@ public T visitFillNull(FillNull fillNull, C context) {
343343
return visitChildren(fillNull, context);
344344
}
345345

346-
public T visitWindow(Window window, C context) {
347-
return visitChildren(window, context);
346+
public T visitPatterns(Patterns patterns, C context) {
347+
return visitChildren(patterns, context);
348348
}
349349

350350
public T visitJoin(Join node, C context) {

‎core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@
5858
import org.opensearch.sql.ast.tree.Head;
5959
import org.opensearch.sql.ast.tree.Limit;
6060
import org.opensearch.sql.ast.tree.Parse;
61+
import org.opensearch.sql.ast.tree.Patterns;
6162
import org.opensearch.sql.ast.tree.Project;
6263
import org.opensearch.sql.ast.tree.RareTopN;
6364
import org.opensearch.sql.ast.tree.RareTopN.CommandType;
@@ -71,7 +72,6 @@
7172
import org.opensearch.sql.ast.tree.Trendline;
7273
import org.opensearch.sql.ast.tree.UnresolvedPlan;
7374
import org.opensearch.sql.ast.tree.Values;
74-
import org.opensearch.sql.ast.tree.Window;
7575

7676
/** Class of static methods to create specific node instances. */
7777
@UtilityClass
@@ -504,7 +504,7 @@ public static Parse parse(
504504
return new Parse(parseMethod, sourceField, pattern, arguments, input);
505505
}
506506

507-
public static Window window(
507+
public static Patterns patterns(
508508
UnresolvedPlan input,
509509
PatternMethod patternMethod,
510510
UnresolvedExpression sourceField,
@@ -513,7 +513,7 @@ public static Window window(
513513
List<UnresolvedExpression> funArgs = new ArrayList<>();
514514
funArgs.add(sourceField);
515515
funArgs.addAll(arguments);
516-
return new Window(
516+
return new Patterns(
517517
new Alias(
518518
alias,
519519
new WindowFunction(

‎core/src/main/java/org/opensearch/sql/ast/expression/ParseMethod.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@
1111
@RequiredArgsConstructor
1212
public enum ParseMethod {
1313
REGEX("regex"),
14-
GROK("grok");
14+
GROK("grok"),
15+
PATTERNS("patterns");
1516

1617
@Getter private final String name;
1718
}

‎core/src/main/java/org/opensearch/sql/ast/tree/Window.java renamed to ‎core/src/main/java/org/opensearch/sql/ast/tree/Patterns.java

+7-3
Original file line numberDiff line numberDiff line change
@@ -16,20 +16,24 @@
1616
import org.opensearch.sql.ast.AbstractNodeVisitor;
1717
import org.opensearch.sql.ast.expression.UnresolvedExpression;
1818

19+
/**
20+
* Logical plan node of Patterns command to represent complex nested function calling than single
21+
* window function.
22+
*/
1923
@Getter
2024
@Setter
2125
@ToString
2226
@EqualsAndHashCode(callSuper = false)
2327
@RequiredArgsConstructor
2428
@AllArgsConstructor
25-
public class Window extends UnresolvedPlan {
29+
public class Patterns extends UnresolvedPlan {
2630

2731
private final UnresolvedExpression windowFunction;
2832

2933
private UnresolvedPlan child;
3034

3135
@Override
32-
public Window attach(UnresolvedPlan child) {
36+
public Patterns attach(UnresolvedPlan child) {
3337
this.child = child;
3438
return this;
3539
}
@@ -41,6 +45,6 @@ public List<UnresolvedPlan> getChild() {
4145

4246
@Override
4347
public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) {
44-
return nodeVisitor.visitWindow(this, context);
48+
return nodeVisitor.visitPatterns(this, context);
4549
}
4650
}

‎core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java

+15-6
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import static org.opensearch.sql.ast.tree.Sort.SortOrder.ASC;
1313
import static org.opensearch.sql.ast.tree.Sort.SortOrder.DESC;
1414

15+
import com.google.common.base.Strings;
1516
import com.google.common.collect.ImmutableList;
1617
import com.google.common.collect.Iterables;
1718
import java.util.ArrayList;
@@ -31,8 +32,8 @@
3132
import org.apache.calcite.rex.RexLiteral;
3233
import org.apache.calcite.rex.RexNode;
3334
import org.apache.calcite.rex.RexWindowBounds;
34-
import org.apache.calcite.sql.fun.SqlLibraryOperators;
3535
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
36+
import org.apache.calcite.sql.type.SqlTypeName;
3637
import org.apache.calcite.tools.RelBuilder;
3738
import org.apache.calcite.tools.RelBuilder.AggCall;
3839
import org.apache.calcite.util.Holder;
@@ -78,6 +79,7 @@
7879
import org.opensearch.sql.calcite.utils.JoinAndLookupUtils;
7980
import org.opensearch.sql.exception.CalciteUnsupportedException;
8081
import org.opensearch.sql.exception.SemanticCheckException;
82+
import org.opensearch.sql.expression.function.PPLFuncImpTable;
8183
import org.opensearch.sql.utils.ParseUtils;
8284

8385
public class CalciteRelNodeVisitor extends AbstractNodeVisitor<RelNode, CalcitePlanContext> {
@@ -283,21 +285,28 @@ public RelNode visitHead(Head node, CalcitePlanContext context) {
283285
@Override
284286
public RelNode visitParse(Parse node, CalcitePlanContext context) {
285287
visitChildren(node, context);
286-
List<String> originalFieldNames = context.relBuilder.peek().getRowType().getFieldNames();
287288
RexNode sourceField = rexVisitor.analyze(node.getSourceField(), context);
288289
ParseMethod parseMethod = node.getParseMethod();
289290
java.util.Map<String, Literal> arguments = node.getArguments();
290-
String pattern = (String) node.getPattern().getValue();
291+
String patternValue = (String) node.getPattern().getValue();
292+
String pattern =
293+
ParseMethod.PATTERNS.equals(parseMethod) && Strings.isNullOrEmpty(patternValue)
294+
? "[a-zA-Z0-9]"
295+
: patternValue;
291296
List<String> groupCandidates =
292297
ParseUtils.getNamedGroupCandidates(parseMethod, pattern, arguments);
293298
List<RexNode> newFields =
294299
groupCandidates.stream()
295300
.map(
296301
group ->
297-
context.rexBuilder.makeCall(
298-
SqlLibraryOperators.REGEXP_EXTRACT,
302+
PPLFuncImpTable.INSTANCE.resolve(
303+
context.rexBuilder,
304+
ParseUtils.BUILTIN_FUNCTION_MAP.get(parseMethod),
299305
sourceField,
300-
context.rexBuilder.makeLiteral(pattern)))
306+
context.rexBuilder.makeLiteral(
307+
pattern,
308+
context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR),
309+
true)))
301310
.toList();
302311
projectPlusOverriding(newFields, groupCandidates, context);
303312
return context.relBuilder.peek();

‎core/src/main/java/org/opensearch/sql/expression/DSL.java

-4
Original file line numberDiff line numberDiff line change
@@ -990,10 +990,6 @@ public static FunctionExpression brain(Expression... args) {
990990
return compile(FunctionProperties.None, BuiltinFunctionName.BRAIN, args);
991991
}
992992

993-
public static FunctionExpression simple_pattern(Expression... args) {
994-
return compile(FunctionProperties.None, BuiltinFunctionName.SIMPLE_PATTERN, args);
995-
}
996-
997993
@SuppressWarnings("unchecked")
998994
private static <T extends FunctionImplementation> T compile(
999995
FunctionProperties functionProperties, BuiltinFunctionName bfn, Expression... args) {

‎core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java

+8-2
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,13 @@
99
import java.util.Locale;
1010
import java.util.Map;
1111
import java.util.Optional;
12+
import lombok.AllArgsConstructor;
1213
import lombok.Getter;
1314
import lombok.RequiredArgsConstructor;
1415

1516
/** Builtin Function Name. */
1617
@Getter
18+
@AllArgsConstructor
1719
@RequiredArgsConstructor
1820
public enum BuiltinFunctionName {
1921
/** Mathematical Functions. */
@@ -224,7 +226,6 @@ public enum BuiltinFunctionName {
224226
RANK(FunctionName.of("rank")),
225227
DENSE_RANK(FunctionName.of("dense_rank")),
226228

227-
SIMPLE_PATTERN(FunctionName.of("simple_pattern")),
228229
BRAIN(FunctionName.of("brain")),
229230

230231
INTERVAL(FunctionName.of("interval")),
@@ -268,9 +269,14 @@ public enum BuiltinFunctionName {
268269
MULTIMATCH(FunctionName.of("multimatch")),
269270
MULTIMATCHQUERY(FunctionName.of("multimatchquery")),
270271
WILDCARDQUERY(FunctionName.of("wildcardquery")),
271-
WILDCARD_QUERY(FunctionName.of("wildcard_query"));
272+
WILDCARD_QUERY(FunctionName.of("wildcard_query")),
273+
274+
/** Internal functions that are not exposed to customers. */
275+
INTERNAL_REGEXP_EXTRACT(FunctionName.of("regexp_extract"), true),
276+
INTERNAL_REGEXP_REPLACE_2(FunctionName.of("regexp_replace_2"), true);
272277

273278
private final FunctionName name;
279+
private boolean isInternal;
274280

275281
private static final Map<FunctionName, BuiltinFunctionName> ALL_NATIVE_FUNCTIONS;
276282

‎core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java

+4
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@
2828
import static org.opensearch.sql.expression.function.BuiltinFunctionName.FLOOR;
2929
import static org.opensearch.sql.expression.function.BuiltinFunctionName.GREATER;
3030
import static org.opensearch.sql.expression.function.BuiltinFunctionName.GTE;
31+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_REGEXP_EXTRACT;
32+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_REGEXP_REPLACE_2;
3133
import static org.opensearch.sql.expression.function.BuiltinFunctionName.IS_NOT_NULL;
3234
import static org.opensearch.sql.expression.function.BuiltinFunctionName.IS_NULL;
3335
import static org.opensearch.sql.expression.function.BuiltinFunctionName.LEFT;
@@ -268,6 +270,8 @@ void populate() {
268270
registerOperator(RIGHT, SqlLibraryOperators.RIGHT);
269271
registerOperator(LEFT, SqlLibraryOperators.LEFT);
270272
registerOperator(LOG2, SqlLibraryOperators.LOG2);
273+
registerOperator(INTERNAL_REGEXP_EXTRACT, SqlLibraryOperators.REGEXP_EXTRACT);
274+
registerOperator(INTERNAL_REGEXP_REPLACE_2, SqlLibraryOperators.REGEXP_REPLACE_2);
271275

272276
// Register PPL UDF operator
273277
registerOperator(SPAN, PPLBuiltinOperators.SPAN);

‎core/src/main/java/org/opensearch/sql/expression/window/WindowFunctions.java

-15
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
import org.opensearch.sql.expression.function.FunctionName;
2222
import org.opensearch.sql.expression.function.FunctionSignature;
2323
import org.opensearch.sql.expression.window.patterns.BufferPatternWindowFunction;
24-
import org.opensearch.sql.expression.window.patterns.StreamPatternWindowFunction;
2524
import org.opensearch.sql.expression.window.ranking.DenseRankFunction;
2625
import org.opensearch.sql.expression.window.ranking.RankFunction;
2726
import org.opensearch.sql.expression.window.ranking.RankingWindowFunction;
@@ -41,7 +40,6 @@ public void register(BuiltinFunctionRepository repository) {
4140
repository.register(rank());
4241
repository.register(denseRank());
4342
repository.register(brain());
44-
repository.register(simplePattern());
4543
}
4644

4745
private DefaultFunctionResolver rowNumber() {
@@ -77,19 +75,6 @@ private DefaultFunctionResolver brain() {
7775
functionBuilder));
7876
}
7977

80-
private DefaultFunctionResolver simplePattern() {
81-
FunctionName functionName = BuiltinFunctionName.SIMPLE_PATTERN.getName();
82-
FunctionBuilder functionBuilder =
83-
(functionProperties, arguments) -> new StreamPatternWindowFunction(arguments);
84-
return new DefaultFunctionResolver(
85-
functionName,
86-
ImmutableMap.of(
87-
new FunctionSignature(functionName, ImmutableList.of(STRING)), functionBuilder,
88-
new FunctionSignature(functionName, ImmutableList.of(STRING, STRING)), functionBuilder,
89-
new FunctionSignature(functionName, ImmutableList.of(STRING, STRING, STRING)),
90-
functionBuilder));
91-
}
92-
9378
private DefaultFunctionResolver rankingFunction(
9479
FunctionName functionName, Supplier<RankingWindowFunction> constructor) {
9580
FunctionSignature functionSignature = new FunctionSignature(functionName, emptyList());

‎core/src/main/java/org/opensearch/sql/expression/window/patterns/StreamPatternWindowFunction.java

-70
This file was deleted.

0 commit comments

Comments
 (0)