From 01c6b1ee2e762802dc64c07e70b812962366045d Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Wed, 19 Nov 2025 15:20:13 +0800 Subject: [PATCH 01/10] lewSupport basic nested queries when applying filter on a nested filed Signed-off-by: Yuanchun Shen # Conflicts: # integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java # opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java --- .../sql/calcite/remote/CalciteExplainIT.java | 15 +++ .../calcite/filter_on_nested.yaml | 9 ++ .../calcite_no_pushdown/filter_on_nested.yaml | 10 ++ .../rest-api-spec/test/issues/4508.yml | 94 +++++++++++++++++++ .../opensearch/request/PredicateAnalyzer.java | 56 ++++++++++- .../storage/scan/CalciteLogicalIndexScan.java | 4 +- 6 files changed, 184 insertions(+), 4 deletions(-) create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/filter_on_nested.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_on_nested.yaml create mode 100644 integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4508.yml diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index b6cd327989a..81fcf4682ac 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -9,6 +9,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ALIAS; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DEEP_NESTED; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_LOGS; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_SIMPLE; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_OTEL_LOGS; @@ -27,6 +28,8 @@ import org.junit.Test; import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.common.setting.Settings.Key; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.ppl.ExplainIT; import org.opensearch.sql.protocol.response.format.Format; @@ -47,6 +50,7 @@ public void init() throws Exception { loadIndex(Index.WORK_INFORMATION); loadIndex(Index.WEBLOG); loadIndex(Index.DATA_TYPE_ALIAS); + loadIndex(Index.DEEP_NESTED); } @Override @@ -2368,4 +2372,15 @@ public void testExplainBWC() throws IOException { explainQueryToStringBWC(query, format)); } } + + @Test + public void testFilterOnNestedFields() throws IOException { + assertYamlEqualsIgnoreId( + loadExpectedPlan("filter_on_nested.yaml"), + explainQueryYaml( + StringUtils.format( + "source=%s | eval proj_name_len=length(projects.name) | fields projects.name," + + " proj_name_len | where proj_name_len > 29", + TEST_INDEX_DEEP_NESTED))); + } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/filter_on_nested.yaml b/integ-test/src/test/resources/expectedOutput/calcite/filter_on_nested.yaml new file mode 100644 index 00000000000..e646df565b6 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/filter_on_nested.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalFilter(condition=[>($1, 29)]) + LogicalProject(projects.name=[$3], proj_name_len=[CHAR_LENGTH($3)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_deep_nested]]) + physical: | + EnumerableCalc(expr#0=[{inputs}], expr#1=[CHAR_LENGTH($t0)], proj#0..1=[{exprs}]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_deep_nested]], PushDownContext=[[PROJECT->[projects.name], SCRIPT->>(CHAR_LENGTH($0), 29), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"nested":{"query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCIXsKICAib3AiOiB7CiAgICAibmFtZSI6ICI+IiwKICAgICJraW5kIjogIkdSRUFURVJfVEhBTiIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkNIQVJfTEVOR1RIIiwKICAgICAgICAia2luZCI6ICJDSEFSX0xFTkdUSCIsCiAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["projects.name",29]}},"boost":1.0}},"path":"projects","ignore_unmapped":false,"score_mode":"none","boost":1.0}},"_source":{"includes":["projects.name"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_on_nested.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_on_nested.yaml new file mode 100644 index 00000000000..5af4a2aceeb --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_on_nested.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalFilter(condition=[>($1, 29)]) + LogicalProject(projects.name=[$3], proj_name_len=[CHAR_LENGTH($3)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_deep_nested]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..15=[{inputs}], expr#16=[CHAR_LENGTH($t3)], expr#17=[29], expr#18=[>($t16, $t17)], projects.name=[$t3], proj_name_len=[$t16], $condition=[$t18]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_deep_nested]]) diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4508.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4508.yml new file mode 100644 index 00000000000..893cf353237 --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4508.yml @@ -0,0 +1,94 @@ +setup: + - do: + indices.create: + index: test_nested_eval_filter + body: + mappings: + properties: + "id": + type: keyword + "items": + type: nested + properties: + "name": + type: keyword + - do: + bulk: + index: test_nested_eval_filter + refresh: true + body: + - '{"index": {"_id": "1"}}' + - '{"id": "order1", "items": [{"name": "apple"}]}' + - '{"index": {"_id": "2"}}' + - '{"id": "order2", "items": [{"name": "banana"}]}' + - '{"index": {"_id": "3"}}' + - '{"id": "order3", "items": [{"name": "orange"}]}' + +--- +"eval on nested field without filter": + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_nested_eval_filter | eval NameLen=LENGTH(items.name) | fields id, items.name, NameLen + + - match: { total: 3 } + - match: {"schema": [{"name": "id", "type": "string"}, {"name": "items.name", "type": "string"}, {"name": "NameLen", "type": "int"}]} + - match: {"datarows": [["order1", "apple", 5], ["order2", "banana", 6], ["order3", "orange", 6]]} + +--- +"eval on nested field with filter on computed field": + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_nested_eval_filter | eval NameLen=LENGTH(items.name) | fields id, items.name, NameLen | where NameLen > 5 + + - match: { total: 2 } + - match: {"schema": [{"name": "id", "type": "string"}, {"name": "items.name", "type": "string"}, {"name": "NameLen", "type": "int"}]} + - match: {"datarows": [["order2", "banana", 6], ["order3", "orange", 6]]} + +--- +"comparison with regular field - eval and filter works correctly": + - skip: + features: + - headers + - do: + indices.create: + index: test_regular_eval_filter + body: + mappings: + properties: + "id": + type: keyword + "name": + type: keyword + - do: + bulk: + index: test_regular_eval_filter + refresh: true + body: + - '{"index": {"_id": "1"}}' + - '{"id": "order1", "name": "apple"}' + - '{"index": {"_id": "2"}}' + - '{"id": "order2", "name": "banana"}' + - '{"index": {"_id": "3"}}' + - '{"id": "order3", "name": "orange"}' + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_regular_eval_filter | eval NameLen=LENGTH(name) | fields id, name, NameLen | where NameLen > 5 + + - match: { total: 2 } + - match: {"schema": [{"name": "id", "type": "string"}, {"name": "name", "type": "string"}, {"name": "NameLen", "type": "int"}]} + - match: {"datarows": [["order2", "banana", 6], ["order3", "orange", 6]]} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java index 9c7f16c6536..d865f3c995c 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java @@ -84,6 +84,7 @@ import org.opensearch.index.mapper.DateFieldMapper; import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.query.RangeQueryBuilder; import org.opensearch.index.query.ScriptQueryBuilder; import org.opensearch.script.Script; @@ -91,6 +92,7 @@ import org.opensearch.sql.calcite.type.ExprIPType; import org.opensearch.sql.calcite.type.ExprSqlType; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.ExprUDT; +import org.opensearch.sql.calcite.utils.PlanUtils; import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils; import org.opensearch.sql.data.model.ExprIpValue; import org.opensearch.sql.data.model.ExprTimestampValue; @@ -1464,6 +1466,8 @@ public static class ScriptQueryExpression extends QueryExpression { private final Supplier codeGenerator; private String generatedCode; private final ScriptParameterHelper parameterHelper; + private final Map fieldTypes; + private final List referredFields; public ScriptQueryExpression( RexNode rexNode, @@ -1487,6 +1491,12 @@ public ScriptQueryExpression( () -> SerializationWrapper.wrapWithLangType( ScriptEngineType.CALCITE, serializer.serialize(rexNode, parameterHelper)); + this.referredFields = + PlanUtils.getInputRefs(rexNode).stream() + .map(RexInputRef::getIndex) + .map(rowType.getFieldNames()::get) + .toList(); + this.fieldTypes = fieldTypes; } // For filter script, this method will be called after planning phase; @@ -1501,7 +1511,12 @@ private String getOrCreateGeneratedCode() { @Override public QueryBuilder builder() { - return new ScriptQueryBuilder(getScript()); + ScriptQueryBuilder scriptQuery = QueryBuilders.scriptQuery(getScript()); + String nestedPath = findNestedPath(fieldTypes); + if (nestedPath != null) { + return QueryBuilders.nestedQuery(nestedPath, scriptQuery, ScoreMode.None); + } + return scriptQuery; } public Script getScript() { @@ -1527,6 +1542,45 @@ public void updateAnalyzedNodes(RexNode rexNode) { public List getUnAnalyzableNodes() { return List.of(); } + + /** + * Find the nested path for fields referenced in the expression. If multiple nested paths exist, + * returns the top one. + * + * @param fieldTypes Map of field names to their types + * @return The nested path, or null if no nested fields are found + */ + private String findNestedPath(Map fieldTypes) { + if (fieldTypes == null || fieldTypes.isEmpty()) { + return null; + } + + for (String fieldName : referredFields) { + // Check if the field is part of a nested structure + // For a field like "items.name", we need to check if "items" is nested + if (fieldName.contains(".")) { + String[] parts = fieldName.split("\\."); + StringBuilder pathBuilder = new StringBuilder(); + + // Build up the path progressively and check if any parent is nested + for (int i = 0; i < parts.length - 1; i++) { + if (i > 0) { + pathBuilder.append("."); + } + pathBuilder.append(parts[i]); + String currentPath = pathBuilder.toString(); + + // Check if this path exists in fieldTypes and is nested + ExprType pathType = fieldTypes.get(currentPath); + // OpenSearchDataType.Nested is mapped to ExprCoreType.ARRAY + if (pathType == ExprCoreType.ARRAY) { + return currentPath; + } + } + } + } + return null; + } } /** diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java index e50e1d4fc85..c1f904d5028 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java @@ -142,9 +142,7 @@ public AbstractRelNode pushDownFilter(Filter filter) { RelDataType rowType = this.getRowType(); List schema = buildSchema(); Map fieldTypes = - this.osIndex.getAllFieldTypes().entrySet().stream() - .filter(entry -> schema.contains(entry.getKey())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + this.osIndex.getAllFieldTypes(); QueryExpression queryExpression = PredicateAnalyzer.analyzeExpression( filter.getCondition(), schema, fieldTypes, rowType, getCluster()); From f51f1afb2a5c4307be281593ffad5eab79bd3763 Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Wed, 19 Nov 2025 16:20:03 +0800 Subject: [PATCH 02/10] Test filter on both nested and root level fields Signed-off-by: Yuanchun Shen --- .../sql/calcite/remote/CalciteExplainIT.java | 13 ++++++++++++- ...{filter_on_nested.yaml => filter_nested.yaml} | 0 .../calcite/filter_root_and_nested.yaml | 8 ++++++++ ...{filter_on_nested.yaml => filter_nested.yaml} | 0 .../filter_root_and_nested.yaml | 10 ++++++++++ .../resources/rest-api-spec/test/issues/4508.yml | 16 ++++++++++++++++ 6 files changed, 46 insertions(+), 1 deletion(-) rename integ-test/src/test/resources/expectedOutput/calcite/{filter_on_nested.yaml => filter_nested.yaml} (100%) create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/filter_root_and_nested.yaml rename integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/{filter_on_nested.yaml => filter_nested.yaml} (100%) create mode 100644 integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_root_and_nested.yaml diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 81fcf4682ac..f4387539d4e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -2376,11 +2376,22 @@ public void testExplainBWC() throws IOException { @Test public void testFilterOnNestedFields() throws IOException { assertYamlEqualsIgnoreId( - loadExpectedPlan("filter_on_nested.yaml"), + loadExpectedPlan("filter_nested.yaml"), explainQueryYaml( StringUtils.format( "source=%s | eval proj_name_len=length(projects.name) | fields projects.name," + " proj_name_len | where proj_name_len > 29", TEST_INDEX_DEEP_NESTED))); } + + @Test + public void testFilterOnNestedAndRootFields() throws IOException { + assertYamlEqualsIgnoreId( + loadExpectedPlan("filter_root_and_nested.yaml"), + // city.name is not in a nested object + explainQueryYaml( + StringUtils.format( + "source=%s | where city.name = 'Seattle' and length(projects.name) > 29", + TEST_INDEX_DEEP_NESTED))); + } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/filter_on_nested.yaml b/integ-test/src/test/resources/expectedOutput/calcite/filter_nested.yaml similarity index 100% rename from integ-test/src/test/resources/expectedOutput/calcite/filter_on_nested.yaml rename to integ-test/src/test/resources/expectedOutput/calcite/filter_nested.yaml diff --git a/integ-test/src/test/resources/expectedOutput/calcite/filter_root_and_nested.yaml b/integ-test/src/test/resources/expectedOutput/calcite/filter_root_and_nested.yaml new file mode 100644 index 00000000000..b3b9366b808 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/filter_root_and_nested.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(accounts=[$0], projects=[$2], city=[$4], account=[$8]) + LogicalFilter(condition=[AND(=($7, 'Seattle'), >(CHAR_LENGTH($3), 29))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_deep_nested]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_deep_nested]], PushDownContext=[[PROJECT->[accounts, projects, projects.name, city, city.name, account], SCRIPT->AND(=($4, 'Seattle'), >(CHAR_LENGTH($2), 29)), PROJECT->[accounts, projects, city, account], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"term":{"city.name":{"value":"Seattle","boost":1.0}}},{"nested":{"query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCIXsKICAib3AiOiB7CiAgICAibmFtZSI6ICI+IiwKICAgICJraW5kIjogIkdSRUFURVJfVEhBTiIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkNIQVJfTEVOR1RIIiwKICAgICAgICAia2luZCI6ICJDSEFSX0xFTkdUSCIsCiAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["projects.name",29]}},"boost":1.0}},"path":"projects","ignore_unmapped":false,"score_mode":"none","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["accounts","projects","city","account"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_on_nested.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_nested.yaml similarity index 100% rename from integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_on_nested.yaml rename to integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_nested.yaml diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_root_and_nested.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_root_and_nested.yaml new file mode 100644 index 00000000000..d635e8ae8e0 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_root_and_nested.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(accounts=[$0], projects=[$2], city=[$4], account=[$8]) + LogicalFilter(condition=[AND(=($7, 'Seattle'), >(CHAR_LENGTH($3), 29))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_deep_nested]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..15=[{inputs}], expr#16=['Seattle':VARCHAR], expr#17=[=($t7, $t16)], expr#18=[CHAR_LENGTH($t3)], expr#19=[29], expr#20=[>($t18, $t19)], expr#21=[AND($t17, $t20)], accounts=[$t0], projects=[$t2], city=[$t4], account=[$t8], $condition=[$t21]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_deep_nested]]) diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4508.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4508.yml index 893cf353237..690179dcc2e 100644 --- a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4508.yml +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4508.yml @@ -92,3 +92,19 @@ setup: - match: { total: 2 } - match: {"schema": [{"name": "id", "type": "string"}, {"name": "name", "type": "string"}, {"name": "NameLen", "type": "int"}]} - match: {"datarows": [["order2", "banana", 6], ["order3", "orange", 6]]} + +--- +"filter on both nested and root level fields ": + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_nested_eval_filter | eval NameLen=LENGTH(items.name) | where NameLen> 5 and id = 'order2' | fields id, items.name, NameLen + + - match: { total: 1 } + - match: {"schema": [{"name": "id", "type": "string"}, {"name": "items.name", "type": "string"}, {"name": "NameLen", "type": "int"}]} + - match: {"datarows": [["order2", "banana", 6]]} From 18a8431279fb671bbb061ae0be84d1c05b3d74e9 Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Wed, 19 Nov 2025 19:01:01 +0800 Subject: [PATCH 03/10] Support nested queries for simple queries: range, term, terms, etc Signed-off-by: Yuanchun Shen --- .../sql/calcite/remote/CalciteExplainIT.java | 39 +++++- .../calcite/remote/CalciteWhereCommandIT.java | 6 + .../sql/legacy/SQLIntegTestCase.java | 5 + .../opensearch/sql/legacy/TestsConstants.java | 1 + .../src/test/resources/cascaded_nested.json | 6 + ...ested.yaml => filter_computed_nested.yaml} | 0 ...filter_multiple_nested_cascaded_range.yaml | 8 ++ .../calcite/filter_nested_term.yaml | 8 ++ .../calcite/filter_nested_terms.yaml | 8 ++ ...ested.yaml => filter_computed_nested.yaml} | 0 ...filter_multiple_nested_cascaded_range.yaml | 10 ++ .../filter_nested_term.yaml | 10 ++ .../filter_nested_terms.yaml | 10 ++ .../cascaded_nested_index_mapping.json | 42 +++++++ .../opensearch/request/PredicateAnalyzer.java | 113 +++++++++--------- 15 files changed, 207 insertions(+), 59 deletions(-) create mode 100644 integ-test/src/test/resources/cascaded_nested.json rename integ-test/src/test/resources/expectedOutput/calcite/{filter_nested.yaml => filter_computed_nested.yaml} (100%) create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/filter_multiple_nested_cascaded_range.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/filter_nested_term.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/filter_nested_terms.yaml rename integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/{filter_nested.yaml => filter_computed_nested.yaml} (100%) create mode 100644 integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_multiple_nested_cascaded_range.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_nested_term.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_nested_terms.yaml create mode 100644 integ-test/src/test/resources/indexDefinitions/cascaded_nested_index_mapping.json diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index f4387539d4e..82ab82b0a39 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -9,6 +9,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ALIAS; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_CASCADED_NESTED; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DEEP_NESTED; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_LOGS; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_SIMPLE; @@ -51,6 +52,7 @@ public void init() throws Exception { loadIndex(Index.WEBLOG); loadIndex(Index.DATA_TYPE_ALIAS); loadIndex(Index.DEEP_NESTED); + loadIndex(Index.CASCADED_NESTED); } @Override @@ -2374,9 +2376,9 @@ public void testExplainBWC() throws IOException { } @Test - public void testFilterOnNestedFields() throws IOException { + public void testFilterOnComputedNestedFields() throws IOException { assertYamlEqualsIgnoreId( - loadExpectedPlan("filter_nested.yaml"), + loadExpectedPlan("filter_computed_nested.yaml"), explainQueryYaml( StringUtils.format( "source=%s | eval proj_name_len=length(projects.name) | fields projects.name," @@ -2388,10 +2390,41 @@ public void testFilterOnNestedFields() throws IOException { public void testFilterOnNestedAndRootFields() throws IOException { assertYamlEqualsIgnoreId( loadExpectedPlan("filter_root_and_nested.yaml"), - // city.name is not in a nested object + // city is not in a nested object explainQueryYaml( StringUtils.format( "source=%s | where city.name = 'Seattle' and length(projects.name) > 29", TEST_INDEX_DEEP_NESTED))); } + + @Test + public void testFilterOnNestedFields() throws IOException { + assertYamlEqualsIgnoreId( + loadExpectedPlan("filter_nested_term.yaml"), + // address is a nested object + explainQueryYaml( + StringUtils.format( + "source=%s | where address.city = 'New york city'", TEST_INDEX_NESTED_SIMPLE))); + + assertYamlEqualsIgnoreId( + loadExpectedPlan("filter_nested_terms.yaml"), + explainQueryYaml( + StringUtils.format( + "source=%s | where address.city in ('Miami', 'san diego')", + TEST_INDEX_NESTED_SIMPLE))); + } + + @Test + public void testFilterOnMultipleCascadedNestedFields() throws IOException { + // 1. Access two different hierarchies of nested fields, one at author.books.reviews, another at + // author.books + // 2. One is pushed as nested range query, another is pushed as nested filter query. + assertYamlEqualsIgnoreId( + loadExpectedPlan("filter_multiple_nested_cascaded_range.yaml"), + explainQueryYaml( + StringUtils.format( + "source=%s | where author.books.reviews.rating >=4 and author.books.reviews.rating" + + " < 6 and author.books.title = 'The Shining'", + TEST_INDEX_CASCADED_NESTED))); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java index 582ce47000d..3e894203500 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java @@ -5,6 +5,7 @@ package org.opensearch.sql.calcite.remote; +import org.junit.Test; import org.opensearch.sql.ppl.WhereCommandIT; public class CalciteWhereCommandIT extends WhereCommandIT { @@ -12,6 +13,8 @@ public class CalciteWhereCommandIT extends WhereCommandIT { public void init() throws Exception { super.init(); enableCalcite(); + loadIndex(Index.NESTED_SIMPLE); + loadIndex(Index.CASCADED_NESTED); } @Override @@ -19,4 +22,7 @@ protected String getIncompatibleTypeErrMsg() { return "In expression types are incompatible: fields type LONG, values type [INTEGER, INTEGER," + " STRING]"; } + + @Test + public void testWhereOnNestedField() {} } diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index 50ee11b765a..d9b76f757f9 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -704,6 +704,11 @@ public enum Index { "_doc", getDeepNestedIndexMapping(), "src/test/resources/deep_nested_index_data.json"), + CASCADED_NESTED( + TestsConstants.TEST_INDEX_CASCADED_NESTED, + "_doc", + getMappingFile("cascaded_nested_index_mapping.json"), + "src/test/resources/cascaded_nested.json"), TELEMETRY( TestsConstants.TEST_INDEX_TELEMETRY, "_doc", diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java index 76923dbd984..ad8a232bab3 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java @@ -54,6 +54,7 @@ public class TestsConstants { public static final String TEST_INDEX_TIME_DATA = TEST_INDEX + "_time_data"; public static final String TEST_INDEX_DEEP_NESTED = TEST_INDEX + "_deep_nested"; + public static final String TEST_INDEX_CASCADED_NESTED = TEST_INDEX + "_cascaded_nested"; public static final String TEST_INDEX_TELEMETRY = TEST_INDEX + "_telemetry"; public static final String TEST_INDEX_STRINGS = TEST_INDEX + "_strings"; public static final String TEST_INDEX_DATATYPE_NUMERIC = TEST_INDEX + "_datatypes_numeric"; diff --git a/integ-test/src/test/resources/cascaded_nested.json b/integ-test/src/test/resources/cascaded_nested.json new file mode 100644 index 00000000000..2f2436b59e6 --- /dev/null +++ b/integ-test/src/test/resources/cascaded_nested.json @@ -0,0 +1,6 @@ +{"index": {"_id": "1"}} +{"author": {"name": "J.K. Rowling", "books": [{"title": "Harry Potter and the Sorcerer's Stone", "reviews": [{"rating": 5, "comment": "Magical and enchanting!", "review_date": "2023-01-15"}, {"rating": 4, "comment": "Great for kids and adults", "review_date": "2023-06-22"}]}, {"title": "Harry Potter and the Chamber of Secrets", "reviews": [{"rating": 5, "comment": "Even better than the first", "review_date": "2023-02-10"}, {"rating": 4, "comment": "Darker tone emerging", "review_date": "2023-07-18"}]}]}} +{"index": {"_id": "2"}} +{"author": {"name": "George R.R. Martin", "books": [{"title": "A Game of Thrones", "reviews": [{"rating": 4, "comment": "Epic fantasy masterpiece", "review_date": "2022-11-05"}, {"rating": 3, "comment": "Too many characters to track", "review_date": "2023-03-20"}]}, {"title": "A Clash of Kings", "reviews": [{"rating": 2, "comment": "Incredible plot twists", "review_date": "2023-08-14"}]}]}} +{"index": {"_id": "3"}} +{"author": {"name": "Stephen King", "books": [{"title": "The Shining", "reviews": [{"rating": 3, "comment": "Brilliant but terrifying", "review_date": "2022-09-03"}, {"rating": 4, "comment": "Psychological horror at its best", "review_date": "2023-04-12"}, {"rating": 2, "comment": "Too slow in places", "review_date": "2023-10-28"}]}]}} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/filter_nested.yaml b/integ-test/src/test/resources/expectedOutput/calcite/filter_computed_nested.yaml similarity index 100% rename from integ-test/src/test/resources/expectedOutput/calcite/filter_nested.yaml rename to integ-test/src/test/resources/expectedOutput/calcite/filter_computed_nested.yaml diff --git a/integ-test/src/test/resources/expectedOutput/calcite/filter_multiple_nested_cascaded_range.yaml b/integ-test/src/test/resources/expectedOutput/calcite/filter_multiple_nested_cascaded_range.yaml new file mode 100644 index 00000000000..9613c9b6962 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/filter_multiple_nested_cascaded_range.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(author=[$0]) + LogicalFilter(condition=[AND(SEARCH($4, Sarg[[4..6)]), =($6, 'The Shining'))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_cascaded_nested]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_cascaded_nested]], PushDownContext=[[PROJECT->[author, author.books.reviews.rating, author.books.title], FILTER->AND(SEARCH($1, Sarg[[4..6)]), =($2, 'The Shining')), PROJECT->[author], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"range":{"author.books.reviews.rating":{"from":4.0,"to":6.0,"include_lower":true,"include_upper":false,"boost":1.0}}},{"term":{"author.books.title.keyword":{"value":"The Shining","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["author"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/filter_nested_term.yaml b/integ-test/src/test/resources/expectedOutput/calcite/filter_nested_term.yaml new file mode 100644 index 00000000000..5b040b26901 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/filter_nested_term.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(name=[$0], address=[$1], id=[$6], age=[$7]) + LogicalFilter(condition=[=($2, 'New york city')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_nested_simple]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_nested_simple]], PushDownContext=[[PROJECT->[name, address, address.city, id, age], FILTER->=($2, 'New york city'), PROJECT->[name, address, id, age], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"nested":{"query":{"term":{"address.city.keyword":{"value":"New york city","boost":1.0}}},"path":"address","ignore_unmapped":false,"score_mode":"none","boost":1.0}},"_source":{"includes":["name","address","id","age"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/filter_nested_terms.yaml b/integ-test/src/test/resources/expectedOutput/calcite/filter_nested_terms.yaml new file mode 100644 index 00000000000..09590ff9945 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/filter_nested_terms.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(name=[$0], address=[$1], id=[$6], age=[$7]) + LogicalFilter(condition=[SEARCH($2, Sarg['Miami':VARCHAR, 'san diego':VARCHAR]:VARCHAR)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_nested_simple]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_nested_simple]], PushDownContext=[[PROJECT->[name, address, address.city, id, age], FILTER->SEARCH($2, Sarg['Miami':VARCHAR, 'san diego':VARCHAR]:VARCHAR), PROJECT->[name, address, id, age], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"nested":{"query":{"terms":{"address.city.keyword":["Miami","san diego"],"boost":1.0}},"path":"address","ignore_unmapped":false,"score_mode":"none","boost":1.0}},"_source":{"includes":["name","address","id","age"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_nested.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_computed_nested.yaml similarity index 100% rename from integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_nested.yaml rename to integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_computed_nested.yaml diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_multiple_nested_cascaded_range.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_multiple_nested_cascaded_range.yaml new file mode 100644 index 00000000000..84926b80826 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_multiple_nested_cascaded_range.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(author=[$0]) + LogicalFilter(condition=[AND(SEARCH($4, Sarg[[4..6)]), =($6, 'The Shining'))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_cascaded_nested]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..13=[{inputs}], expr#14=[Sarg[[4..6)]], expr#15=[SEARCH($t4, $t14)], expr#16=['The Shining':VARCHAR], expr#17=[=($t6, $t16)], expr#18=[AND($t15, $t17)], author=[$t0], $condition=[$t18]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_cascaded_nested]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_nested_term.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_nested_term.yaml new file mode 100644 index 00000000000..0e43ea85d83 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_nested_term.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(name=[$0], address=[$1], id=[$6], age=[$7]) + LogicalFilter(condition=[=($2, 'New york city')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_nested_simple]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..13=[{inputs}], expr#14=['New york city':VARCHAR], expr#15=[=($t2, $t14)], proj#0..1=[{exprs}], id=[$t6], age=[$t7], $condition=[$t15]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_nested_simple]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_nested_terms.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_nested_terms.yaml new file mode 100644 index 00000000000..1db1bc3bc80 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_nested_terms.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(name=[$0], address=[$1], id=[$6], age=[$7]) + LogicalFilter(condition=[SEARCH($2, Sarg['Miami':VARCHAR, 'san diego':VARCHAR]:VARCHAR)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_nested_simple]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..13=[{inputs}], expr#14=[Sarg['Miami':VARCHAR, 'san diego':VARCHAR]:VARCHAR], expr#15=[SEARCH($t2, $t14)], proj#0..1=[{exprs}], id=[$t6], age=[$t7], $condition=[$t15]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_nested_simple]]) diff --git a/integ-test/src/test/resources/indexDefinitions/cascaded_nested_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/cascaded_nested_index_mapping.json new file mode 100644 index 00000000000..301a26fe662 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/cascaded_nested_index_mapping.json @@ -0,0 +1,42 @@ +{ + "mappings": { + "properties": { + "author": { + "type": "nested", + "properties": { + "name": { + "type": "keyword" + }, + "books": { + "type": "nested", + "properties": { + "title": { + "type": "keyword" + }, + "reviews": { + "type": "nested", + "properties": { + "rating": { + "type": "integer" + }, + "comment": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "review_date": { + "type": "date", + "format": "yyyy-MM-dd" + } + } + } + } + } + } + } + } + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java index d865f3c995c..f720d58b254 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java @@ -45,6 +45,7 @@ import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.SINGLE_FIELD_RELEVANCE_FUNCTION_SET; import static org.opensearch.sql.opensearch.storage.script.CompoundedScriptEngine.COMPOUNDED_LANG_NAME; +import com.google.common.base.Strings; import com.google.common.collect.BoundType; import com.google.common.collect.Range; import java.math.BigDecimal; @@ -57,9 +58,12 @@ import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.function.Predicate; import java.util.function.Supplier; import javax.annotation.Nullable; +import java.util.stream.Collectors; import lombok.Getter; +import lombok.RequiredArgsConstructor; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.type.RelDataType; @@ -1207,7 +1211,7 @@ public QueryBuilder builder() { if (builder == null) { throw new IllegalStateException("Builder was not initialized"); } - if (rel != null && rel.nestedPath != null) { + if (rel != null && !Strings.isNullOrEmpty(rel.nestedPath)) { return nestedQuery(rel.nestedPath, builder, ScoreMode.None); } return builder; @@ -1512,9 +1516,22 @@ private String getOrCreateGeneratedCode() { @Override public QueryBuilder builder() { ScriptQueryBuilder scriptQuery = QueryBuilders.scriptQuery(getScript()); - String nestedPath = findNestedPath(fieldTypes); - if (nestedPath != null) { - return QueryBuilders.nestedQuery(nestedPath, scriptQuery, ScoreMode.None); + List nestedPaths = + referredFields.stream() + .map(p -> resolveNestedPath(p, fieldTypes)) + .filter(Predicate.not(Strings::isNullOrEmpty)) + .distinct() + .collect(Collectors.toUnmodifiableList()); + if (nestedPaths.size() > 1) { + throw new UnsupportedOperationException( + String.format( + Locale.ROOT, + "Accessing multiple nested fields under different hierarchies is not supported:" + + " %s", + nestedPaths)); + } + if (!nestedPaths.isEmpty()) { + return nestedQuery(nestedPaths.get(0), scriptQuery, ScoreMode.None); } return scriptQuery; } @@ -1542,45 +1559,6 @@ public void updateAnalyzedNodes(RexNode rexNode) { public List getUnAnalyzableNodes() { return List.of(); } - - /** - * Find the nested path for fields referenced in the expression. If multiple nested paths exist, - * returns the top one. - * - * @param fieldTypes Map of field names to their types - * @return The nested path, or null if no nested fields are found - */ - private String findNestedPath(Map fieldTypes) { - if (fieldTypes == null || fieldTypes.isEmpty()) { - return null; - } - - for (String fieldName : referredFields) { - // Check if the field is part of a nested structure - // For a field like "items.name", we need to check if "items" is nested - if (fieldName.contains(".")) { - String[] parts = fieldName.split("\\."); - StringBuilder pathBuilder = new StringBuilder(); - - // Build up the path progressively and check if any parent is nested - for (int i = 0; i < parts.length - 1; i++) { - if (i > 0) { - pathBuilder.append("."); - } - pathBuilder.append(parts[i]); - String currentPath = pathBuilder.toString(); - - // Check if this path exists in fieldTypes and is nested - ExprType pathType = fieldTypes.get(currentPath); - // OpenSearchDataType.Nested is mapped to ExprCoreType.ARRAY - if (pathType == ExprCoreType.ARRAY) { - return currentPath; - } - } - } - } - return null; - } } /** @@ -1631,23 +1609,22 @@ static boolean isCastExpression(Expression exp) { } /** Used for bind variables. */ + @RequiredArgsConstructor public static final class NamedFieldExpression implements TerminalExpression { private final String name; private final ExprType type; - private final String nestedPath; + @Getter private final String nestedPath; public NamedFieldExpression( int refIndex, List schema, Map filedTypes) { this.name = refIndex >= schema.size() ? null : schema.get(refIndex); this.type = filedTypes.get(name); - this.nestedPath = Utils.resolveNestedPath(this.name, filedTypes); + this.nestedPath = resolveNestedPath(name, filedTypes); } private NamedFieldExpression() { - this.name = null; - this.type = null; - this.nestedPath = null; + this(null, null, ""); } private NamedFieldExpression( @@ -1655,17 +1632,11 @@ private NamedFieldExpression( this.name = (ref == null || ref.getIndex() >= schema.size()) ? null : schema.get(ref.getIndex()); this.type = filedTypes.get(name); - this.nestedPath = Utils.resolveNestedPath(this.name, filedTypes); + this.nestedPath = resolveNestedPath(name, filedTypes); } private NamedFieldExpression(RexLiteral literal) { - this.name = literal == null ? null : RexLiteral.stringValue(literal); - this.type = null; - this.nestedPath = null; - } - - public @Nullable String getNestedPath() { - return nestedPath; + this(literal == null ? null : RexLiteral.stringValue(literal), null, ""); } public String getRootName() { @@ -1872,4 +1843,34 @@ private static void checkForNestedFieldOperands(RexCall call) throws PredicateAn call.getKind())); } } + + /** + * Find the nested path for fields referenced in the expression. If multiple nested paths exist, + * returns the deepest one. + * + * @param name Field name to resolve + * @param fieldTypes Map of field names to their types. It HAS TO contain parent-level mappings + * @return The nested path, or empty string if no nested fields are found + */ + private static String resolveNestedPath(String name, Map fieldTypes) { + if (fieldTypes == null || fieldTypes.isEmpty()) { + return ""; + } + // Check if the field is part of a nested structure + // For a field like "a.b.c.d", we check "a.b.c" first, then "a.b", then "a" + if (name.contains(".")) { + String[] parts = name.split("\\."); + // Start from the deepest parent path and work backwards + // For "a.b.c.d", check "a.b.c" first, then "a.b", then "a" + for (int depth = parts.length - 1; depth > 0; depth--) { + String currentPath = String.join(".", java.util.Arrays.copyOfRange(parts, 0, depth)); + ExprType pathType = fieldTypes.get(currentPath); + // OpenSearchDataType.Nested is mapped to ExprCoreType.ARRAY + if (pathType == ExprCoreType.ARRAY) { + return currentPath; + } + } + } + return ""; + } } From 8bf28be9cf32e5c1c530d23f296e4e324429f1a3 Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Wed, 19 Nov 2025 19:35:30 +0800 Subject: [PATCH 04/10] Add integration tests for filtering on nested Signed-off-by: Yuanchun Shen --- .../calcite/remote/CalciteWhereCommandIT.java | 96 ++++++++++++++++++- ...filter_multiple_nested_cascaded_range.yaml | 2 +- 2 files changed, 96 insertions(+), 2 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java index 3e894203500..a2ae2e81cf9 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java @@ -5,6 +5,18 @@ package org.opensearch.sql.calcite.remote; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_CASCADED_NESTED; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DEEP_NESTED; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_SIMPLE; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import org.json.JSONObject; import org.junit.Test; import org.opensearch.sql.ppl.WhereCommandIT; @@ -14,6 +26,7 @@ public void init() throws Exception { super.init(); enableCalcite(); loadIndex(Index.NESTED_SIMPLE); + loadIndex(Index.DEEP_NESTED); loadIndex(Index.CASCADED_NESTED); } @@ -24,5 +37,86 @@ protected String getIncompatibleTypeErrMsg() { } @Test - public void testWhereOnNestedField() {} + public void testFilterOnComputedNestedFields() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | eval proj_name_len=length(projects.name) | fields projects.name," + + " proj_name_len | where proj_name_len > 29", + TEST_INDEX_DEEP_NESTED)); + verifySchema(result, schema("projects.name", "string"), schema("proj_name_len", "int")); + verifyDataRows(result, rows("AWS Redshift Spectrum querying", 30)); + } + + @Test + public void testFilterOnNestedAndRootFields() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | where city.name = 'Seattle' and length(projects.name) > 29 | fields" + + " city.name, projects.name", + TEST_INDEX_DEEP_NESTED)); + verifySchema(result, schema("city.name", "string"), schema("projects.name", "string")); + verifyDataRows(result, rows("Seattle", "AWS Redshift Spectrum querying")); + } + + @Test + public void testFilterOnNestedFields() throws IOException { + // address is a nested object + JSONObject result1 = + executeQuery( + String.format( + "source=%s | where address.city = 'New york city' | fields address.city", + TEST_INDEX_NESTED_SIMPLE)); + verifySchema(result1, schema("address.city", "string")); + verifyDataRows(result1, rows("New york city")); + + JSONObject result2 = + executeQuery( + String.format( + "source=%s | where address.city in ('Miami', 'san diego') | fields address.city", + TEST_INDEX_NESTED_SIMPLE)); + verifyDataRows(result2, rows("Miami"), rows("san diego")); + } + + @Test + public void testFilterOnMultipleCascadedNestedFields() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | where author.books.reviews.rating >=4 and author.books.reviews.rating" + + " < 6 and author.books.title = 'The Shining' | fields author.books", + TEST_INDEX_CASCADED_NESTED)); + verifySchema(result, schema("author.books", "array")); + verifyDataRows( + result, + rows( + List.of( + Map.of( + "title", + "The Shining", + "reviews", + List.of( + Map.of( + "review_date", + "2022-09-03", + "rating", + 3, + "comment", + "Brilliant but terrifying"), + Map.of( + "review_date", + "2023-04-12", + "rating", + 4, + "comment", + "Psychological horror at its best"), + Map.of( + "review_date", + "2023-10-28", + "rating", + 2, + "comment", + "Too slow in places")))))); + } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/filter_multiple_nested_cascaded_range.yaml b/integ-test/src/test/resources/expectedOutput/calcite/filter_multiple_nested_cascaded_range.yaml index 9613c9b6962..ef0cf5d9813 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/filter_multiple_nested_cascaded_range.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/filter_multiple_nested_cascaded_range.yaml @@ -5,4 +5,4 @@ calcite: LogicalFilter(condition=[AND(SEARCH($4, Sarg[[4..6)]), =($6, 'The Shining'))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_cascaded_nested]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_cascaded_nested]], PushDownContext=[[PROJECT->[author, author.books.reviews.rating, author.books.title], FILTER->AND(SEARCH($1, Sarg[[4..6)]), =($2, 'The Shining')), PROJECT->[author], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"range":{"author.books.reviews.rating":{"from":4.0,"to":6.0,"include_lower":true,"include_upper":false,"boost":1.0}}},{"term":{"author.books.title.keyword":{"value":"The Shining","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["author"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_cascaded_nested]], PushDownContext=[[PROJECT->[author, author.books.reviews.rating, author.books.title], FILTER->AND(SEARCH($1, Sarg[[4..6)]), =($2, 'The Shining')), PROJECT->[author], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"nested":{"query":{"range":{"author.books.reviews.rating":{"from":4.0,"to":6.0,"include_lower":true,"include_upper":false,"boost":1.0}}},"path":"author.books.reviews","ignore_unmapped":false,"score_mode":"none","boost":1.0}},{"nested":{"query":{"term":{"author.books.title":{"value":"The Shining","boost":1.0}}},"path":"author.books","ignore_unmapped":false,"score_mode":"none","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["author"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) From 0327b3fe9aa1fa7433acac2acff4019bcda6431c Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Wed, 19 Nov 2025 20:12:38 +0800 Subject: [PATCH 05/10] Add a error case for accessing different levels of nested objects in scripts Signed-off-by: Yuanchun Shen --- .../calcite/remote/CalciteWhereCommandIT.java | 21 +++++++++++++++++++ .../opensearch/request/PredicateAnalyzer.java | 6 +++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java index a2ae2e81cf9..593690c1e2d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java @@ -11,6 +11,7 @@ import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifyErrorMessageContains; import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; @@ -81,6 +82,8 @@ public void testFilterOnNestedFields() throws IOException { @Test public void testFilterOnMultipleCascadedNestedFields() throws IOException { + // SQL's static type system does not allow returning list[int] in place of int + enabledOnlyWhenPushdownIsEnabled(); JSONObject result = executeQuery( String.format( @@ -119,4 +122,22 @@ public void testFilterOnMultipleCascadedNestedFields() throws IOException { "comment", "Too slow in places")))))); } + + @Test + public void testScriptFilterOnDifferentNestedHierarchyShouldThrow() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + Throwable t = + assertThrows( + Exception.class, + () -> + executeQuery( + String.format( + "source=%s | where author.books.reviews.rating + length(author.books.title)" + + " > 10", + TEST_INDEX_CASCADED_NESTED))); + verifyErrorMessageContains( + t, + "Accessing multiple nested fields under different hierarchies in script is not supported:" + + " [author.books.reviews, author.books]"); + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java index f720d58b254..c79b7c715f7 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java @@ -1523,11 +1523,11 @@ public QueryBuilder builder() { .distinct() .collect(Collectors.toUnmodifiableList()); if (nestedPaths.size() > 1) { - throw new UnsupportedOperationException( + throw new UnsupportedScriptException( String.format( Locale.ROOT, - "Accessing multiple nested fields under different hierarchies is not supported:" - + " %s", + "Accessing multiple nested fields under different hierarchies in script is not" + + " supported: %s", nestedPaths)); } if (!nestedPaths.isEmpty()) { From e5145d1b3da95b146e501b31bbb3b28e547289b3 Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Mon, 24 Nov 2025 18:15:23 +0800 Subject: [PATCH 06/10] Add a test for accessing nested filter in filter in aggregation Signed-off-by: Yuanchun Shen --- .../sql/calcite/remote/CalciteExplainIT.java | 10 ++++++++++ .../sql/calcite/remote/CalciteWhereCommandIT.java | 12 ++++++++++++ .../expectedOutput/calcite/agg_filter_nested.yaml | 9 +++++++++ .../calcite_no_pushdown/agg_filter_nested.yaml | 11 +++++++++++ 4 files changed, 42 insertions(+) create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/agg_filter_nested.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_filter_nested.yaml diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 82ab82b0a39..c3622b22a44 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -2427,4 +2427,14 @@ public void testFilterOnMultipleCascadedNestedFields() throws IOException { + " < 6 and author.books.title = 'The Shining'", TEST_INDEX_CASCADED_NESTED))); } + + @Test + public void testAggFilterOnNestedFields() throws IOException { + assertYamlEqualsIgnoreId( + loadExpectedPlan("agg_filter_nested.yaml"), + explainQueryYaml( + StringUtils.format( + "source=%s | stats count(eval(author.name < 'K')) as george_and_jk", + TEST_INDEX_CASCADED_NESTED))); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java index 593690c1e2d..42524b49c7e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java @@ -19,6 +19,7 @@ import java.util.Map; import org.json.JSONObject; import org.junit.Test; +import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.ppl.WhereCommandIT; public class CalciteWhereCommandIT extends WhereCommandIT { @@ -140,4 +141,15 @@ public void testScriptFilterOnDifferentNestedHierarchyShouldThrow() throws IOExc "Accessing multiple nested fields under different hierarchies in script is not supported:" + " [author.books.reviews, author.books]"); } + + @Test + public void testAggFilterOnNestedFields() throws IOException { + JSONObject result = + executeQuery( + StringUtils.format( + "source=%s | stats count(eval(author.name < 'K')) as george_and_jk", + TEST_INDEX_CASCADED_NESTED)); + verifySchema(result, schema("george_and_jk", "bigint")); + verifyDataRows(result, rows(2)); + } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_filter_nested.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_filter_nested.yaml new file mode 100644 index 00000000000..c566e7e18f4 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_filter_nested.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], george_and_jk=[COUNT($0)]) + LogicalProject($f1=[CASE(<($7, 'K'), 1, null:NULL)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_cascaded_nested]]) + physical: | + EnumerableLimit(fetch=[10000]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_cascaded_nested]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},george_and_jk=COUNT() FILTER $0)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"george_and_jk":{"filter":{"nested":{"query":{"range":{"author.name":{"from":null,"to":"K","include_lower":true,"include_upper":false,"boost":1.0}}},"path":"author","ignore_unmapped":false,"score_mode":"none","boost":1.0}},"aggregations":{"george_and_jk":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_filter_nested.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_filter_nested.yaml new file mode 100644 index 00000000000..3c0a1012db7 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_filter_nested.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], george_and_jk=[COUNT($0)]) + LogicalProject($f1=[CASE(<($7, 'K'), 1, null:NULL)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_cascaded_nested]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableAggregate(group=[{}], george_and_jk=[COUNT() FILTER $0]) + EnumerableCalc(expr#0..13=[{inputs}], expr#14=['K'], expr#15=[<($t7, $t14)], expr#16=[IS TRUE($t15)], $f1=[$t16]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_cascaded_nested]]) From 6dfeda9d642be526b99a4b75e096f46c01eb2abc Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Wed, 19 Nov 2025 20:25:33 +0800 Subject: [PATCH 07/10] Chores: remove unnecessary comments Signed-off-by: Yuanchun Shen --- .../opensearch/sql/opensearch/request/PredicateAnalyzer.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java index c79b7c715f7..1742ea45ce9 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java @@ -1856,10 +1856,9 @@ private static String resolveNestedPath(String name, Map field if (fieldTypes == null || fieldTypes.isEmpty()) { return ""; } - // Check if the field is part of a nested structure - // For a field like "a.b.c.d", we check "a.b.c" first, then "a.b", then "a" if (name.contains(".")) { String[] parts = name.split("\\."); + // Check if the field is part of a nested structure // Start from the deepest parent path and work backwards // For "a.b.c.d", check "a.b.c" first, then "a.b", then "a" for (int depth = parts.length - 1; depth > 0; depth--) { From 41567e23b5b4e8e1df570417c983047be898d989 Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Wed, 14 Jan 2026 17:15:03 +0800 Subject: [PATCH 08/10] Update plans Signed-off-by: Yuanchun Shen --- .../sql/calcite/remote/CalciteExplainIT.java | 2 +- .../calcite/remote/CalciteWhereCommandIT.java | 1 + ...plain_dedup_expr4_alternative_mutated.yaml | 12 +++++++ .../explain_dedup_with_expr4_mutated.yaml | 12 +++++++ .../calcite/filter_computed_nested.yaml | 2 +- .../calcite/filter_nested_term.yaml | 4 +-- .../calcite/filter_nested_terms.yaml | 4 +-- .../calcite/filter_root_and_nested.yaml | 2 +- .../agg_filter_nested.yaml | 11 ------ .../filter_nested_term.yaml | 6 ++-- .../filter_nested_terms.yaml | 6 ++-- .../opensearch/request/PredicateAnalyzer.java | 36 ++----------------- .../storage/scan/CalciteLogicalIndexScan.java | 3 +- 13 files changed, 42 insertions(+), 59 deletions(-) create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr4_alternative_mutated.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr4_mutated.yaml delete mode 100644 integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_filter_nested.yaml diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index c3622b22a44..4a91269ed41 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -29,7 +29,6 @@ import org.junit.Test; import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.common.setting.Settings.Key; -import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.ppl.ExplainIT; import org.opensearch.sql.protocol.response.format.Format; @@ -2430,6 +2429,7 @@ public void testFilterOnMultipleCascadedNestedFields() throws IOException { @Test public void testAggFilterOnNestedFields() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); assertYamlEqualsIgnoreId( loadExpectedPlan("agg_filter_nested.yaml"), explainQueryYaml( diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java index 42524b49c7e..93a8b1eaec5 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java @@ -144,6 +144,7 @@ public void testScriptFilterOnDifferentNestedHierarchyShouldThrow() throws IOExc @Test public void testAggFilterOnNestedFields() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); JSONObject result = executeQuery( StringUtils.format( diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr4_alternative_mutated.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr4_alternative_mutated.yaml new file mode 100644 index 00000000000..ea9c46e976c --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr4_alternative_mutated.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$1], sort1=[$3], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5]) + LogicalFilter(condition=[<=($6, 2)]) + LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $4, $5)]) + LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($5))]) + LogicalSort(sort0=[$1], sort1=[$3], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last]) + LogicalProject(account_number=[$0], gender=[$4], age=[$8], state=[$7], new_gender=[LOWER($4)], new_state=[LOWER($7)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0, 1},agg#0=LITERAL_AGG(2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"new_gender":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"missing_bucket":false,"order":"asc"}}},{"new_state":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f2":{"top_hits":{"from":0,"size":2,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["account_number","gender","age","state"],"excludes":[]},"script_fields":{"new_gender":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"ignore_failure":false},"new_state":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"ignore_failure":false}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr4_mutated.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr4_mutated.yaml new file mode 100644 index 00000000000..5bcfb7c6b46 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr4_mutated.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$1], sort1=[$3], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5]) + LogicalFilter(condition=[<=($6, 2)]) + LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $1, $3)]) + LogicalFilter(condition=[AND(IS NOT NULL($1), IS NOT NULL($3))]) + LogicalSort(sort0=[$1], sort1=[$3], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last]) + LogicalProject(account_number=[$0], gender=[$4], age=[$8], state=[$7], new_gender=[LOWER($4)], new_state=[LOWER($7)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0, 1},agg#0=LITERAL_AGG(2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":false,"order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f2":{"top_hits":{"from":0,"size":2,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["gender","state","account_number","age"],"excludes":[]},"script_fields":{"new_state":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"ignore_failure":false},"new_gender":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"ignore_failure":false}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/filter_computed_nested.yaml b/integ-test/src/test/resources/expectedOutput/calcite/filter_computed_nested.yaml index e646df565b6..ca9d3fdd9fd 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/filter_computed_nested.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/filter_computed_nested.yaml @@ -6,4 +6,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_deep_nested]]) physical: | EnumerableCalc(expr#0=[{inputs}], expr#1=[CHAR_LENGTH($t0)], proj#0..1=[{exprs}]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_deep_nested]], PushDownContext=[[PROJECT->[projects.name], SCRIPT->>(CHAR_LENGTH($0), 29), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"nested":{"query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCIXsKICAib3AiOiB7CiAgICAibmFtZSI6ICI+IiwKICAgICJraW5kIjogIkdSRUFURVJfVEhBTiIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkNIQVJfTEVOR1RIIiwKICAgICAgICAia2luZCI6ICJDSEFSX0xFTkdUSCIsCiAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["projects.name",29]}},"boost":1.0}},"path":"projects","ignore_unmapped":false,"score_mode":"none","boost":1.0}},"_source":{"includes":["projects.name"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_deep_nested]], PushDownContext=[[PROJECT->[projects.name], SCRIPT->>(CHAR_LENGTH($0), 29), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"nested":{"query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCHHsKICAib3AiOiB7CiAgICAibmFtZSI6ICI8IiwKICAgICJraW5kIjogIkxFU1NfVEhBTiIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiQ0hBUl9MRU5HVEgiLAogICAgICAgICJraW5kIjogIkNIQVJfTEVOR1RIIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[2,0],"DIGESTS":[29,"projects.name"]}},"boost":1.0}},"path":"projects","ignore_unmapped":false,"score_mode":"none","boost":1.0}},"_source":{"includes":["projects.name"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/filter_nested_term.yaml b/integ-test/src/test/resources/expectedOutput/calcite/filter_nested_term.yaml index 5b040b26901..68c8800a968 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/filter_nested_term.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/filter_nested_term.yaml @@ -1,8 +1,8 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(name=[$0], address=[$1], id=[$6], age=[$7]) - LogicalFilter(condition=[=($2, 'New york city')]) + LogicalProject(name=[$0], address=[$1], id=[$7], age=[$8]) + LogicalFilter(condition=[=($3, 'New york city')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_nested_simple]]) physical: | CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_nested_simple]], PushDownContext=[[PROJECT->[name, address, address.city, id, age], FILTER->=($2, 'New york city'), PROJECT->[name, address, id, age], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"nested":{"query":{"term":{"address.city.keyword":{"value":"New york city","boost":1.0}}},"path":"address","ignore_unmapped":false,"score_mode":"none","boost":1.0}},"_source":{"includes":["name","address","id","age"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/filter_nested_terms.yaml b/integ-test/src/test/resources/expectedOutput/calcite/filter_nested_terms.yaml index 09590ff9945..44169558a1d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/filter_nested_terms.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/filter_nested_terms.yaml @@ -1,8 +1,8 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(name=[$0], address=[$1], id=[$6], age=[$7]) - LogicalFilter(condition=[SEARCH($2, Sarg['Miami':VARCHAR, 'san diego':VARCHAR]:VARCHAR)]) + LogicalProject(name=[$0], address=[$1], id=[$7], age=[$8]) + LogicalFilter(condition=[SEARCH($3, Sarg['Miami':VARCHAR, 'san diego':VARCHAR]:VARCHAR)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_nested_simple]]) physical: | CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_nested_simple]], PushDownContext=[[PROJECT->[name, address, address.city, id, age], FILTER->SEARCH($2, Sarg['Miami':VARCHAR, 'san diego':VARCHAR]:VARCHAR), PROJECT->[name, address, id, age], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"nested":{"query":{"terms":{"address.city.keyword":["Miami","san diego"],"boost":1.0}},"path":"address","ignore_unmapped":false,"score_mode":"none","boost":1.0}},"_source":{"includes":["name","address","id","age"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/filter_root_and_nested.yaml b/integ-test/src/test/resources/expectedOutput/calcite/filter_root_and_nested.yaml index b3b9366b808..06868a06e8f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/filter_root_and_nested.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/filter_root_and_nested.yaml @@ -5,4 +5,4 @@ calcite: LogicalFilter(condition=[AND(=($7, 'Seattle'), >(CHAR_LENGTH($3), 29))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_deep_nested]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_deep_nested]], PushDownContext=[[PROJECT->[accounts, projects, projects.name, city, city.name, account], SCRIPT->AND(=($4, 'Seattle'), >(CHAR_LENGTH($2), 29)), PROJECT->[accounts, projects, city, account], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"term":{"city.name":{"value":"Seattle","boost":1.0}}},{"nested":{"query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCIXsKICAib3AiOiB7CiAgICAibmFtZSI6ICI+IiwKICAgICJraW5kIjogIkdSRUFURVJfVEhBTiIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkNIQVJfTEVOR1RIIiwKICAgICAgICAia2luZCI6ICJDSEFSX0xFTkdUSCIsCiAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["projects.name",29]}},"boost":1.0}},"path":"projects","ignore_unmapped":false,"score_mode":"none","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["accounts","projects","city","account"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_deep_nested]], PushDownContext=[[PROJECT->[accounts, projects, projects.name, city, city.name, account], SCRIPT->AND(=($4, 'Seattle'), >(CHAR_LENGTH($2), 29)), PROJECT->[accounts, projects, city, account], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"term":{"city.name":{"value":"Seattle","boost":1.0}}},{"nested":{"query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCHHsKICAib3AiOiB7CiAgICAibmFtZSI6ICI8IiwKICAgICJraW5kIjogIkxFU1NfVEhBTiIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiQ0hBUl9MRU5HVEgiLAogICAgICAgICJraW5kIjogIkNIQVJfTEVOR1RIIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[2,0],"DIGESTS":[29,"projects.name"]}},"boost":1.0}},"path":"projects","ignore_unmapped":false,"score_mode":"none","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["accounts","projects","city","account"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_filter_nested.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_filter_nested.yaml deleted file mode 100644 index 3c0a1012db7..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_filter_nested.yaml +++ /dev/null @@ -1,11 +0,0 @@ -calcite: - logical: | - LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalAggregate(group=[{}], george_and_jk=[COUNT($0)]) - LogicalProject($f1=[CASE(<($7, 'K'), 1, null:NULL)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_cascaded_nested]]) - physical: | - EnumerableLimit(fetch=[10000]) - EnumerableAggregate(group=[{}], george_and_jk=[COUNT() FILTER $0]) - EnumerableCalc(expr#0..13=[{inputs}], expr#14=['K'], expr#15=[<($t7, $t14)], expr#16=[IS TRUE($t15)], $f1=[$t16]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_cascaded_nested]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_nested_term.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_nested_term.yaml index 0e43ea85d83..f530facd5ef 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_nested_term.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_nested_term.yaml @@ -1,10 +1,10 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(name=[$0], address=[$1], id=[$6], age=[$7]) - LogicalFilter(condition=[=($2, 'New york city')]) + LogicalProject(name=[$0], address=[$1], id=[$7], age=[$8]) + LogicalFilter(condition=[=($3, 'New york city')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_nested_simple]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..13=[{inputs}], expr#14=['New york city':VARCHAR], expr#15=[=($t2, $t14)], proj#0..1=[{exprs}], id=[$t6], age=[$t7], $condition=[$t15]) + EnumerableCalc(expr#0..14=[{inputs}], expr#15=['New york city':VARCHAR], expr#16=[=($t3, $t15)], proj#0..1=[{exprs}], id=[$t7], age=[$t8], $condition=[$t16]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_nested_simple]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_nested_terms.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_nested_terms.yaml index 1db1bc3bc80..29618da1840 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_nested_terms.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_nested_terms.yaml @@ -1,10 +1,10 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(name=[$0], address=[$1], id=[$6], age=[$7]) - LogicalFilter(condition=[SEARCH($2, Sarg['Miami':VARCHAR, 'san diego':VARCHAR]:VARCHAR)]) + LogicalProject(name=[$0], address=[$1], id=[$7], age=[$8]) + LogicalFilter(condition=[SEARCH($3, Sarg['Miami':VARCHAR, 'san diego':VARCHAR]:VARCHAR)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_nested_simple]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..13=[{inputs}], expr#14=[Sarg['Miami':VARCHAR, 'san diego':VARCHAR]:VARCHAR], expr#15=[SEARCH($t2, $t14)], proj#0..1=[{exprs}], id=[$t6], age=[$t7], $condition=[$t15]) + EnumerableCalc(expr#0..14=[{inputs}], expr#15=[Sarg['Miami':VARCHAR, 'san diego':VARCHAR]:VARCHAR], expr#16=[SEARCH($t3, $t15)], proj#0..1=[{exprs}], id=[$t7], age=[$t8], $condition=[$t16]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_nested_simple]]) diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java index 1742ea45ce9..5ccaad6d6eb 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java @@ -60,7 +60,6 @@ import java.util.Set; import java.util.function.Predicate; import java.util.function.Supplier; -import javax.annotation.Nullable; import java.util.stream.Collectors; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -1518,7 +1517,7 @@ public QueryBuilder builder() { ScriptQueryBuilder scriptQuery = QueryBuilders.scriptQuery(getScript()); List nestedPaths = referredFields.stream() - .map(p -> resolveNestedPath(p, fieldTypes)) + .map(p -> Utils.resolveNestedPath(p, fieldTypes)) .filter(Predicate.not(Strings::isNullOrEmpty)) .distinct() .collect(Collectors.toUnmodifiableList()); @@ -1620,7 +1619,7 @@ public NamedFieldExpression( int refIndex, List schema, Map filedTypes) { this.name = refIndex >= schema.size() ? null : schema.get(refIndex); this.type = filedTypes.get(name); - this.nestedPath = resolveNestedPath(name, filedTypes); + this.nestedPath = Utils.resolveNestedPath(name, filedTypes); } private NamedFieldExpression() { @@ -1632,7 +1631,7 @@ private NamedFieldExpression( this.name = (ref == null || ref.getIndex() >= schema.size()) ? null : schema.get(ref.getIndex()); this.type = filedTypes.get(name); - this.nestedPath = resolveNestedPath(name, filedTypes); + this.nestedPath = Utils.resolveNestedPath(name, filedTypes); } private NamedFieldExpression(RexLiteral literal) { @@ -1843,33 +1842,4 @@ private static void checkForNestedFieldOperands(RexCall call) throws PredicateAn call.getKind())); } } - - /** - * Find the nested path for fields referenced in the expression. If multiple nested paths exist, - * returns the deepest one. - * - * @param name Field name to resolve - * @param fieldTypes Map of field names to their types. It HAS TO contain parent-level mappings - * @return The nested path, or empty string if no nested fields are found - */ - private static String resolveNestedPath(String name, Map fieldTypes) { - if (fieldTypes == null || fieldTypes.isEmpty()) { - return ""; - } - if (name.contains(".")) { - String[] parts = name.split("\\."); - // Check if the field is part of a nested structure - // Start from the deepest parent path and work backwards - // For "a.b.c.d", check "a.b.c" first, then "a.b", then "a" - for (int depth = parts.length - 1; depth > 0; depth--) { - String currentPath = String.join(".", java.util.Arrays.copyOfRange(parts, 0, depth)); - ExprType pathType = fieldTypes.get(currentPath); - // OpenSearchDataType.Nested is mapped to ExprCoreType.ARRAY - if (pathType == ExprCoreType.ARRAY) { - return currentPath; - } - } - } - return ""; - } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java index c1f904d5028..dbe8306d4b2 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java @@ -141,8 +141,7 @@ public AbstractRelNode pushDownFilter(Filter filter) { try { RelDataType rowType = this.getRowType(); List schema = buildSchema(); - Map fieldTypes = - this.osIndex.getAllFieldTypes(); + Map fieldTypes = this.osIndex.getAllFieldTypes(); QueryExpression queryExpression = PredicateAnalyzer.analyzeExpression( filter.getCondition(), schema, fieldTypes, rowType, getCluster()); From 993f47e53160eab848dd6a3eb55af5f186bf07af Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Thu, 15 Jan 2026 10:37:45 +0800 Subject: [PATCH 09/10] Remove unused plans Signed-off-by: Yuanchun Shen --- .../explain_dedup_expr4_alternative_mutated.yaml | 12 ------------ .../calcite/explain_dedup_with_expr4_mutated.yaml | 12 ------------ 2 files changed, 24 deletions(-) delete mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr4_alternative_mutated.yaml delete mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr4_mutated.yaml diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr4_alternative_mutated.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr4_alternative_mutated.yaml deleted file mode 100644 index ea9c46e976c..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_expr4_alternative_mutated.yaml +++ /dev/null @@ -1,12 +0,0 @@ -calcite: - logical: | - LogicalSystemLimit(sort0=[$1], sort1=[$3], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5]) - LogicalFilter(condition=[<=($6, 2)]) - LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $4, $5)]) - LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($5))]) - LogicalSort(sort0=[$1], sort1=[$3], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last]) - LogicalProject(account_number=[$0], gender=[$4], age=[$8], state=[$7], new_gender=[LOWER($4)], new_state=[LOWER($7)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0, 1},agg#0=LITERAL_AGG(2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"new_gender":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"missing_bucket":false,"order":"asc"}}},{"new_state":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f2":{"top_hits":{"from":0,"size":2,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["account_number","gender","age","state"],"excludes":[]},"script_fields":{"new_gender":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"ignore_failure":false},"new_state":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"ignore_failure":false}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr4_mutated.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr4_mutated.yaml deleted file mode 100644 index 5bcfb7c6b46..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_with_expr4_mutated.yaml +++ /dev/null @@ -1,12 +0,0 @@ -calcite: - logical: | - LogicalSystemLimit(sort0=[$1], sort1=[$3], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5]) - LogicalFilter(condition=[<=($6, 2)]) - LogicalProject(account_number=[$0], gender=[$1], age=[$2], state=[$3], new_gender=[$4], new_state=[$5], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $1, $3)]) - LogicalFilter(condition=[AND(IS NOT NULL($1), IS NOT NULL($3))]) - LogicalSort(sort0=[$1], sort1=[$3], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last]) - LogicalProject(account_number=[$0], gender=[$4], age=[$8], state=[$7], new_gender=[LOWER($4)], new_state=[LOWER($7)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0, 1},agg#0=LITERAL_AGG(2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":false,"order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f2":{"top_hits":{"from":0,"size":2,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["gender","state","account_number","age"],"excludes":[]},"script_fields":{"new_state":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"ignore_failure":false},"new_gender":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}},"ignore_failure":false}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) From a8826cf011eb8445e884cf51058ee1bbcd8d1fc6 Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Thu, 15 Jan 2026 10:49:57 +0800 Subject: [PATCH 10/10] Use null instead of empty string as a default to nest path attribute Signed-off-by: Yuanchun Shen --- .../sql/opensearch/request/PredicateAnalyzer.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java index 5ccaad6d6eb..355262b2d6a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java @@ -61,6 +61,7 @@ import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; +import javax.annotation.Nullable; import lombok.Getter; import lombok.RequiredArgsConstructor; import org.apache.calcite.plan.RelOptCluster; @@ -1613,7 +1614,7 @@ public static final class NamedFieldExpression implements TerminalExpression { private final String name; private final ExprType type; - @Getter private final String nestedPath; + @Getter @Nullable private final String nestedPath; public NamedFieldExpression( int refIndex, List schema, Map filedTypes) { @@ -1623,7 +1624,7 @@ public NamedFieldExpression( } private NamedFieldExpression() { - this(null, null, ""); + this(null, null, (String) null); } private NamedFieldExpression( @@ -1635,7 +1636,7 @@ private NamedFieldExpression( } private NamedFieldExpression(RexLiteral literal) { - this(literal == null ? null : RexLiteral.stringValue(literal), null, ""); + this(literal == null ? null : RexLiteral.stringValue(literal), null, null); } public String getRootName() {