[GRPC][Joining queries] Implement GRPC Nested query, fix InnerHits, and bump to 0.19.0 protobufs

CHANGELOG.md

@@ -38,6 +38,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Search Stats] Add search & star-tree search query failure count metrics ([#19210](https://github.com/opensearch-project/OpenSearch/issues/19210))
 - [Star-tree] Support for multi-terms aggregation ([#18398](https://github.com/opensearch-project/OpenSearch/issues/18398))
 - Add stream search feature flag and auto fallback logic ([#19373](https://github.com/opensearch-project/OpenSearch/pull/19373))
+- Implement GRPC Exists, Regexp, and Wildcard queries ([#19392](https://github.com/opensearch-project/OpenSearch/pull/19392))
+- Implement GRPC GeoBoundingBox, GeoDistance queries ([#19451](https://github.com/opensearch-project/OpenSearch/pull/19451))
+- Implement GRPC Ids, Range, and Terms Set queries ([#19448](https://github.com/opensearch-project/OpenSearch/pull/19448))
+- Implement GRPC Nested query ([#19453](https://github.com/opensearch-project/OpenSearch/pull/19453))
 
 ### Changed
 - Refactor `if-else` chains to use `Java 17 pattern matching switch expressions`(([#18965](https://github.com/opensearch-project/OpenSearch/pull/18965))
@@ -59,6 +63,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Migrate usages of deprecated `Operations#union` from Lucene ([#19397](https://github.com/opensearch-project/OpenSearch/pull/19397))
 - Delegate primitive write methods with ByteSizeCachingDirectory wrapped IndexOutput ([#19432](https://github.com/opensearch-project/OpenSearch/pull/19432))
 - Bump opensearch-protobufs dependency to 0.18.0 and update transport-grpc module compatibility ([#19447](https://github.com/opensearch-project/OpenSearch/issues/19447))
+- Bump opensearch-protobufs dependency to 0.19.0 ([#19453](https://github.com/opensearch-project/OpenSearch/issues/19453))
 
 ### Fixed
 - Fix unnecessary refreshes on update preparation failures ([#15261](https://github.com/opensearch-project/OpenSearch/issues/15261))

gradle/libs.versions.toml

@@ -22,7 +22,7 @@ kotlin            = "1.7.10"
 antlr4            = "4.13.1"
 guava             = "33.2.1-jre"
 gson              = "2.13.2"
-opensearchprotobufs = "0.18.0"
+opensearchprotobufs = "0.19.0"
 protobuf          = "3.25.8"
 jakarta_annotation = "1.3.5"
 google_http_client = "1.44.1"

modules/transport-grpc/licenses/protobufs-0.19.0.jar.sha1

@@ -0,0 +1 @@
+14b425a0cb280ccad20983daeaabf3d1c83c3670

modules/transport-grpc/spi/licenses/protobufs-0.19.0.jar.sha1

@@ -0,0 +1 @@
+14b425a0cb280ccad20983daeaabf3d1c83c3670

modules/transport-grpc/src/main/java/org/opensearch/transport/grpc/proto/request/search/CollapseBuilderProtoUtils.java

@@ -13,6 +13,7 @@
 import org.opensearch.search.collapse.CollapseBuilder;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 
 /**
@@ -45,7 +46,10 @@ protected static CollapseBuilder fromProto(FieldCollapse collapseProto) throws I
             collapseBuilder.setMaxConcurrentGroupRequests(collapseProto.getMaxConcurrentGroupSearches());
         }
         if (collapseProto.getInnerHitsCount() > 0) {
-            List<InnerHitBuilder> innerHitBuilders = InnerHitsBuilderProtoUtils.fromProto(collapseProto.getInnerHitsList());
+            List<InnerHitBuilder> innerHitBuilders = new ArrayList<>();
+            for (org.opensearch.protobufs.InnerHits innerHits : collapseProto.getInnerHitsList()) {
+                innerHitBuilders.add(InnerHitsBuilderProtoUtils.fromProto(innerHits));
+            }
             collapseBuilder.setInnerHits(innerHitBuilders);
         }
 

modules/transport-grpc/src/main/java/org/opensearch/transport/grpc/proto/request/search/FieldAndFormatProtoUtils.java

@@ -30,10 +30,18 @@ private FieldAndFormatProtoUtils() {
      * @param fieldAndFormatProto The Protocol Buffer FieldAndFormat to convert
      * @return A configured FieldAndFormat instance
      */
-    protected static FieldAndFormat fromProto(org.opensearch.protobufs.FieldAndFormat fieldAndFormatProto) {
+    public static FieldAndFormat fromProto(org.opensearch.protobufs.FieldAndFormat fieldAndFormatProto) {
+        if (fieldAndFormatProto == null) {
+            throw new IllegalArgumentException("FieldAndFormat protobuf cannot be null");
+        }
 
-        // TODO how is this field used?
-        // fieldAndFormatProto.getIncludeUnmapped();
-        return new FieldAndFormat(fieldAndFormatProto.getField(), fieldAndFormatProto.getFormat());
+        String fieldName = fieldAndFormatProto.getField();
+        if (fieldName == null || fieldName.trim().isEmpty()) {
+            throw new IllegalArgumentException("Field name cannot be null or empty");
+        }
+        String format = fieldAndFormatProto.hasFormat() ? fieldAndFormatProto.getFormat() : null;
+
+        return new FieldAndFormat(fieldName, format);
     }
+
 }

modules/transport-grpc/src/main/java/org/opensearch/transport/grpc/proto/request/search/InnerHitsBuilderProtoUtils.java

@@ -75,19 +75,18 @@ public static InnerHitBuilder fromProto(InnerHits innerHits) throws IOException
             innerHitBuilder.setStoredFieldNames(innerHits.getStoredFieldsList());
         }
         if (innerHits.getDocvalueFieldsCount() > 0) {
-            List<FieldAndFormat> fieldAndFormatList = new ArrayList<>();
+            List<FieldAndFormat> docvalueFieldsList = new ArrayList<>();
             for (org.opensearch.protobufs.FieldAndFormat fieldAndFormat : innerHits.getDocvalueFieldsList()) {
-                fieldAndFormatList.add(FieldAndFormatProtoUtils.fromProto(fieldAndFormat));
+                docvalueFieldsList.add(FieldAndFormatProtoUtils.fromProto(fieldAndFormat));
             }
-            innerHitBuilder.setDocValueFields(fieldAndFormatList);
+            innerHitBuilder.setDocValueFields(docvalueFieldsList);
         }
         if (innerHits.getFieldsCount() > 0) {
-            List<FieldAndFormat> fieldAndFormatList = new ArrayList<>();
-            // TODO: this is not correct, we need to use FieldAndFormatProtoUtils.fromProto() and fix the protobufs in 0.11.0
-            for (String fieldName : innerHits.getFieldsList()) {
-                fieldAndFormatList.add(new FieldAndFormat(fieldName, null));
+            List<FieldAndFormat> fieldsList = new ArrayList<>();
+            for (org.opensearch.protobufs.FieldAndFormat fieldAndFormat : innerHits.getFieldsList()) {
+                fieldsList.add(FieldAndFormatProtoUtils.fromProto(fieldAndFormat));
             }
-            innerHitBuilder.setFetchFields(fieldAndFormatList);
+            innerHitBuilder.setFetchFields(fieldsList);
         }
         if (innerHits.getScriptFieldsCount() > 0) {
             Set<SearchSourceBuilder.ScriptField> scriptFields = new HashSet<>();
@@ -118,24 +117,4 @@ public static InnerHitBuilder fromProto(InnerHits innerHits) throws IOException
         return innerHitBuilder;
     }
 
-    /**
-     * Converts a list of protobuf InnerHits to a list of OpenSearch InnerHitBuilder objects.
-     * Each InnerHits protobuf message represents ONE inner hit definition.
-     *
-     * @param innerHitsList the list of protobuf InnerHits to convert
-     * @return the list of converted OpenSearch InnerHitBuilder objects
-     * @throws IOException if there's an error during parsing
-     */
-    public static List<InnerHitBuilder> fromProto(List<InnerHits> innerHitsList) throws IOException {
-        if (innerHitsList == null) {
-            throw new IllegalArgumentException("InnerHits list cannot be null");
-        }
-
-        List<InnerHitBuilder> innerHitBuilders = new ArrayList<>();
-        for (InnerHits innerHits : innerHitsList) {
-            innerHitBuilders.add(fromProto(innerHits));
-        }
-        return innerHitBuilders;
-    }
-
 }

modules/transport-grpc/src/main/java/org/opensearch/transport/grpc/proto/request/search/query/NestedQueryBuilderProtoConverter.java

@@ -0,0 +1,47 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.transport.grpc.proto.request.search.query;
+
+import org.opensearch.index.query.QueryBuilder;
+import org.opensearch.protobufs.QueryContainer;
+import org.opensearch.transport.grpc.spi.QueryBuilderProtoConverter;
+import org.opensearch.transport.grpc.spi.QueryBuilderProtoConverterRegistry;
+
+/**
+ * Converter for NestedQuery protobuf messages to OpenSearch QueryBuilder objects.
+ * Handles the conversion of nested query protobuf messages to OpenSearch NestedQueryBuilder.
+ */
+public class NestedQueryBuilderProtoConverter implements QueryBuilderProtoConverter {
+
+    /**
+     * Default constructor for NestedQueryBuilderProtoConverter.
+     */
+    public NestedQueryBuilderProtoConverter() {}
+
+    private QueryBuilderProtoConverterRegistry registry;
+
+    @Override
+    public void setRegistry(QueryBuilderProtoConverterRegistry registry) {
+        this.registry = registry;
+        // The utility class no longer stores the registry statically, it's passed directly to fromProto
+    }
+
+    @Override
+    public QueryContainer.QueryContainerCase getHandledQueryCase() {
+        return QueryContainer.QueryContainerCase.NESTED;
+    }
+
+    @Override
+    public QueryBuilder fromProto(QueryContainer queryContainer) {
+        if (queryContainer == null || queryContainer.getQueryContainerCase() != QueryContainer.QueryContainerCase.NESTED) {
+            throw new IllegalArgumentException("QueryContainer must contain a NestedQuery");
+        }
+        return NestedQueryBuilderProtoUtils.fromProto(queryContainer.getNested(), registry);
+    }
+}

modules/transport-grpc/src/main/java/org/opensearch/transport/grpc/proto/request/search/query/NestedQueryBuilderProtoUtils.java

@@ -0,0 +1,123 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.transport.grpc.proto.request.search.query;
+
+import org.apache.lucene.search.join.ScoreMode;
+import org.opensearch.index.query.AbstractQueryBuilder;
+import org.opensearch.index.query.InnerHitBuilder;
+import org.opensearch.index.query.NestedQueryBuilder;
+import org.opensearch.index.query.QueryBuilder;
+import org.opensearch.protobufs.ChildScoreMode;
+import org.opensearch.protobufs.NestedQuery;
+import org.opensearch.protobufs.QueryContainer;
+import org.opensearch.transport.grpc.proto.request.search.InnerHitsBuilderProtoUtils;
+import org.opensearch.transport.grpc.spi.QueryBuilderProtoConverterRegistry;
+
+/**
+ * Utility class for converting protobuf NestedQuery to OpenSearch NestedQueryBuilder.
+ * Handles the conversion of nested query protobuf messages to OpenSearch NestedQueryBuilder objects.
+ */
+class NestedQueryBuilderProtoUtils {
+
+    private NestedQueryBuilderProtoUtils() {
+        // Utility class, no instances
+    }
+
+    /**
+     * Converts a protobuf NestedQuery to an OpenSearch NestedQueryBuilder.
+     * Similar to {@link NestedQueryBuilder#fromXContent(org.opensearch.core.xcontent.XContentParser)}, this method
+     * parses the Protocol Buffer representation and creates a properly configured
+     * NestedQueryBuilder with the appropriate path, query, score mode, inner hits, boost, and query name.
+     *
+     * @param nestedQueryProto the protobuf NestedQuery to convert
+     * @param registry The registry to use for converting nested queries
+     * @return the converted OpenSearch NestedQueryBuilder
+     * @throws IllegalArgumentException if the protobuf query is invalid
+     */
+    static NestedQueryBuilder fromProto(NestedQuery nestedQueryProto, QueryBuilderProtoConverterRegistry registry) {
+        if (nestedQueryProto == null) {
+            throw new IllegalArgumentException("NestedQuery cannot be null");
+        }
+
+        float boost = AbstractQueryBuilder.DEFAULT_BOOST;
+        ScoreMode scoreMode = ScoreMode.Avg;
+        String queryName = null;
+        QueryBuilder query = null;
+        InnerHitBuilder innerHitBuilder = null;
+        boolean ignoreUnmapped = NestedQueryBuilder.DEFAULT_IGNORE_UNMAPPED;
+
+        String path = nestedQueryProto.getPath();
+        if (path.isEmpty()) {
+            throw new IllegalArgumentException("Path is required for NestedQuery");
+        }
+
+        if (!nestedQueryProto.hasQuery()) {
+            throw new IllegalArgumentException("Query is required for NestedQuery");
+        }
+        try {
+            QueryContainer queryContainer = nestedQueryProto.getQuery();
+            query = registry.fromProto(queryContainer);
+        } catch (Exception e) {
+            throw new IllegalArgumentException("Failed to convert inner query for NestedQuery: " + e.getMessage(), e);
+        }
+
+        if (nestedQueryProto.hasScoreMode()) {
+            scoreMode = parseScoreMode(nestedQueryProto.getScoreMode());
+        }
+
+        if (nestedQueryProto.hasIgnoreUnmapped()) {
+            ignoreUnmapped = nestedQueryProto.getIgnoreUnmapped();
+        }
+
+        if (nestedQueryProto.hasBoost()) {
+            boost = nestedQueryProto.getBoost();
+        }
+
+        if (nestedQueryProto.hasXName()) {
+            queryName = nestedQueryProto.getXName();
+        }
+
+        if (nestedQueryProto.hasInnerHits()) {
+            try {
+                innerHitBuilder = InnerHitsBuilderProtoUtils.fromProto(nestedQueryProto.getInnerHits());
+            } catch (Exception e) {
+                throw new IllegalArgumentException("Failed to convert inner hits for NestedQuery: " + e.getMessage(), e);
+            }
+        }
+
+        NestedQueryBuilder queryBuilder = new NestedQueryBuilder(path, query, scoreMode, innerHitBuilder).ignoreUnmapped(ignoreUnmapped)
+            .queryName(queryName)
+            .boost(boost);
+
+        return queryBuilder;
+    }
+
+    /**
+     * Converts protobuf ChildScoreMode to Lucene ScoreMode.
+     *
+     * @param childScoreMode the protobuf ChildScoreMode
+     * @return the converted Lucene ScoreMode
+     */
+    private static ScoreMode parseScoreMode(ChildScoreMode childScoreMode) {
+        switch (childScoreMode) {
+            case CHILD_SCORE_MODE_AVG:
+                return ScoreMode.Avg;
+            case CHILD_SCORE_MODE_MAX:
+                return ScoreMode.Max;
+            case CHILD_SCORE_MODE_MIN:
+                return ScoreMode.Min;
+            case CHILD_SCORE_MODE_NONE:
+                return ScoreMode.None;
+            case CHILD_SCORE_MODE_SUM:
+                return ScoreMode.Total;
+            default:
+                return ScoreMode.Avg; // Default value
+        }
+    }
+}

modules/transport-grpc/src/main/java/org/opensearch/transport/grpc/proto/request/search/query/QueryBuilderProtoConverterRegistryImpl.java

@@ -57,6 +57,7 @@ protected void registerBuiltInConverters() {
         delegate.registerConverter(new WildcardQueryBuilderProtoConverter());
         delegate.registerConverter(new GeoBoundingBoxQueryBuilderProtoConverter());
         delegate.registerConverter(new GeoDistanceQueryBuilderProtoConverter());
+        delegate.registerConverter(new NestedQueryBuilderProtoConverter());
 
         // Set the registry on all converters so they can access each other
         delegate.setRegistryOnAllConverters(this);

modules/transport-grpc/src/test/java/org/opensearch/transport/grpc/proto/request/search/FieldAndFormatProtoUtilsTests.java

@@ -0,0 +1,79 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.transport.grpc.proto.request.search;
+
+import org.opensearch.search.fetch.subphase.FieldAndFormat;
+import org.opensearch.test.OpenSearchTestCase;
+
+/**
+ * Unit tests for FieldAndFormatProtoUtils.
+ */
+public class FieldAndFormatProtoUtilsTests extends OpenSearchTestCase {
+
+    public void testFromProto_WithFieldOnly() {
+        org.opensearch.protobufs.FieldAndFormat fieldAndFormatProto = org.opensearch.protobufs.FieldAndFormat.newBuilder()
+            .setField("test_field")
+            .build();
+
+        FieldAndFormat result = FieldAndFormatProtoUtils.fromProto(fieldAndFormatProto);
+
+        assertNotNull(result);
+        assertEquals("test_field", result.field);
+        assertNull(result.format);
+    }
+
+    public void testFromProto_WithFieldAndFormat() {
+        org.opensearch.protobufs.FieldAndFormat fieldAndFormatProto = org.opensearch.protobufs.FieldAndFormat.newBuilder()
+            .setField("date_field")
+            .setFormat("yyyy-MM-dd")
+            .build();
+
+        FieldAndFormat result = FieldAndFormatProtoUtils.fromProto(fieldAndFormatProto);
+
+        assertNotNull(result);
+        assertEquals("date_field", result.field);
+        assertEquals("yyyy-MM-dd", result.format);
+    }
+
+    public void testFromProto_WithEmptyFieldName() {
+        org.opensearch.protobufs.FieldAndFormat fieldAndFormatProto = org.opensearch.protobufs.FieldAndFormat.newBuilder()
+            .setField("")
+            .setFormat("yyyy-MM-dd")
+            .build();
+
+        try {
+            FieldAndFormatProtoUtils.fromProto(fieldAndFormatProto);
+            fail("Should have thrown IllegalArgumentException for empty field name");
+        } catch (IllegalArgumentException e) {
+            assertEquals("Field name cannot be null or empty", e.getMessage());
+        }
+    }
+
+    public void testFromProto_WithNullProtobuf() {
+        try {
+            FieldAndFormatProtoUtils.fromProto(null);
+            fail("Should have thrown IllegalArgumentException for null protobuf");
+        } catch (IllegalArgumentException e) {
+            assertEquals("FieldAndFormat protobuf cannot be null", e.getMessage());
+        }
+    }
+
+    public void testFromProto_WithComplexFormat() {
+        org.opensearch.protobufs.FieldAndFormat fieldAndFormatProto = org.opensearch.protobufs.FieldAndFormat.newBuilder()
+            .setField("timestamp")
+            .setFormat("epoch_millis")
+            .build();
+
+        FieldAndFormat result = FieldAndFormatProtoUtils.fromProto(fieldAndFormatProto);
+
+        assertNotNull(result);
+        assertEquals("timestamp", result.field);
+        assertEquals("epoch_millis", result.format);
+    }
+}