Merge remote-tracking branch 'ossupstream/main' into boolquery

Author: karenyrx

CHANGELOG.md

@@ -40,6 +40,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Replace centos:8 with almalinux:8 since centos docker images are deprecated ([#19154](https://github.com/opensearch-project/OpenSearch/pull/19154))
 - Add CompletionStage variants to IndicesAdminClient as an alternative to ActionListener ([#19161](https://github.com/opensearch-project/OpenSearch/pull/19161))
 - Remove cap on Java version used by forbidden APIs ([#19163](https://github.com/opensearch-project/OpenSearch/pull/19163))
+- Omit maxScoreCollector for field collapsing when sort by score descending ([#19181](https://github.com/opensearch-project/OpenSearch/pull/19181))
 - Disable pruning for `doc_values` for the wildcard field mapper ([#18568](https://github.com/opensearch-project/OpenSearch/pull/18568))
 - Make all methods in Engine.Result public ([#19276](https://github.com/opensearch-project/OpenSearch/pull/19275))
 - Create and attach interclusterTest and yamlRestTest code coverage reports to gradle check task([#19165](https://github.com/opensearch-project/OpenSearch/pull/19165))
@@ -49,6 +50,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add failureaccess as runtime dependency to transport-grpc module  ([#19339](https://github.com/opensearch-project/OpenSearch/pull/19339))
 - Migrate usages of deprecated `Operations#union` from Lucene ([#19397](https://github.com/opensearch-project/OpenSearch/pull/19397))
 - Delegate primitive write methods with ByteSizeCachingDirectory wrapped IndexOutput ([#19432](https://github.com/opensearch-project/OpenSearch/pull/19432))
+- Bump opensearch-protobufs dependency to 0.18.0 and update transport-grpc module compatibility ([#19447](https://github.com/opensearch-project/OpenSearch/issues/19447))
 
 ### Fixed
 - Fix unnecessary refreshes on update preparation failures ([#15261](https://github.com/opensearch-project/OpenSearch/issues/15261))

DEVELOPER_GUIDE.md

@@ -76,9 +76,9 @@ Fork [opensearch-project/OpenSearch](https://github.com/opensearch-project/OpenS
 
 #### JDK
 
-OpenSearch recommends building with the [Temurin/Adoptium](https://adoptium.net/temurin/releases/) distribution. JDK 11 is the minimum supported, and JDK-24 is the newest supported. You must have a supported JDK installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK installation, e.g. `JAVA_HOME=/usr/lib/jvm/jdk-21`. 
+OpenSearch recommends building with the [Temurin/Adoptium](https://adoptium.net/temurin/releases/) distribution. JDK 11 is the minimum supported, and JDK-24 is the newest supported. You must have a supported JDK installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK installation, e.g. `JAVA_HOME=/usr/lib/jvm/jdk-21`.
 
-Download Java 11 from [here](https://adoptium.net/releases.html?variant=openjdk11). 
+Download Java 11 from [here](https://adoptium.net/releases.html?variant=openjdk11).
 
 
 In addition, certain backward compatibility tests check out and compile the previous major version of OpenSearch, and therefore require installing [JDK 11](https://adoptium.net/temurin/releases/?version=11) and [JDK 17](https://adoptium.net/temurin/releases/?version=17) and setting the `JAVA11_HOME` and `JAVA17_HOME` environment variables. More to that, since 8.10 release, Gradle has deprecated the usage of the any JDKs below JDK-16. For smooth development experience, the recommendation is to install at least [JDK 17](https://adoptium.net/temurin/releases/?version=17) or [JDK 21](https://adoptium.net/temurin/releases/?version=21). If you still want to build with JDK-11 only, please add `-Dorg.gradle.warning.mode=none` when invoking any Gradle build task from command line, for example:
@@ -178,6 +178,23 @@ Run OpenSearch using `gradlew run`.
 ./gradlew run -PinstalledPlugins="['plugin1', 'plugin2']"
 ```
 
+External plugins may also be fetched and installed from maven snapshots:
+
+```bash
+./gradlew run -PinstalledPlugins="['opensearch-job-scheduler', 'opensearch-sql-plugin']"
+```
+
+You can specify a plugin version to pull to test a specific version in the org.opensearch.plugin groupId:
+```bash
+./gradlew run -PinstalledPlugins="['opensearch-job-scheduler:3.3.x.x']"
+```
+
+or install with fully qualified maven coordinates:
+```bash
+./gradlew run -PinstalledPlugins="['com.example:my-cool-plugin:3.3.x.x']"
+```
+
+
 That will build OpenSearch and start it, writing its log above Gradle's status message. We log a lot of stuff on startup, specifically these lines tell you that OpenSearch is ready.
 
 ```

gradle/libs.versions.toml

@@ -22,7 +22,7 @@ kotlin            = "1.7.10"
 antlr4            = "4.13.1"
 guava             = "33.2.1-jre"
 gson              = "2.13.2"
-opensearchprotobufs = "0.13.0"
+opensearchprotobufs = "0.18.0"
 protobuf          = "3.25.8"
 jakarta_annotation = "1.3.5"
 google_http_client = "1.44.1"

gradle/run.gradle

@@ -28,6 +28,7 @@
  * under the License.
  */
 import org.opensearch.gradle.testclusters.RunTask
+import org.opensearch.gradle.VersionProperties
 
 apply plugin: 'opensearch.testclusters'
 
@@ -41,8 +42,45 @@ testClusters {
     if (numNodes > 1) numberOfNodes = numNodes
     if (findProperty("installedPlugins")) {
       installedPlugins = Eval.me(installedPlugins)
+
+      def resolveMavenPlugin = { coords ->
+        // Add default groupId if not fully qualified (less than 2 colons)
+        String[] parts = coords.split(':')
+        if (parts.length == 2 && parts[0].contains('.')) {
+          throw new IllegalArgumentException("version is required if groupdId is specified '${coords}' Use format: groupId:artifactId:version")
+        }
+        String fullCoords = parts.length < 3 ? 'org.opensearch.plugin:' + coords : coords
+        def config = project.configurations.detachedConfiguration(
+          project.dependencies.create(fullCoords)
+        )
+        config.resolutionStrategy.cacheChangingModulesFor 0, 'seconds'
+        plugin(project.layout.file(project.provider { config.singleFile }))
+      }
+
       for (String p : installedPlugins) {
-        plugin('plugins:'.concat(p))
+        // check if its a local plugin first
+        if (project.findProject(':plugins:' + p) != null) {
+          plugin('plugins:' + p)
+        } else {
+          // attempt to fetch it from maven
+          project.repositories.mavenLocal()
+          project.repositories {
+            maven {
+              name = 'OpenSearch Snapshots'
+              url = 'https://central.sonatype.com/repository/maven-snapshots/'
+            }
+          }
+          if (p.contains(':')) {
+            // Maven coordinates with version specified
+            String coords = p.contains('@') ? p : (p + '@zip')
+            resolveMavenPlugin(coords)
+          } else {
+            // Not found locally, try Maven with current OS version + 0
+            String version = VersionProperties.getOpenSearch().replace('-SNAPSHOT', '.0-SNAPSHOT')
+            String coords = p + ':' + version + '@zip'
+            resolveMavenPlugin(coords)
+          }
+        }
         if (p.equals("arrow-flight-rpc")) {
           // Add system properties for Netty configuration
           systemProperty 'io.netty.allocator.numDirectArenas', '1'

modules/transport-grpc/licenses/protobufs-0.13.0.jar.sha1

@@ -0,0 +1 @@
+e8dc93c60df892184d7c2010e1bd4f15a777c292

modules/transport-grpc/licenses/protobufs-0.18.0.jar.sha1

@@ -0,0 +1 @@
+e8dc93c60df892184d7c2010e1bd4f15a777c292

modules/transport-grpc/spi/licenses/protobufs-0.13.0.jar.sha1

@@ -99,9 +99,7 @@ private QueryContainer createMockTermQueryContainer() {
      * Helper method to create a mock range query container
      */
     private QueryContainer createMockRangeQueryContainer() {
-        return QueryContainer.newBuilder()
-            .setRange(org.opensearch.protobufs.RangeQuery.newBuilder().setField("range_field").build())
-            .build();
+        return QueryContainer.newBuilder().setRange(org.opensearch.protobufs.RangeQuery.newBuilder().build()).build();
     }
 
     /**

modules/transport-grpc/spi/licenses/protobufs-0.18.0.jar.sha1

@@ -45,8 +45,8 @@ public static FetchSourceContext parseFromProtoRequest(org.opensearch.protobufs.
         // Set up source context if source parameters are provided
         if (request.hasXSource()) {
             switch (request.getXSource().getSourceConfigParamCase()) {
-                case BOOL_VALUE:
-                    fetchSource = request.getXSource().getBoolValue();
+                case BOOL:
+                    fetchSource = request.getXSource().getBool();
                     break;
                 case STRING_ARRAY:
                     sourceIncludes = request.getXSource().getStringArray().getStringArrayList().toArray(new String[0]);
@@ -84,8 +84,8 @@ public static FetchSourceContext parseFromProtoRequest(org.opensearch.protobufs.
         if (request.hasXSource()) {
             SourceConfigParam source = request.getXSource();
 
-            if (source.hasBoolValue()) {
-                fetchSource = source.getBoolValue();
+            if (source.hasBool()) {
+                fetchSource = source.getBool();
             } else {
                 sourceIncludes = source.getStringArray().getStringArrayList().toArray(new String[0]);
             }