[improve][misc] Sync commits from apache into 4.0_ds (Oct 15th)

.github/workflows/ci-documentbot.yml

@@ -36,7 +36,7 @@ jobs:
     if: (github.repository == 'apache/pulsar') && (github.event.pull_request.state == 'open')
     permissions:
       pull-requests: write 
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Labeling
         uses: apache/pulsar-test-infra/docbot@master

.github/workflows/ci-go-functions.yaml

@@ -38,7 +38,7 @@ env:
 jobs:
   preconditions:
     name: Preconditions
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     outputs:
       docs_only: ${{ steps.check_changes.outputs.docs_only }}
     steps:
@@ -73,7 +73,7 @@ jobs:
     needs: preconditions
     if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
     name: Go ${{ matrix.go-version }} Functions style check
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     strategy:
       matrix:
         go-version: ['1.23']

.github/workflows/ci-maven-cache-update.yaml

@@ -59,7 +59,7 @@ jobs:
       matrix:
         include:
           - name: all modules
-            runs-on: ubuntu-22.04
+            runs-on: ubuntu-24.04
             cache_name: 'm2-dependencies-all'
             mvn_arguments: ''
 
@@ -68,7 +68,7 @@ jobs:
             cache_name: 'm2-dependencies-all'
 
           - name: core-modules
-            runs-on: ubuntu-22.04
+            runs-on: ubuntu-24.04
             cache_name: 'm2-dependencies-core-modules'
             mvn_arguments: '-Pcore-modules,-main'
 

.github/workflows/ci-owasp-dependency-check.yaml

@@ -35,7 +35,7 @@ jobs:
     env:
       JOB_NAME: Check ${{ matrix.branch }}
       DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 75
     strategy:
       fail-fast: false

.github/workflows/ci-pulsarbot.yaml

@@ -24,7 +24,7 @@ on:
 
 jobs:
   pulsarbot:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 10
     if: github.event_name == 'issue_comment' && contains(github.event.comment.body, '/pulsarbot')
     steps:

.github/workflows/pulsar-ci-flaky.yaml

@@ -87,7 +87,7 @@ env:
 jobs:
   preconditions:
     name: Preconditions
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     outputs:
       docs_only: ${{ steps.check_changes.outputs.docs_only }}
       changed_tests: ${{ steps.changes.outputs.tests_files }}
@@ -176,7 +176,7 @@ jobs:
       THREAD_LEAK_DETECTOR_DIR: ${{ github.workspace }}/target/thread-leak-dumps
       NETTY_LEAK_DETECTION: "${{ needs.preconditions.outputs.netty_leak_detection }}"
       NETTY_LEAK_DUMP_DIR: ${{ github.workspace }}/target/netty-leak-dumps
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 100
     if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
     steps:

.github/workflows/pulsar-ci.yaml

@@ -87,7 +87,7 @@ env:
 jobs:
   preconditions:
     name: Preconditions
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     outputs:
       docs_only: ${{ steps.check_changes.outputs.docs_only }}
       changed_tests: ${{ steps.changes.outputs.tests_files }}
@@ -169,7 +169,7 @@ jobs:
       JOB_NAME: Build and License check
       DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
       CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 60
     if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
     steps:
@@ -252,7 +252,7 @@ jobs:
       THREAD_LEAK_DETECTOR_DIR: ${{ github.workspace }}/target/thread-leak-dumps
       NETTY_LEAK_DETECTION: "${{ needs.preconditions.outputs.netty_leak_detection }}"
       NETTY_LEAK_DUMP_DIR: ${{ github.workspace }}/target/netty-leak-dumps
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: ${{ matrix.timeout || 60 }}
     needs: ['preconditions', 'build-and-license-check']
     if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
@@ -407,7 +407,7 @@ jobs:
 
   unit-tests-upload-coverage:
     name: CI - Unit - Upload Coverage
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 30
     needs: ['preconditions', 'unit-tests']
     env:
@@ -489,7 +489,7 @@ jobs:
 
   pulsar-java-test-image:
     name: Build Pulsar java-test-image docker image
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 60
     needs: ['preconditions', 'build-and-license-check']
     if: ${{ needs.preconditions.outputs.docs_only != 'true'}}
@@ -571,7 +571,7 @@ jobs:
 
   integration-tests:
     name: CI - Integration - ${{ matrix.name }}
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: ${{ matrix.timeout || 60 }}
     needs: ['preconditions', 'pulsar-java-test-image']
     if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
@@ -774,7 +774,7 @@ jobs:
 
   integration-tests-upload-coverage:
     name: CI - Integration - Upload Coverage
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 30
     needs: ['preconditions', 'integration-tests']
     if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }}
@@ -861,7 +861,7 @@ jobs:
 
   delete-integration-test-docker-image-artifact:
     name: "Delete integration test docker image artifact"
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 10
     needs: [
       'preconditions',
@@ -885,7 +885,7 @@ jobs:
 
   pulsar-test-latest-version-image:
     name: Build Pulsar docker image
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 60
     needs: ['preconditions', 'build-and-license-check']
     if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
@@ -1009,7 +1009,7 @@ jobs:
 
   system-tests:
     name: CI - System - ${{ matrix.name }}
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 60
     needs: ['preconditions', 'pulsar-test-latest-version-image']
     if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
@@ -1171,7 +1171,7 @@ jobs:
 
   system-tests-upload-coverage:
     name: CI - System - Upload Coverage
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 30
     needs: ['preconditions', 'system-tests']
     if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }}
@@ -1259,7 +1259,7 @@ jobs:
 
   flaky-system-tests:
     name: CI Flaky - System - ${{ matrix.name }}
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 60
     needs: [ 'preconditions', 'pulsar-test-latest-version-image' ]
     if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
@@ -1398,7 +1398,7 @@ jobs:
 
   delete-system-test-docker-image-artifact:
     name: "Delete system test docker image artifact"
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 10
     needs: [
       'preconditions',
@@ -1459,7 +1459,7 @@ jobs:
 
   codeql:
     name: Run CodeQL Analysis
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 60
     needs: ['preconditions', 'unit-tests']
     if: ${{ (needs.preconditions.outputs.java_non_tests == 'true' || github.event_name != 'pull_request') && ((github.event_name == 'pull_request' && github.base_ref == 'master') || (github.event_name != 'pull_request' && github.ref_name == 'master')) }}
@@ -1524,7 +1524,7 @@ jobs:
 
   owasp-dep-check:
     name: OWASP dependency check
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 120
     needs: [ 'preconditions', 'integration-tests' ]
     if: ${{ needs.preconditions.outputs.need_owasp == 'true' }}
@@ -1630,7 +1630,7 @@ jobs:
     # this is to allow the workflow scheduled jobs to show as cancelled instead of failed since scheduled
     # jobs are not enabled for other than apache/pulsar repository.
     if: ${{ always() && !(cancelled() && github.repository != 'apache/pulsar' && github.event_name == 'schedule') }}
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 10
     needs: [
       'preconditions',

build/run_unit_group.sh

@@ -174,7 +174,8 @@ function test_group_other() {
            -pl '!com.datastax.oss:distribution,!com.datastax.oss:pulsar-offloader-distribution,!com.datastax.oss:pulsar-server-distribution,!com.datastax.oss:pulsar-io-distribution,!com.datastax.oss:pulsar-all-docker-image,!com.datastax.oss:pulsar-experimental-docker-image' \
            -PskipTestsForUnitGroupOther -DdisableIoMainProfile=true -DskipIntegrationTests \
            -Dexclude='**/ManagedLedgerTest.java,
-                   **/OffloadersCacheTest.java
+                   **/OffloadersCacheTest.java,
+                   **/OffsetsCacheTest.java,
                   **/PrimitiveSchemaTest.java,
                   **/BlobStoreManagedLedgerOffloaderTest.java,
                   **/BlobStoreManagedLedgerOffloaderStreamingTest.java'
@@ -184,6 +185,7 @@ function test_group_other() {
 
   mvn_test -pl tiered-storage/jcloud -Dinclude='**/BlobStoreManagedLedgerOffloaderTest.java'
   mvn_test -pl tiered-storage/jcloud -Dinclude='**/BlobStoreManagedLedgerOffloaderStreamingTest.java'
+  mvn_test -pl tiered-storage/jcloud -Dinclude='**/OffsetsCacheTest.java'
 
   echo "::endgroup::"
   local modules_with_quarantined_tests=$(git grep -l '@Test.*"quarantine"' | grep '/src/test/java/' | \

distribution/server/src/assemble/LICENSE.bin.txt

@@ -263,7 +263,7 @@ The Apache Software License, Version 2.0
  * Caffeine -- com.github.ben-manes.caffeine-caffeine-2.9.1.jar
  * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.5.2.jar
  * Fastutil -- it.unimi.dsi-fastutil-8.5.16.jar
- * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-2.51.0.jar
+ * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-2.59.2.jar
  * Bitbucket -- org.bitbucket.b_c-jose4j-0.9.4.jar
  * Gson
     - com.google.code.gson-gson-2.8.9.jar
@@ -431,26 +431,26 @@ The Apache Software License, Version 2.0
      - org.jetbrains.kotlin-kotlin-stdlib-jdk8-1.8.20.jar
      - org.jetbrains-annotations-13.0.jar
  * gRPC
-    - io.grpc-grpc-all-1.72.0.jar
-    - io.grpc-grpc-auth-1.72.0.jar
-    - io.grpc-grpc-context-1.72.0.jar
-    - io.grpc-grpc-core-1.72.0.jar
-    - io.grpc-grpc-protobuf-1.72.0.jar
-    - io.grpc-grpc-protobuf-lite-1.72.0.jar
-    - io.grpc-grpc-stub-1.72.0.jar
-    - io.grpc-grpc-alts-1.72.0.jar
-    - io.grpc-grpc-api-1.72.0.jar
-    - io.grpc-grpc-grpclb-1.72.0.jar
-    - io.grpc-grpc-netty-shaded-1.72.0.jar
-    - io.grpc-grpc-services-1.72.0.jar
-    - io.grpc-grpc-xds-1.72.0.jar
-    - io.grpc-grpc-rls-1.72.0.jar
-    - io.grpc-grpc-servlet-1.72.0.jar
-    - io.grpc-grpc-servlet-jakarta-1.72.0.jar
+    - io.grpc-grpc-all-1.75.0.jar
+    - io.grpc-grpc-auth-1.75.0.jar
+    - io.grpc-grpc-context-1.75.0.jar
+    - io.grpc-grpc-core-1.75.0.jar
+    - io.grpc-grpc-protobuf-1.75.0.jar
+    - io.grpc-grpc-protobuf-lite-1.75.0.jar
+    - io.grpc-grpc-stub-1.75.0.jar
+    - io.grpc-grpc-alts-1.75.0.jar
+    - io.grpc-grpc-api-1.75.0.jar
+    - io.grpc-grpc-grpclb-1.75.0.jar
+    - io.grpc-grpc-netty-shaded-1.75.0.jar
+    - io.grpc-grpc-services-1.75.0.jar
+    - io.grpc-grpc-xds-1.75.0.jar
+    - io.grpc-grpc-rls-1.75.0.jar
+    - io.grpc-grpc-servlet-1.75.0.jar
+    - io.grpc-grpc-servlet-jakarta-1.75.0.jar
     - io.grpc-grpc-util-1.60.0.jar
-    - io.grpc-grpc-opentelemetry-1.72.0.jar
-    - io.grpc-grpc-gcp-csm-observability-1.72.0.jar
-    - io.grpc-grpc-inprocess-1.72.0.jar
+    - io.grpc-grpc-opentelemetry-1.75.0.jar
+    - io.grpc-grpc-gcp-csm-observability-1.75.0.jar
+    - io.grpc-grpc-inprocess-1.75.0.jar
   * Perfmark
     - io.perfmark-perfmark-api-0.26.0.jar
   * OpenCensus
@@ -484,8 +484,8 @@ The Apache Software License, Version 2.0
   * Prometheus
     - io.prometheus-simpleclient_httpserver-0.16.0.jar
   * Oxia
-    - io.github.oxia-db-oxia-client-api-0.6.2.jar
-    - io.github.oxia-db-oxia-client-0.6.2.jar
+    - io.github.oxia-db-oxia-client-api-0.7.0.jar
+    - io.github.oxia-db-oxia-client-0.7.0.jar
   * OpenHFT
     - net.openhft-zero-allocation-hashing-0.16.jar
   * Java JSON WebTokens
@@ -541,7 +541,7 @@ The Apache Software License, Version 2.0
     - io.opentelemetry.instrumentation-opentelemetry-runtime-telemetry-java8-1.33.6-alpha.jar
     - io.opentelemetry.semconv-opentelemetry-semconv-1.29.0-alpha.jar
     - com.google.cloud.opentelemetry-detector-resources-support-0.33.0.jar
-    - io.opentelemetry.contrib-opentelemetry-gcp-resources-1.43.0-alpha.jar
+    - io.opentelemetry.contrib-opentelemetry-gcp-resources-1.48.0-alpha.jar
   * Spotify completable-futures
     - com.spotify-completable-futures-0.3.6.jar
   * JSpecify

managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java

@@ -2586,7 +2586,8 @@ public void addWaitingEntryCallBack(WaitingEntryCallBack cb) {
 
     public void maybeUpdateCursorBeforeTrimmingConsumedLedger() {
         for (ManagedCursor cursor : cursors) {
-            Position lastAckedPosition = cursor.getMarkDeletedPosition();
+            Position lastAckedPosition = cursor.getPersistentMarkDeletedPosition() != null
+                    ? cursor.getPersistentMarkDeletedPosition() : cursor.getMarkDeletedPosition();
             LedgerInfo currPointedLedger = ledgers.get(lastAckedPosition.getLedgerId());
             LedgerInfo nextPointedLedger = Optional.ofNullable(ledgers.higherEntry(lastAckedPosition.getLedgerId()))
                     .map(Map.Entry::getValue).orElse(null);
@@ -3716,11 +3717,17 @@ public long getNumberOfEntries(Range<Position> range) {
         boolean toIncluded = range.upperBoundType() == BoundType.CLOSED;
 
         if (fromPosition.getLedgerId() == toPosition.getLedgerId()) {
-            // If the 2 positions are in the same ledger
-            long count = toPosition.getEntryId() - fromPosition.getEntryId() - 1;
-            count += fromIncluded ? 1 : 0;
-            count += toIncluded ? 1 : 0;
-            return count;
+            LedgerInfo li = ledgers.get(toPosition.getLedgerId());
+            if (li != null) {
+                // If the 2 positions are in the same ledger
+                long count = toPosition.getEntryId() - fromPosition.getEntryId() - 1;
+                count += fromIncluded ? 1 : 0;
+                count += toIncluded ? 1 : 0;
+                return count;
+            } else {
+                // if the ledgerId is not in the ledgers, it means it has been deleted
+                return 0;
+            }
         } else {
             long count = 0;
             // If the from & to are pointing to different ledgers, then we need to :