[improve][misc] LS: 1408 cherry pick commit from apache pulsar branch-4.0 to datastax 4.0_ds after 16th jan 2026

conf/broker.conf

@@ -1040,6 +1040,9 @@ httpServerThreadPoolQueueSize=8192
 # Capacity for accept queue in the HTTP server
 httpServerAcceptQueueSize=8192
 
+# Idle timeout for HTTP server connections in milliseconds
+httpServerIdleTimeout=30000
+
 # Maximum number of inbound http connections. (0 to disable limiting)
 maxHttpServerConnections=2048
 

conf/proxy.conf

@@ -315,6 +315,9 @@ httpServerThreadPoolQueueSize=8192
 # Capacity for accept queue in the HTTP server
 httpServerAcceptQueueSize=8192
 
+#Idle timeout for HTTP server connections in milliseconds
+httpServerIdleTimeout=30000
+
 # Maximum number of inbound http connections. (0 to disable limiting)
 maxHttpServerConnections=2048
 
@@ -326,6 +329,9 @@ maxConcurrentHttpRequests=1024
 # denial of service attacks.
 httpMaxRequestHeaderSize = 8192
 
+# The idle timeout value for HTTP proxy is in millisecond
+httpProxyIdleTimeout=30000
+
 ## Configure the datasource of basic authenticate, supports the file and Base64 format.
 # file:
 # basicAuthConf=/path/my/.htpasswd

conf/standalone.conf

@@ -1435,4 +1435,7 @@ topicCompactionRetainNullKey=false
 # If value is "org.apache.pulsar.compaction.EventTimeCompactionServiceFactory",
 # will create topic compaction service based on message eventTime.
 # By default compaction service is based on message publishing order.
-compactionServiceFactoryClassName=org.apache.pulsar.compaction.PulsarCompactionServiceFactory
+compactionServiceFactoryClassName=org.apache.pulsar.compaction.PulsarCompactionServiceFactory
+
+# Idle timeout for HTTP server connections in milliseconds
+httpServerIdleTimeout=30000

distribution/server/src/assemble/LICENSE.bin.txt

@@ -283,15 +283,15 @@ The Apache Software License, Version 2.0
     - com.yahoo.datasketches-sketches-core-0.8.3.jar
  * Apache Commons
     - commons-beanutils-commons-beanutils-1.11.0.jar
-    - commons-cli-commons-cli-1.9.0.jar
-    - commons-codec-commons-codec-1.18.0.jar
-    - commons-io-commons-io-2.19.0.jar
+    - commons-cli-commons-cli-1.11.0.jar
+    - commons-codec-commons-codec-1.20.0.jar
+    - commons-io-commons-io-2.21.0.jar
     - commons-logging-commons-logging-1.3.5.jar
     - org.apache.commons-commons-collections4-4.5.0.jar
-    - org.apache.commons-commons-compress-1.27.1.jar
+    - org.apache.commons-commons-compress-1.28.0.jar
     - org.apache.commons-commons-configuration2-2.12.0.jar
-    - org.apache.commons-commons-lang3-3.18.0.jar
-    - org.apache.commons-commons-text-1.13.1.jar
+    - org.apache.commons-commons-lang3-3.19.0.jar
+    - org.apache.commons-commons-text-1.14.0.jar
  * Netty
     - io.netty-netty-buffer-4.1.130.Final.jar
     - io.netty-netty-codec-4.1.130.Final.jar
@@ -338,12 +338,12 @@ The Apache Software License, Version 2.0
     - io.prometheus-simpleclient_tracer_otel-0.16.0.jar
     - io.prometheus-simpleclient_tracer_otel_agent-0.16.0.jar
  * Prometheus exporter
-    - io.prometheus-prometheus-metrics-config-1.3.4.jar
-    - io.prometheus-prometheus-metrics-exporter-common-1.3.4.jar
-    - io.prometheus-prometheus-metrics-exporter-httpserver-1.3.4.jar
-    - io.prometheus-prometheus-metrics-exposition-formats-1.3.4.jar
-    - io.prometheus-prometheus-metrics-model-1.3.4.jar
-    - io.prometheus-prometheus-metrics-exposition-textformats-1.3.4.jar
+    - io.prometheus-prometheus-metrics-config-1.3.10.jar
+    - io.prometheus-prometheus-metrics-exporter-common-1.3.10.jar
+    - io.prometheus-prometheus-metrics-exporter-httpserver-1.3.10.jar
+    - io.prometheus-prometheus-metrics-exposition-formats-no-protobuf-1.3.10.jar
+    - io.prometheus-prometheus-metrics-model-1.3.10.jar
+    - io.prometheus-prometheus-metrics-exposition-textformats-1.3.10.jar
  * Jakarta Bean Validation API
     - jakarta.validation-jakarta.validation-api-2.0.2.jar
     - javax.validation-validation-api-1.1.0.Final.jar
@@ -357,34 +357,34 @@ The Apache Software License, Version 2.0
     - net.java.dev.jna-jna-jpms-5.12.1.jar
     - net.java.dev.jna-jna-platform-jpms-5.12.1.jar
  * BookKeeper
-    - com.datastax.oss-bookkeeper-common-4.17.1.0.0.4.jar
-    - com.datastax.oss-bookkeeper-common-allocator-4.17.1.0.0.4.jar
-    - com.datastax.oss-bookkeeper-proto-4.17.1.0.0.4.jar
-    - com.datastax.oss-bookkeeper-server-4.17.1.0.0.4.jar
-    - com.datastax.oss-bookkeeper-tools-framework-4.17.1.0.0.4.jar
-    - com.datastax.oss-circe-checksum-4.17.1.0.0.4.jar
-    - com.datastax.oss-cpu-affinity-4.17.1.0.0.4.jar
-    - com.datastax.oss-statelib-4.17.1.0.0.4.jar
-    - com.datastax.oss-stream-storage-api-4.17.1.0.0.4.jar
-    - com.datastax.oss-stream-storage-common-4.17.1.0.0.4.jar
-    - com.datastax.oss-stream-storage-java-client-4.17.1.0.0.4.jar
-    - com.datastax.oss-stream-storage-java-client-base-4.17.1.0.0.4.jar
-    - com.datastax.oss-stream-storage-proto-4.17.1.0.0.4.jar
-    - com.datastax.oss-stream-storage-server-4.17.1.0.0.4.jar
-    - com.datastax.oss-stream-storage-service-api-4.17.1.0.0.4.jar
-    - com.datastax.oss-stream-storage-service-impl-4.17.1.0.0.4.jar
-    - com.datastax.oss-http-server-4.17.1.0.0.4.jar
-    - com.datastax.oss-vertx-http-server-4.17.1.0.0.4.jar
-    - com.datastax.oss-bookkeeper-stats-api-4.17.1.0.0.4.jar
-    - com.datastax.oss-prometheus-metrics-provider-4.17.1.0.0.4.jar
-    - com.datastax.oss-distributedlog-common-4.17.1.0.0.4.jar
-    - com.datastax.oss-distributedlog-core-4.17.1.0.0.4-tests.jar
-    - com.datastax.oss-distributedlog-core-4.17.1.0.0.4.jar
-    - com.datastax.oss-distributedlog-protocol-4.17.1.0.0.4.jar
-    - com.datastax.oss-codahale-metrics-provider-4.17.1.0.0.4.jar
-    - com.datastax.oss-bookkeeper-slogger-api-4.17.1.0.0.4.jar
-    - com.datastax.oss-bookkeeper-slogger-slf4j-4.17.1.0.0.4.jar
-    - com.datastax.oss-native-io-4.17.1.0.0.4.jar
+    - com.datastax.oss-bookkeeper-common-4.17.1.0.0.5.jar
+    - com.datastax.oss-bookkeeper-common-allocator-4.17.1.0.0.5.jar
+    - com.datastax.oss-bookkeeper-proto-4.17.1.0.0.5.jar
+    - com.datastax.oss-bookkeeper-server-4.17.1.0.0.5.jar
+    - com.datastax.oss-bookkeeper-tools-framework-4.17.1.0.0.5.jar
+    - com.datastax.oss-circe-checksum-4.17.1.0.0.5.jar
+    - com.datastax.oss-cpu-affinity-4.17.1.0.0.5.jar
+    - com.datastax.oss-statelib-4.17.1.0.0.5.jar
+    - com.datastax.oss-stream-storage-api-4.17.1.0.0.5.jar
+    - com.datastax.oss-stream-storage-common-4.17.1.0.0.5.jar
+    - com.datastax.oss-stream-storage-java-client-4.17.1.0.0.5.jar
+    - com.datastax.oss-stream-storage-java-client-base-4.17.1.0.0.5.jar
+    - com.datastax.oss-stream-storage-proto-4.17.1.0.0.5.jar
+    - com.datastax.oss-stream-storage-server-4.17.1.0.0.5.jar
+    - com.datastax.oss-stream-storage-service-api-4.17.1.0.0.5.jar
+    - com.datastax.oss-stream-storage-service-impl-4.17.1.0.0.5.jar
+    - com.datastax.oss-http-server-4.17.1.0.0.5.jar
+    - com.datastax.oss-vertx-http-server-4.17.1.0.0.5.jar
+    - com.datastax.oss-bookkeeper-stats-api-4.17.1.0.0.5.jar
+    - com.datastax.oss-prometheus-metrics-provider-4.17.1.0.0.5.jar
+    - com.datastax.oss-distributedlog-common-4.17.1.0.0.5.jar
+    - com.datastax.oss-distributedlog-core-4.17.1.0.0.5-tests.jar
+    - com.datastax.oss-distributedlog-core-4.17.1.0.0.5.jar
+    - com.datastax.oss-distributedlog-protocol-4.17.1.0.0.5.jar
+    - com.datastax.oss-codahale-metrics-provider-4.17.1.0.0.5.jar
+    - com.datastax.oss-bookkeeper-slogger-api-4.17.1.0.0.5.jar
+    - com.datastax.oss-bookkeeper-slogger-slf4j-4.17.1.0.0.5.jar
+    - com.datastax.oss-native-io-4.17.1.0.0.5.jar
   * Apache HTTP Client
     - org.apache.httpcomponents-httpclient-4.5.13.jar
     - org.apache.httpcomponents-httpcore-4.4.15.jar
@@ -418,17 +418,16 @@ The Apache Software License, Version 2.0
  * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.45.0.jar
  * Apache Thrift - org.apache.thrift-libthrift-0.14.2.jar
  * OkHttp3
-     - com.squareup.okhttp3-logging-interceptor-4.9.3.jar
-     - com.squareup.okhttp3-okhttp-4.9.3.jar
+     - com.squareup.okhttp3-logging-interceptor-5.3.1.jar
+     - com.squareup.okhttp3-okhttp-5.3.1.jar
+     - com.squareup.okhttp3-okhttp-jvm-5.3.1.jar
  * Okio
-     - com.squareup.okio-okio-3.4.0.jar
-     - com.squareup.okio-okio-jvm-3.4.0.jar
+     - com.squareup.okio-okio-3.16.3.jar
+     - com.squareup.okio-okio-jvm-3.16.3.jar
  * Javassist -- org.javassist-javassist-3.25.0-GA.jar
  * Kotlin Standard Lib
      - org.jetbrains.kotlin-kotlin-stdlib-1.8.20.jar
      - org.jetbrains.kotlin-kotlin-stdlib-common-1.8.20.jar
-     - org.jetbrains.kotlin-kotlin-stdlib-jdk7-1.8.20.jar
-     - org.jetbrains.kotlin-kotlin-stdlib-jdk8-1.8.20.jar
      - org.jetbrains-annotations-13.0.jar
  * gRPC
     - io.grpc-grpc-all-1.75.0.jar
@@ -505,7 +504,7 @@ The Apache Software License, Version 2.0
     - org.apache.zookeeper-zookeeper-jute-3.9.4.jar
     - org.apache.zookeeper-zookeeper-prometheus-metrics-3.9.4.jar
   * Snappy Java
-    - org.xerial.snappy-snappy-java-1.1.10.5.jar
+    - org.xerial.snappy-snappy-java-1.1.10.8.jar
   * Google HTTP Client
     - com.google.http-client-google-http-client-gson-1.41.0.jar
     - com.google.http-client-google-http-client-1.41.0.jar
@@ -519,27 +518,28 @@ The Apache Software License, Version 2.0
   * RoaringBitmap
     - org.roaringbitmap-RoaringBitmap-1.2.0.jar
   * OpenTelemetry
-    - io.opentelemetry-opentelemetry-api-1.45.0.jar
-    - io.opentelemetry-opentelemetry-api-incubator-1.45.0-alpha.jar
-    - io.opentelemetry-opentelemetry-context-1.45.0.jar
-    - io.opentelemetry-opentelemetry-exporter-common-1.45.0.jar
-    - io.opentelemetry-opentelemetry-exporter-otlp-1.45.0.jar
-    - io.opentelemetry-opentelemetry-exporter-otlp-common-1.45.0.jar
-    - io.opentelemetry-opentelemetry-exporter-prometheus-1.45.0-alpha.jar
-    - io.opentelemetry-opentelemetry-exporter-sender-okhttp-1.45.0.jar
-    - io.opentelemetry-opentelemetry-sdk-1.45.0.jar
-    - io.opentelemetry-opentelemetry-sdk-common-1.45.0.jar
-    - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-1.45.0.jar
-    - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-spi-1.45.0.jar
-    - io.opentelemetry-opentelemetry-sdk-logs-1.45.0.jar
-    - io.opentelemetry-opentelemetry-sdk-metrics-1.45.0.jar
-    - io.opentelemetry-opentelemetry-sdk-trace-1.45.0.jar
-    - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-1.33.6.jar
-    - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-semconv-1.33.6-alpha.jar
-    - io.opentelemetry.instrumentation-opentelemetry-resources-1.33.6-alpha.jar
-    - io.opentelemetry.instrumentation-opentelemetry-runtime-telemetry-java17-1.33.6-alpha.jar
-    - io.opentelemetry.instrumentation-opentelemetry-runtime-telemetry-java8-1.33.6-alpha.jar
-    - io.opentelemetry.semconv-opentelemetry-semconv-1.29.0-alpha.jar
+    - io.opentelemetry-opentelemetry-api-1.56.0.jar
+    - io.opentelemetry-opentelemetry-api-incubator-1.56.0-alpha.jar
+    - io.opentelemetry-opentelemetry-common-1.56.0.jar
+    - io.opentelemetry-opentelemetry-context-1.56.0.jar
+    - io.opentelemetry-opentelemetry-exporter-common-1.56.0.jar
+    - io.opentelemetry-opentelemetry-exporter-otlp-1.56.0.jar
+    - io.opentelemetry-opentelemetry-exporter-otlp-common-1.56.0.jar
+    - io.opentelemetry-opentelemetry-exporter-prometheus-1.56.0-alpha.jar
+    - io.opentelemetry-opentelemetry-exporter-sender-okhttp-1.56.0.jar
+    - io.opentelemetry-opentelemetry-sdk-1.56.0.jar
+    - io.opentelemetry-opentelemetry-sdk-common-1.56.0.jar
+    - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-1.56.0.jar
+    - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-spi-1.56.0.jar
+    - io.opentelemetry-opentelemetry-sdk-logs-1.56.0.jar
+    - io.opentelemetry-opentelemetry-sdk-metrics-1.56.0.jar
+    - io.opentelemetry-opentelemetry-sdk-trace-1.56.0.jar
+    - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-2.21.0.jar
+    - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-incubator-2.21.0-alpha.jar
+    - io.opentelemetry.instrumentation-opentelemetry-resources-2.21.0-alpha.jar
+    - io.opentelemetry.instrumentation-opentelemetry-runtime-telemetry-java17-2.21.0-alpha.jar
+    - io.opentelemetry.instrumentation-opentelemetry-runtime-telemetry-java8-2.21.0-alpha.jar
+    - io.opentelemetry.semconv-opentelemetry-semconv-1.37.0.jar
     - com.google.cloud.opentelemetry-detector-resources-support-0.33.0.jar
     - io.opentelemetry.contrib-opentelemetry-gcp-resources-1.48.0-alpha.jar
   * Spotify completable-futures

distribution/shell/src/assemble/LICENSE.bin.txt

@@ -338,11 +338,11 @@ The Apache Software License, Version 2.0
     - memory-0.8.3.jar
     - sketches-core-0.8.3.jar
  * Apache Commons
-    - commons-codec-1.18.0.jar
-    - commons-io-2.19.0.jar
-    - commons-lang3-3.18.0.jar
-    - commons-text-1.13.1.jar
-    - commons-compress-1.27.1.jar
+    - commons-codec-1.20.0.jar
+    - commons-io-2.21.0.jar
+    - commons-lang3-3.19.0.jar
+    - commons-text-1.14.0.jar
+    - commons-compress-1.28.0.jar
  * Netty
     - netty-buffer-4.1.130.Final.jar
     - netty-codec-4.1.130.Final.jar
@@ -385,14 +385,15 @@ The Apache Software License, Version 2.0
     - log4j-slf4j2-impl-2.25.3.jar
     - log4j-web-2.25.3.jar
  * OpenTelemetry
-    - opentelemetry-api-1.45.0.jar
-    - opentelemetry-api-incubator-1.45.0-alpha.jar
-    - opentelemetry-context-1.45.0.jar
+    - opentelemetry-api-1.56.0.jar
+    - opentelemetry-api-incubator-1.56.0-alpha.jar
+    - opentelemetry-common-1.56.0.jar
+    - opentelemetry-context-1.56.0.jar
 
  * BookKeeper
-    - bookkeeper-common-allocator-4.17.1.0.0.4.jar
-    - cpu-affinity-4.17.1.0.0.4.jar
-    - circe-checksum-4.17.1.0.0.4.jar
+    - bookkeeper-common-allocator-4.17.1.0.0.5.jar
+    - cpu-affinity-4.17.1.0.0.5.jar
+    - circe-checksum-4.17.1.0.0.5.jar
   * AirCompressor
      - aircompressor-0.27.jar
  * AsyncHttpClient

managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryCountEstimator.java

@@ -22,6 +22,7 @@
 import java.util.Collection;
 import java.util.Map;
 import java.util.NavigableMap;
+import java.util.NoSuchElementException;
 import org.apache.bookkeeper.client.LedgerHandle;
 import org.apache.bookkeeper.mledger.Position;
 import org.apache.bookkeeper.mledger.PositionFactory;
@@ -82,17 +83,15 @@ static int internalEstimateEntryCountByBytesSize(int maxEntries, long maxSizeByt
             return maxEntries;
         }
 
-        // Adjust the read position to ensure it falls within the valid range of available ledgers.
-        // This handles special cases such as EARLIEST and LATEST positions by resetting them
-        // to the first available ledger or the last active ledger, respectively.
-        if (lastLedgerId != null && readPosition.getLedgerId() > lastLedgerId.longValue()) {
-            readPosition = PositionFactory.create(lastLedgerId, Math.max(lastLedgerTotalEntries - 1, 0));
-        } else if (lastLedgerId == null && readPosition.getLedgerId() > ledgersInfo.lastKey()) {
-            Map.Entry<Long, MLDataFormats.ManagedLedgerInfo.LedgerInfo> lastEntry = ledgersInfo.lastEntry();
-            readPosition =
-                    PositionFactory.create(lastEntry.getKey(), Math.max(lastEntry.getValue().getEntries() - 1, 0));
-        } else if (readPosition.getLedgerId() < ledgersInfo.firstKey()) {
-            readPosition = PositionFactory.create(ledgersInfo.firstKey(), 0);
+        if (ledgersInfo.isEmpty()) {
+            return 1;
+        }
+
+        try {
+            readPosition = adjustReadPosition(readPosition, ledgersInfo, lastLedgerId, lastLedgerTotalEntries);
+        } catch (NoSuchElementException e) {
+            // there was a race condition where ledgersInfo became empty just before adjustReadPosition was called
+            return 1;
         }
 
         long estimatedEntryCount = 0;
@@ -183,4 +182,28 @@ static int internalEstimateEntryCountByBytesSize(int maxEntries, long maxSizeByt
         // Ensure at least one entry is always returned as the result
         return Math.max((int) Math.min(estimatedEntryCount, maxEntries), 1);
     }
+
+    private static Position adjustReadPosition(Position readPosition,
+                                               NavigableMap<Long, MLDataFormats.ManagedLedgerInfo.LedgerInfo>
+                                                       ledgersInfo,
+                                               Long lastLedgerId, long lastLedgerTotalEntries) {
+        // Adjust the read position to ensure it falls within the valid range of available ledgers.
+        // This handles special cases such as EARLIEST and LATEST positions by resetting them
+        // to the first available ledger or the last active ledger, respectively.
+        if (lastLedgerId != null && readPosition.getLedgerId() > lastLedgerId.longValue()) {
+            return PositionFactory.create(lastLedgerId, Math.max(lastLedgerTotalEntries - 1, 0));
+        }
+        long lastKey = ledgersInfo.lastKey();
+        if (lastLedgerId == null && readPosition.getLedgerId() > lastKey) {
+            Map.Entry<Long, MLDataFormats.ManagedLedgerInfo.LedgerInfo> lastEntry = ledgersInfo.lastEntry();
+            if (lastEntry != null && lastEntry.getKey() == lastKey) {
+                return PositionFactory.create(lastEntry.getKey(), Math.max(lastEntry.getValue().getEntries() - 1, 0));
+            }
+        }
+        long firstKey = ledgersInfo.firstKey();
+        if (readPosition.getLedgerId() < firstKey) {
+            return PositionFactory.create(firstKey, 0);
+        }
+        return readPosition;
+    }
 }

managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java

@@ -685,7 +685,7 @@ protected void recoverFromLedger(final ManagedCursorInfo info, final VoidCallbac
         };
         try {
             bookkeeper.asyncOpenLedger(ledgerId, digestType, getConfig().getPassword(), openCallback,
-                    null);
+                    null, true);
         } catch (Throwable t) {
             log.error("[{}] Encountered error on opening cursor ledger {} for cursor {}",
                 ledger.getName(), ledgerId, name, t);
@@ -2676,10 +2676,7 @@ public void asyncDelete(Iterable<Position> positions, AsyncCallbacks.DeleteCallb
         }
 
         try {
-            Map<String, Long> properties = lastMarkDeleteEntry != null ? lastMarkDeleteEntry.properties
-                    : Collections.emptyMap();
-
-            internalAsyncMarkDelete(newMarkDeletePosition, properties, new MarkDeleteCallback() {
+            internalAsyncMarkDelete(newMarkDeletePosition, null, new MarkDeleteCallback() {
                 @Override
                 public void markDeleteComplete(Object ctx) {
                     callback.deleteComplete(ctx);

managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java

@@ -472,7 +472,7 @@ public void operationComplete(ManagedLedgerInfo mlInfo, Stat stat) {
                         log.debug("[{}] Opening ledger {}", name, id);
                     }
                     mbean.startDataLedgerOpenOp();
-                    bookKeeper.asyncOpenLedger(id, digestType, config.getPassword(), opencb, null);
+                    bookKeeper.asyncOpenLedger(id, digestType, config.getPassword(), opencb, null, true);
                 } else {
                     initializeBookKeeper(callback);
                 }
@@ -1809,7 +1809,7 @@ synchronized void addEntryFailedDueToConcurrentlyModified(final LedgerHandle cur
                 handleBadVersion(new BadVersionException("the current ledger " + currentLedger.getId()
                     + " was concurrent modified by a other bookie client. The error code is: " + errorCode));
             }
-        }, null);
+        }, null, true);
     }
 
     synchronized void ledgerClosed(final LedgerHandle lh) {