diff --git a/.github/actions/tune-runner-vm/action.yml b/.github/actions/tune-runner-vm/action.yml index 7e5f77f9a83..32b7ed6bba3 100644 --- a/.github/actions/tune-runner-vm/action.yml +++ b/.github/actions/tune-runner-vm/action.yml @@ -76,13 +76,6 @@ runs: # stop Azure Linux agent to save RAM sudo systemctl stop walinuxagent.service || true - - # enable docker experimental mode which is - # required for using "docker build --squash" / "-Ddocker.squash=true" - daemon_json="$(sudo cat /etc/docker/daemon.json | jq '.experimental = true')" - echo "$daemon_json" | sudo tee /etc/docker/daemon.json - # restart docker daemon - sudo systemctl restart docker echo '::endgroup::' # show memory diff --git a/.github/workflows/bk-ci.yml b/.github/workflows/bk-ci.yml index e04462f0f54..ca8a3addf41 100644 --- a/.github/workflows/bk-ci.yml +++ b/.github/workflows/bk-ci.yml @@ -24,14 +24,17 @@ on: branches: - master - branch-* + - ds-* push: branches: - master - branch-* + - ds-* workflow_dispatch: env: MAVEN_OPTS: -Xss1500k -Xmx1500m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 + NIST_NVD_API_KEY: ${{ secrets.NIST_NVD_API_KEY }} concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -154,6 +157,13 @@ jobs: distribution: 'temurin' java-version: 11 + - name: Tune Java DNS TTL settings + run: | + sudo tee -a $JAVA_HOME/conf/security/java.security <io.netty netty-buffer + + org.mockito + mockito-core + test + + + org.mockito + mockito-inline + test + diff --git a/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/allocator/ByteBufAllocatorBuilder.java b/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/allocator/ByteBufAllocatorBuilder.java index 3e36a23d170..cb244140b35 100644 --- a/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/allocator/ByteBufAllocatorBuilder.java +++ b/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/allocator/ByteBufAllocatorBuilder.java @@ -92,4 +92,6 @@ static ByteBufAllocatorBuilder create() { *

Default is {@link LeakDetectionPolicy#Disabled} */ ByteBufAllocatorBuilder leakDetectionPolicy(LeakDetectionPolicy leakDetectionPolicy); + + ByteBufAllocatorBuilder exitOnOutOfMemory(boolean exitOnOutOfMemory); } diff --git a/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/allocator/impl/ByteBufAllocatorBuilderImpl.java b/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/allocator/impl/ByteBufAllocatorBuilderImpl.java index 69c57232aff..4b5469a3f7e 100644 --- a/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/allocator/impl/ByteBufAllocatorBuilderImpl.java +++ b/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/allocator/impl/ByteBufAllocatorBuilderImpl.java @@ -37,11 +37,12 @@ public class ByteBufAllocatorBuilderImpl implements ByteBufAllocatorBuilder { OutOfMemoryPolicy outOfMemoryPolicy = OutOfMemoryPolicy.FallbackToHeap; Consumer outOfMemoryListener = null; LeakDetectionPolicy leakDetectionPolicy = LeakDetectionPolicy.Disabled; + boolean exitOnOutOfMemory = false; @Override public ByteBufAllocatorWithOomHandler build() { return new ByteBufAllocatorImpl(pooledAllocator, unpooledAllocator, poolingPolicy, poolingConcurrency, - outOfMemoryPolicy, outOfMemoryListener, leakDetectionPolicy); + outOfMemoryPolicy, outOfMemoryListener, leakDetectionPolicy, exitOnOutOfMemory); } @Override @@ -86,4 +87,10 @@ public ByteBufAllocatorBuilder leakDetectionPolicy(LeakDetectionPolicy leakDetec return this; } + @Override + public ByteBufAllocatorBuilder exitOnOutOfMemory(boolean exitOnOutOfMemory) { + this.exitOnOutOfMemory = exitOnOutOfMemory; + return this; + } + } diff --git a/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/allocator/impl/ByteBufAllocatorImpl.java b/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/allocator/impl/ByteBufAllocatorImpl.java index 87582cca92c..3bc06f8e7ea 100644 --- a/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/allocator/impl/ByteBufAllocatorImpl.java +++ b/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/allocator/impl/ByteBufAllocatorImpl.java @@ -29,6 +29,7 @@ import org.apache.bookkeeper.common.allocator.LeakDetectionPolicy; import org.apache.bookkeeper.common.allocator.OutOfMemoryPolicy; import org.apache.bookkeeper.common.allocator.PoolingPolicy; +import org.apache.bookkeeper.common.util.ShutdownUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,15 +49,24 @@ public class ByteBufAllocatorImpl extends AbstractByteBufAllocator implements By private final PoolingPolicy poolingPolicy; private final OutOfMemoryPolicy outOfMemoryPolicy; private Consumer outOfMemoryListener; + private final boolean exitOnOutOfMemory; ByteBufAllocatorImpl(ByteBufAllocator pooledAllocator, ByteBufAllocator unpooledAllocator, PoolingPolicy poolingPolicy, int poolingConcurrency, OutOfMemoryPolicy outOfMemoryPolicy, Consumer outOfMemoryListener, LeakDetectionPolicy leakDetectionPolicy) { - super(poolingPolicy == PoolingPolicy.PooledDirect /* preferDirect */); + this(pooledAllocator, unpooledAllocator, poolingPolicy, poolingConcurrency, outOfMemoryPolicy, + outOfMemoryListener, leakDetectionPolicy, false); + } + ByteBufAllocatorImpl(ByteBufAllocator pooledAllocator, ByteBufAllocator unpooledAllocator, + PoolingPolicy poolingPolicy, int poolingConcurrency, OutOfMemoryPolicy outOfMemoryPolicy, + Consumer outOfMemoryListener, + LeakDetectionPolicy leakDetectionPolicy, boolean exitOnOutOfMemory) { + super(poolingPolicy == PoolingPolicy.PooledDirect /* preferDirect */); this.poolingPolicy = poolingPolicy; this.outOfMemoryPolicy = outOfMemoryPolicy; + this.exitOnOutOfMemory = exitOnOutOfMemory; if (outOfMemoryListener == null) { this.outOfMemoryListener = (v) -> { log.error("Unable to allocate memory", v); @@ -146,7 +156,7 @@ protected ByteBuf newHeapBuffer(int initialCapacity, int maxCapacity) { : unpooledAllocator; return alloc.heapBuffer(initialCapacity, maxCapacity); } catch (OutOfMemoryError e) { - outOfMemoryListener.accept(e); + consumeOOMError(e); throw e; } } @@ -166,12 +176,12 @@ private ByteBuf newDirectBuffer(int initialCapacity, int maxCapacity, boolean ca try { return unpooledAllocator.heapBuffer(initialCapacity, maxCapacity); } catch (OutOfMemoryError e2) { - outOfMemoryListener.accept(e2); + consumeOOMError(e2); throw e2; } } else { // ThrowException - outOfMemoryListener.accept(e); + consumeOOMError(e); throw e; } } @@ -181,12 +191,24 @@ private ByteBuf newDirectBuffer(int initialCapacity, int maxCapacity, boolean ca try { return unpooledAllocator.directBuffer(initialCapacity, maxCapacity); } catch (OutOfMemoryError e) { - outOfMemoryListener.accept(e); - throw e; + consumeOOMError(e); + throw e; } } } + private void consumeOOMError(OutOfMemoryError outOfMemoryError) { + try { + outOfMemoryListener.accept(outOfMemoryError); + } catch (Throwable e) { + log.warn("Consume outOfMemory error failed.", e); + } + if (exitOnOutOfMemory) { + log.info("Exiting JVM process for OOM error: {}", outOfMemoryError.getMessage(), outOfMemoryError); + ShutdownUtil.triggerImmediateForcefulShutdown(); + } + } + @Override public boolean isDirectBufferPooled() { return pooledAllocator != null && pooledAllocator.isDirectBufferPooled(); diff --git a/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/util/ShutdownUtil.java b/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/util/ShutdownUtil.java new file mode 100644 index 00000000000..a398b57fe74 --- /dev/null +++ b/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/util/ShutdownUtil.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.bookkeeper.common.util; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import lombok.extern.slf4j.Slf4j; + +/** + * Forked from Pulsar. + */ +@Slf4j +public class ShutdownUtil { + private static final Method log4j2ShutdownMethod; + + static { + // use reflection to find org.apache.logging.log4j.LogManager.shutdown method + Method shutdownMethod = null; + try { + shutdownMethod = Class.forName("org.apache.logging.log4j.LogManager") + .getMethod("shutdown"); + } catch (ClassNotFoundException | NoSuchMethodException e) { + // ignore when Log4j2 isn't found, log at debug level + log.debug("Cannot find org.apache.logging.log4j.LogManager.shutdown method", e); + } + log4j2ShutdownMethod = shutdownMethod; + } + + /** + * Triggers an immediate forceful shutdown of the current process. + * + * @param status Termination status. By convention, a nonzero status code indicates abnormal termination. + * @see Runtime#halt(int) + */ + public static void triggerImmediateForcefulShutdown(int status) { + triggerImmediateForcefulShutdown(status, true); + } + public static void triggerImmediateForcefulShutdown(int status, boolean logging) { + try { + if (status != 0 && logging) { + log.warn("Triggering immediate shutdown of current process with status {}", status, + new Exception("Stacktrace for immediate shutdown")); + } + shutdownLogging(); + } finally { + Runtime.getRuntime().halt(status); + } + } + + private static void shutdownLogging() { + // flush log buffers and shutdown log4j2 logging to prevent log truncation + if (log4j2ShutdownMethod != null) { + try { + // use reflection to call org.apache.logging.log4j.LogManager.shutdown() + log4j2ShutdownMethod.invoke(null); + } catch (IllegalAccessException | InvocationTargetException e) { + log.error("Unable to call org.apache.logging.log4j.LogManager.shutdown using reflection.", e); + } + } + } + + /** + * Triggers an immediate forceful shutdown of the current process using 1 as the status code. + * + * @see Runtime#halt(int) + */ + public static void triggerImmediateForcefulShutdown() { + triggerImmediateForcefulShutdown(1); + } +} \ No newline at end of file diff --git a/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/util/package-info.java b/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/util/package-info.java new file mode 100644 index 00000000000..55031dd8f8d --- /dev/null +++ b/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/util/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * defines the utilities for allocator used across the project. + */ +package org.apache.bookkeeper.common.util; \ No newline at end of file diff --git a/bookkeeper-common-allocator/src/test/java/org/apache/bookkeeper/common/allocator/impl/ByteBufAllocatorBuilderTest.java b/bookkeeper-common-allocator/src/test/java/org/apache/bookkeeper/common/allocator/impl/ByteBufAllocatorBuilderTest.java index 6f2538d6c81..40c41fa65bc 100644 --- a/bookkeeper-common-allocator/src/test/java/org/apache/bookkeeper/common/allocator/impl/ByteBufAllocatorBuilderTest.java +++ b/bookkeeper-common-allocator/src/test/java/org/apache/bookkeeper/common/allocator/impl/ByteBufAllocatorBuilderTest.java @@ -23,6 +23,7 @@ import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.when; import io.netty.buffer.ByteBuf; @@ -35,7 +36,10 @@ import org.apache.bookkeeper.common.allocator.ByteBufAllocatorBuilder; import org.apache.bookkeeper.common.allocator.OutOfMemoryPolicy; import org.apache.bookkeeper.common.allocator.PoolingPolicy; +import org.apache.bookkeeper.common.util.ShutdownUtil; import org.junit.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; /** * Tests for {@link ByteBufAllocatorBuilderImpl}. @@ -87,6 +91,30 @@ public void testOomWithException() { assertEquals(outOfDirectMemException, receivedException.get()); } + @Test() + public void testOomExit() { + ByteBufAllocator baseAlloc = mock(ByteBufAllocator.class); + when(baseAlloc.directBuffer(anyInt(), anyInt())).thenThrow(outOfDirectMemException); + + ByteBufAllocator alloc = ByteBufAllocatorBuilder.create() + .pooledAllocator(baseAlloc) + .outOfMemoryPolicy(OutOfMemoryPolicy.ThrowException) + .exitOnOutOfMemory(true) + .build(); + + MockedStatic mockedStatic = mockStatic(ShutdownUtil.class); + + try { + alloc.buffer(); + fail("Should have thrown exception"); + } catch (OutOfMemoryError e) { + // Expected + assertEquals(outOfDirectMemException, e); + } + + mockedStatic.verify(() -> ShutdownUtil.triggerImmediateForcefulShutdown(), Mockito.times(1)); + } + @Test public void testOomWithFallback() { ByteBufAllocator baseAlloc = mock(ByteBufAllocator.class); diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index 52ea9954fe1..790e67297fd 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -211,38 +211,38 @@ Apache Software License, Version 2. - lib/com.google.guava-guava-32.0.1-jre.jar [4] - lib/com.google.guava-failureaccess-1.0.1.jar [4] - lib/com.google.guava-listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar [4] -- lib/commons-cli-commons-cli-1.2.jar [5] -- lib/commons-codec-commons-codec-1.6.jar [6] +- lib/commons-cli-commons-cli-1.9.0.jar [5] +- lib/commons-codec-commons-codec-1.18.0.jar [6] - lib/commons-configuration-commons-configuration-1.10.jar [7] -- lib/commons-io-commons-io-2.7.jar [8] +- lib/commons-io-commons-io-2.19.0.jar [8] - lib/commons-lang-commons-lang-2.6.jar [9] - lib/commons-logging-commons-logging-1.1.1.jar [10] -- lib/io.netty-netty-buffer-4.1.111.Final.jar [11] -- lib/io.netty-netty-codec-4.1.111.Final.jar [11] -- lib/io.netty-netty-codec-dns-4.1.111.Final.jar [11] -- lib/io.netty-netty-codec-http-4.1.111.Final.jar [11] -- lib/io.netty-netty-codec-http2-4.1.111.Final.jar [11] -- lib/io.netty-netty-codec-socks-4.1.111.Final.jar [11] -- lib/io.netty-netty-common-4.1.111.Final.jar [11] -- lib/io.netty-netty-handler-4.1.111.Final.jar [11] -- lib/io.netty-netty-handler-proxy-4.1.111.Final.jar [11] -- lib/io.netty-netty-resolver-4.1.111.Final.jar [11] -- lib/io.netty-netty-resolver-dns-4.1.111.Final.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-linux-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-osx-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-osx-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-windows-x86_64.jar [11] -- lib/io.netty-netty-tcnative-classes-2.0.65.Final.jar [11] -- lib/io.netty-netty-transport-4.1.111.Final.jar [11] -- lib/io.netty-netty-transport-classes-epoll-4.1.111.Final.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.111.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.111.Final-linux-x86_64.jar [11] -- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-x86_64.jar [11] -- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-aarch_64.jar [11] -- lib/io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.25.Final.jar [11] -- lib/io.netty-netty-transport-native-unix-common-4.1.111.Final.jar [11] +- lib/io.netty-netty-buffer-4.1.121.Final.jar [11] +- lib/io.netty-netty-codec-4.1.121.Final.jar [11] +- lib/io.netty-netty-codec-dns-4.1.121.Final.jar [11] +- lib/io.netty-netty-codec-http-4.1.121.Final.jar [11] +- lib/io.netty-netty-codec-http2-4.1.121.Final.jar [11] +- lib/io.netty-netty-codec-socks-4.1.121.Final.jar [11] +- lib/io.netty-netty-common-4.1.121.Final.jar [11] +- lib/io.netty-netty-handler-4.1.121.Final.jar [11] +- lib/io.netty-netty-handler-proxy-4.1.121.Final.jar [11] +- lib/io.netty-netty-resolver-4.1.121.Final.jar [11] +- lib/io.netty-netty-resolver-dns-4.1.121.Final.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-windows-x86_64.jar [11] +- lib/io.netty-netty-tcnative-classes-2.0.70.Final.jar [11] +- lib/io.netty-netty-transport-4.1.121.Final.jar [11] +- lib/io.netty-netty-transport-classes-epoll-4.1.121.Final.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.121.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.121.Final-linux-x86_64.jar [11] +- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar [11] +- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar [11] +- lib/io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.26.Final.jar [11] +- lib/io.netty-netty-transport-native-unix-common-4.1.121.Final.jar [11] - lib/io.prometheus-simpleclient-0.15.0.jar [12] - lib/io.prometheus-simpleclient_common-0.15.0.jar [12] - lib/io.prometheus-simpleclient_hotspot-0.15.0.jar [12] @@ -251,31 +251,31 @@ Apache Software License, Version 2. - lib/io.prometheus-simpleclient_tracer_common-0.15.0.jar [12] - lib/io.prometheus-simpleclient_tracer_otel-0.15.0.jar [12] - lib/io.prometheus-simpleclient_tracer_otel_agent-0.15.0.jar [12] -- lib/io.vertx-vertx-auth-common-4.5.7.jar [13] -- lib/io.vertx-vertx-bridge-common-4.5.7.jar [14] -- lib/io.vertx-vertx-core-4.5.7.jar [15] -- lib/io.vertx-vertx-web-4.5.7.jar [16] -- lib/io.vertx-vertx-web-common-4.5.7.jar [16] +- lib/io.vertx-vertx-auth-common-4.5.11.jar [13] +- lib/io.vertx-vertx-bridge-common-4.5.11.jar [14] +- lib/io.vertx-vertx-core-4.5.11.jar [15] +- lib/io.vertx-vertx-web-4.5.11.jar [16] +- lib/io.vertx-vertx-web-common-4.5.11.jar [16] - lib/org.apache.logging.log4j-log4j-api-2.18.0.jar [17] - lib/org.apache.logging.log4j-log4j-core-2.18.0.jar [17] - lib/org.apache.logging.log4j-log4j-slf4j-impl-2.18.0.jar [17] - lib/org.apache.commons-commons-collections4-4.1.jar [19] -- lib/org.apache.commons-commons-lang3-3.6.jar [20] -- lib/org.apache.zookeeper-zookeeper-3.8.3.jar [21] -- lib/org.apache.zookeeper-zookeeper-jute-3.8.3.jar [21] -- lib/org.apache.zookeeper-zookeeper-3.8.3-tests.jar [21] -- lib/org.eclipse.jetty-jetty-http-9.4.53.v20231009.jar [22] -- lib/org.eclipse.jetty-jetty-io-9.4.53.v20231009.jar [22] -- lib/org.eclipse.jetty-jetty-security-9.4.53.v20231009.jar [22] -- lib/org.eclipse.jetty-jetty-server-9.4.53.v20231009.jar [22] -- lib/org.eclipse.jetty-jetty-servlet-9.4.53.v20231009.jar [22] -- lib/org.eclipse.jetty-jetty-util-9.4.53.v20231009.jar [22] -- lib/org.eclipse.jetty-jetty-util-ajax-9.4.53.v20231009.jar [22] +- lib/org.apache.commons-commons-lang3-3.17.0.jar [58] +- lib/org.apache.zookeeper-zookeeper-3.9.3.jar [21] +- lib/org.apache.zookeeper-zookeeper-jute-3.9.3.jar [21] +- lib/org.apache.zookeeper-zookeeper-3.9.3-tests.jar [21] +- lib/org.eclipse.jetty-jetty-http-9.4.57.v20241219.jar [22] +- lib/org.eclipse.jetty-jetty-io-9.4.57.v20241219.jar [22] +- lib/org.eclipse.jetty-jetty-security-9.4.57.v20241219.jar [22] +- lib/org.eclipse.jetty-jetty-server-9.4.57.v20241219.jar [22] +- lib/org.eclipse.jetty-jetty-servlet-9.4.57.v20241219.jar [22] +- lib/org.eclipse.jetty-jetty-util-9.4.57.v20241219.jar [22] +- lib/org.eclipse.jetty-jetty-util-ajax-9.4.57.v20241219.jar [22] - lib/org.rocksdb-rocksdbjni-7.9.2.jar [23] - lib/com.beust-jcommander-1.82.jar [24] - lib/com.yahoo.datasketches-memory-0.8.3.jar [25] - lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25] -- lib/net.jpountz.lz4-lz4-1.3.0.jar [26] +- lib/at.yawk.lz4-lz4-java-1.10.2.jar [26] - lib/com.google.api.grpc-proto-google-common-protos-2.9.0.jar [28] - lib/com.google.code.gson-gson-2.9.0.jar [29] - lib/io.opencensus-opencensus-api-0.28.0.jar [30] @@ -296,9 +296,9 @@ Apache Software License, Version 2. - lib/io.grpc-grpc-testing-1.54.1.jar [33] - lib/io.grpc-grpc-xds-1.54.1.jar [33] - lib/io.grpc-grpc-rls-1.54.1.jar[33] -- lib/org.apache.curator-curator-client-5.1.0.jar [34] -- lib/org.apache.curator-curator-framework-5.1.0.jar [34] -- lib/org.apache.curator-curator-recipes-5.1.0.jar [34] +- lib/org.apache.curator-curator-client-5.7.1.jar [34] +- lib/org.apache.curator-curator-framework-5.7.1.jar [34] +- lib/org.apache.curator-curator-recipes-5.7.1.jar [34] - lib/com.google.errorprone-error_prone_annotations-2.9.0.jar [36] - lib/org.apache.yetus-audience-annotations-0.12.0.jar [37] - lib/org.jctools-jctools-core-2.1.2.jar [38] @@ -326,13 +326,13 @@ Apache Software License, Version 2. [2] Source available at https://github.com/FasterXML/jackson-core/tree/jackson-core-2.17.1 [3] Source available at https://github.com/FasterXML/jackson-databind/tree/jackson-databind-2.17.1 [4] Source available at https://github.com/google/guava/tree/v32.0.1 -[5] Source available at https://github.com/apache/commons-cli/tree/cli-1.2 -[6] Source available at https://github.com/apache/commons-codec/tree/commons-codec-1.6-RC2 +[5] Source available at https://github.com/apache/commons-cli/tree/rel/commons-cli-1.9.0 +[6] Source available at https://github.com/apache/commons-codec/tree/rel/commons-codec-1.18.0 [7] Source available at https://github.com/apache/commons-configuration/tree/CONFIGURATION_1_10 -[8] Source available at https://github.com/apache/commons-io/tree/rel/commons-io-2.7 +[8] Source available at https://github.com/apache/commons-io/tree/rel/commons-io-2.19.0 [9] Source available at https://github.com/apache/commons-lang/tree/LANG_2_6 [10] Source available at https://github.com/apache/commons-logging/tree/commons-logging-1.1.1 -[11] Source available at https://github.com/netty/netty/tree/netty-4.1.111.Final +[11] Source available at https://github.com/netty/netty/tree/netty-4.1.121.Final [12] Source available at https://github.com/prometheus/client_java/tree/parent-0.15.0 [13] Source available at https://github.com/vert-x3/vertx-auth/tree/4.3.2 [14] Source available at https://github.com/vert-x3/vertx-bridge-common/tree/4.3.2 @@ -346,12 +346,12 @@ Apache Software License, Version 2. [23] Source available at https://github.com/facebook/rocksdb/tree/v6.29.4.1 [24] Source available at https://github.com/cbeust/jcommander/tree/1.82 [25] Source available at https://github.com/DataSketches/sketches-core/tree/sketches-0.8.3 -[26] Source available at https://github.com/lz4/lz4-java/tree/1.3.0 +[26] Source available at https://github.com/yawkat/lz4-java/tree/v1.10.2 [28] Source available at https://github.com/googleapis/java-common-protos/tree/v2.17.0 [29] Source available at https://github.com/google/gson/tree/gson-parent-2.10.1 [30] Source available at https://github.com/census-instrumentation/opencensus-java/tree/v0.28.0 [33] Source available at https://github.com/grpc/grpc-java/tree/v1.56.0 -[34] Source available at https://github.com/apache/curator/releases/tag/apache.curator-5.1.0 +[34] Source available at https://github.com/apache/curator/releases/tag/apache.curator-5.7.1 [36] Source available at https://github.com/google/error-prone/tree/v2.9.0 [37] Source available at https://github.com/apache/yetus/tree/rel/0.12.0 [38] Source available at https://github.com/JCTools/JCTools/tree/v2.1.2 @@ -370,11 +370,12 @@ Apache Software License, Version 2. [51] Source available at https://github.com/ReactiveX/RxJava/tree/v3.0.1 [52] Source available at https://github.com/HdrHistogram/HdrHistogram/tree/HdrHistogram-2.1.10 [53] Source available at https://github.com/carrotsearch/hppc/tree/0.9.1 +[58] Source available at https://github.com/apache/commons-lang/tree/rel/commons-lang-3.17.0 ------------------------------------------------------------------------------------ -lib/io.netty-netty-codec-4.1.111.Final.jar bundles some 3rd party dependencies +lib/io.netty-netty-codec-4.1.121.Final.jar bundles some 3rd party dependencies -lib/io.netty-netty-codec-4.1.111.Final.jar contains the extensions to Java Collections Framework which has +lib/io.netty-netty-codec-4.1.121.Final.jar contains the extensions to Java Collections Framework which has been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * LICENSE: @@ -383,7 +384,7 @@ been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified version of Robert Harder's Public Domain +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified version of Robert Harder's Public Domain Base64 Encoder and Decoder, which can be obtained at: * LICENSE: @@ -391,7 +392,7 @@ Base64 Encoder and Decoder, which can be obtained at: * HOMEPAGE: * http://iharder.sourceforge.net/current/java/base64/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'Webbit', an event based +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'Webbit', an event based WebSocket and HTTP server, which can be obtained at: * LICENSE: @@ -399,7 +400,7 @@ WebSocket and HTTP server, which can be obtained at: * HOMEPAGE: * https://github.com/joewalnes/webbit -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'SLF4J', a simple logging +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'SLF4J', a simple logging facade for Java, which can be obtained at: * LICENSE: @@ -407,7 +408,7 @@ facade for Java, which can be obtained at: * HOMEPAGE: * http://www.slf4j.org/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'Apache Harmony', an open source +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'Apache Harmony', an open source Java SE, which can be obtained at: * NOTICE: @@ -417,7 +418,7 @@ Java SE, which can be obtained at: * HOMEPAGE: * http://archive.apache.org/dist/harmony/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression and decompression library written by Matthew J. Francis. It can be obtained at: * LICENSE: @@ -425,7 +426,7 @@ and decompression library written by Matthew J. Francis. It can be obtained at: * HOMEPAGE: * https://code.google.com/p/jbzip2/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct the suffix array and the Burrows-Wheeler transformed string for any input string of a constant-size alphabet written by Yuta Mori. It can be obtained at: @@ -434,7 +435,7 @@ a constant-size alphabet written by Yuta Mori. It can be obtained at: * HOMEPAGE: * https://github.com/y-256/libdivsufsort -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, which can be obtained at: * LICENSE: @@ -442,7 +443,7 @@ Java Concurrency Tools for the JVM, which can be obtained at: * HOMEPAGE: * https://github.com/JCTools/JCTools -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in pure Java, which can be obtained at: * LICENSE: @@ -450,7 +451,7 @@ pure Java, which can be obtained at: * HOMEPAGE: * http://www.jcraft.com/jzlib/ -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * LICENSE: @@ -458,15 +459,15 @@ decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * HOMEPAGE: * https://github.com/ning/compress -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'lz4', a LZ4 Java compression +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'lz4-java', a LZ4 Java compression and decompression library written by Adrien Grand. It can be obtained at: * LICENSE: * deps/netty/LICENSE.lz4.txt (Apache License 2.0) * HOMEPAGE: - * https://github.com/jpountz/lz4-java + * https://github.com/yawkat/lz4-java -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'lzma-java', a LZMA Java compression +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'lzma-java', a LZMA Java compression and decompression library, which can be obtained at: * LICENSE: @@ -474,7 +475,7 @@ and decompression library, which can be obtained at: * HOMEPAGE: * https://github.com/jponge/lzma-java -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression and decompression library written by William Kinney. It can be obtained at: * LICENSE: @@ -482,7 +483,7 @@ and decompression library written by William Kinney. It can be obtained at: * HOMEPAGE: * https://code.google.com/p/jfastlz/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data interchange format, which can be obtained at: * LICENSE: @@ -490,7 +491,7 @@ Google's data interchange format, which can be obtained at: * HOMEPAGE: * https://github.com/google/protobuf -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate a temporary self-signed X.509 certificate when the JVM does not provide the equivalent functionality. It can be obtained at: @@ -499,7 +500,7 @@ equivalent functionality. It can be obtained at: * HOMEPAGE: * http://www.bouncycastle.org/ -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Snappy', a compression library produced +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Snappy', a compression library produced by Google Inc, which can be obtained at: * LICENSE: @@ -507,7 +508,7 @@ by Google Inc, which can be obtained at: * HOMEPAGE: * https://github.com/google/snappy -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java serialization API, which can be obtained at: * LICENSE: @@ -515,7 +516,7 @@ serialization API, which can be obtained at: * HOMEPAGE: * https://github.com/jboss-remoting/jboss-marshalling -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Caliper', Google's micro- +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Caliper', Google's micro- benchmarking framework, which can be obtained at: * LICENSE: @@ -523,7 +524,7 @@ benchmarking framework, which can be obtained at: * HOMEPAGE: * https://github.com/google/caliper -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Apache Commons Logging', a logging +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Apache Commons Logging', a logging framework, which can be obtained at: * LICENSE: @@ -531,7 +532,7 @@ framework, which can be obtained at: * HOMEPAGE: * http://commons.apache.org/logging/ -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Apache Log4J', a logging framework, which +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Apache Log4J', a logging framework, which can be obtained at: * LICENSE: @@ -539,7 +540,7 @@ can be obtained at: * HOMEPAGE: * http://logging.apache.org/log4j/ -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Aalto XML', an ultra-high performance +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Aalto XML', an ultra-high performance non-blocking XML processor, which can be obtained at: * LICENSE: @@ -547,7 +548,7 @@ non-blocking XML processor, which can be obtained at: * HOMEPAGE: * http://wiki.fasterxml.com/AaltoHome -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * LICENSE: @@ -555,7 +556,7 @@ the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * HOMEPAGE: * https://github.com/twitter/hpack -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * LICENSE: @@ -563,7 +564,7 @@ the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * HOMEPAGE: * https://github.com/python-hyper/hpack/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at: * LICENSE: @@ -571,7 +572,7 @@ the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at * HOMEPAGE: * https://github.com/nghttp2/nghttp2/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library provides utilities for the java.lang API, which can be obtained at: * LICENSE: @@ -580,7 +581,7 @@ provides utilities for the java.lang API, which can be obtained at: * https://commons.apache.org/proper/commons-lang/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', +lib/io.netty-netty-codec-4.1.121.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. * LICENSE: @@ -588,7 +589,7 @@ that provides an easy way to ensure a user has everything necessary to run the M * HOMEPAGE: * https://github.com/takari/maven-wrapper -lib/io.netty-netty-codec-4.1.111.Final.jar contains the dnsinfo.h header file, +lib/io.netty-netty-codec-4.1.121.Final.jar contains the dnsinfo.h header file, that provides a way to retrieve the system DNS configuration on MacOS. This private header is also used by Apple's open source mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/). @@ -640,13 +641,13 @@ This product bundles Google Protocol Buffers, which is available under a "3-clau license. Bundled as - - lib/com.google.protobuf-protobuf-java-3.21.9.jar -Source available at https://github.com/google/protobuf/tree/v3.21.9 + - lib/com.google.protobuf-protobuf-java-3.25.5.jar +Source available at https://github.com/google/protobuf/tree/v3.25.5 For details, see deps/protobuf-3.14.0/LICENSE. Bundled as - - lib/com.google.protobuf-protobuf-java-util-3.21.9.jar -Source available at https://github.com/protocolbuffers/protobuf/tree/v3.21.9 + - lib/com.google.protobuf-protobuf-java-util-3.25.5.jar +Source available at https://github.com/protocolbuffers/protobuf/tree/v3.25.5 For details, see deps/protobuf-3.12.0/LICENSE. ------------------------------------------------------------------------------------ This product bundles the JCP Standard Java Servlet API, which is available under a diff --git a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt index dfd2fea4f0c..9ca54cc559b 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt @@ -211,42 +211,42 @@ Apache Software License, Version 2. - lib/com.google.guava-guava-32.0.1-jre.jar [4] - lib/com.google.guava-failureaccess-1.0.1.jar [4] - lib/com.google.guava-listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar [4] -- lib/commons-cli-commons-cli-1.2.jar [5] -- lib/commons-codec-commons-codec-1.6.jar [6] +- lib/commons-cli-commons-cli-1.9.0.jar [5] +- lib/commons-codec-commons-codec-1.18.0.jar [6] - lib/commons-configuration-commons-configuration-1.10.jar [7] -- lib/commons-io-commons-io-2.7.jar [8] +- lib/commons-io-commons-io-2.19.0.jar [8] - lib/commons-lang-commons-lang-2.6.jar [9] - lib/commons-logging-commons-logging-1.1.1.jar [10] -- lib/io.netty-netty-buffer-4.1.111.Final.jar [11] -- lib/io.netty-netty-codec-4.1.111.Final.jar [11] -- lib/io.netty-netty-common-4.1.111.Final.jar [11] -- lib/io.netty-netty-handler-4.1.111.Final.jar [11] -- lib/io.netty-netty-resolver-4.1.111.Final.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-linux-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-osx-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-osx-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-windows-x86_64.jar [11] -- lib/io.netty-netty-tcnative-classes-2.0.65.Final.jar [11] -- lib/io.netty-netty-transport-4.1.111.Final.jar [11] -- lib/io.netty-netty-transport-classes-epoll-4.1.111.Final.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.111.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.111.Final-linux-x86_64.jar [11] -- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-x86_64.jar [11] -- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-aarch_64.jar [11] -- lib/io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.25.Final.jar [11] -- lib/io.netty-netty-transport-native-unix-common-4.1.111.Final.jar [11] +- lib/io.netty-netty-buffer-4.1.121.Final.jar [11] +- lib/io.netty-netty-codec-4.1.121.Final.jar [11] +- lib/io.netty-netty-common-4.1.121.Final.jar [11] +- lib/io.netty-netty-handler-4.1.121.Final.jar [11] +- lib/io.netty-netty-resolver-4.1.121.Final.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-windows-x86_64.jar [11] +- lib/io.netty-netty-tcnative-classes-2.0.70.Final.jar [11] +- lib/io.netty-netty-transport-4.1.121.Final.jar [11] +- lib/io.netty-netty-transport-classes-epoll-4.1.121.Final.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.121.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.121.Final-linux-x86_64.jar [11] +- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar [11] +- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar [11] +- lib/io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.26.Final.jar [11] +- lib/io.netty-netty-transport-native-unix-common-4.1.121.Final.jar [11] - lib/org.apache.logging.log4j-log4j-api-2.18.0.jar [16] - lib/org.apache.logging.log4j-log4j-core-2.18.0.jar [16] - lib/org.apache.logging.log4j-log4j-slf4j-impl-2.18.0.jar [16] - lib/org.apache.commons-commons-collections4-4.1.jar [18] -- lib/org.apache.commons-commons-lang3-3.6.jar [19] -- lib/org.apache.zookeeper-zookeeper-3.8.3.jar [20] -- lib/org.apache.zookeeper-zookeeper-jute-3.8.3.jar [20] -- lib/org.apache.zookeeper-zookeeper-3.8.3-tests.jar [20] +- lib/org.apache.commons-commons-lang3-3.17.0.jar [55] +- lib/org.apache.zookeeper-zookeeper-3.9.3.jar [20] +- lib/org.apache.zookeeper-zookeeper-jute-3.9.3.jar [20] +- lib/org.apache.zookeeper-zookeeper-3.9.3-tests.jar [20] - lib/com.beust-jcommander-1.82.jar [23] -- lib/net.jpountz.lz4-lz4-1.3.0.jar [25] +- lib/at.yawk.lz4-lz4-java-1.10.2.jar [25] - lib/com.google.api.grpc-proto-google-common-protos-2.9.0.jar [27] - lib/com.google.code.gson-gson-2.9.0.jar [28] - lib/io.opencensus-opencensus-api-0.28.0.jar [29] @@ -267,9 +267,9 @@ Apache Software License, Version 2. - lib/io.grpc-grpc-testing-1.54.1.jar [32] - lib/io.grpc-grpc-xds-1.54.1.jar [32] - lib/io.grpc-grpc-rls-1.54.1.jar[32] -- lib/org.apache.curator-curator-client-5.1.0.jar [33] -- lib/org.apache.curator-curator-framework-5.1.0.jar [33] -- lib/org.apache.curator-curator-recipes-5.1.0.jar [33] +- lib/org.apache.curator-curator-client-5.7.1.jar [33] +- lib/org.apache.curator-curator-framework-5.7.1.jar [33] +- lib/org.apache.curator-curator-recipes-5.7.1.jar [33] - lib/com.google.errorprone-error_prone_annotations-2.9.0.jar [35] - lib/org.apache.yetus-audience-annotations-0.12.0.jar [36] - lib/org.jctools-jctools-core-2.1.2.jar [37] @@ -293,24 +293,24 @@ Apache Software License, Version 2. [2] Source available at https://github.com/FasterXML/jackson-core/tree/jackson-core-2.17.1 [3] Source available at https://github.com/FasterXML/jackson-databind/tree/jackson-databind-2.17.1 [4] Source available at https://github.com/google/guava/tree/v32.0.1 -[5] Source available at https://github.com/apache/commons-cli/tree/cli-1.2 -[6] Source available at https://github.com/apache/commons-codec/tree/commons-codec-1.6-RC2 +[5] Source available at https://github.com/apache/commons-cli/tree/rel/commons-cli-1.9.0 +[6] Source available at https://github.com/apache/commons-codec/tree/rel/commons-codec-1.18.0 [7] Source available at https://github.com/apache/commons-configuration/tree/CONFIGURATION_1_10 -[8] Source available at https://github.com/apache/commons-io/tree/rel/commons-io-2.7 +[8] Source available at https://github.com/apache/commons-io/tree/rel/commons-io-2.19.0 [9] Source available at https://github.com/apache/commons-lang/tree/LANG_2_6 [10] Source available at https://github.com/apache/commons-logging/tree/commons-logging-1.1.1 -[11] Source available at https://github.com/netty/netty/tree/netty-4.1.111.Final +[11] Source available at https://github.com/netty/netty/tree/netty-4.1.121.Final [16] Source available at https://github.com/apache/logging-log4j2/tree/rel/2.18.0 [18] Source available at https://github.com/apache/commons-collections/tree/collections-4.1 [19] Source available at https://github.com/apache/commons-lang/tree/LANG_3_6 [20] Source available at https://github.com/apache/zookeeper/tree/release-3.8.0 [23] Source available at https://github.com/cbeust/jcommander/tree/1.82 -[25] Source available at https://github.com/lz4/lz4-java/tree/1.3.0 +[25] Source available at https://github.com/yawkat/lz4-java/tree/v1.10.2 [27] Source available at https://github.com/googleapis/java-common-protos/tree/v2.17.0 [28] Source available at https://github.com/google/gson/tree/gson-parent-2.10.1 [29] Source available at https://github.com/census-instrumentation/opencensus-java/tree/v0.28.0 [32] Source available at https://github.com/grpc/grpc-java/tree/v1.56.0 -[33] Source available at https://github.com/apache/curator/tree/apache-curator-5.1.0 +[33] Source available at https://github.com/apache/curator/tree/apache-curator-5.7.1 [35] Source available at https://github.com/google/error-prone/tree/v2.9.0 [36] Source available at https://github.com/apache/yetus/tree/rel/0.12.0 [37] Source available at https://github.com/JCTools/JCTools/tree/v2.1.2 @@ -328,11 +328,11 @@ Apache Software License, Version 2. [50] Source available at https://github.com/xerial/snappy-java/releases/tag/v1.1.10.5 [51] Source available at https://github.com/ReactiveX/RxJava/tree/v3.0.1 [52] Source available at https://github.com/carrotsearch/hppc/tree/0.9.1 - +[55] Source available at https://github.com/apache/commons-lang/tree/rel/commons-lang-3.17.0 ------------------------------------------------------------------------------------ -lib/io.netty-netty-codec-4.1.111.Final.jar bundles some 3rd party dependencies +lib/io.netty-netty-codec-4.1.121.Final.jar bundles some 3rd party dependencies -lib/io.netty-netty-codec-4.1.111.Final.jar contains the extensions to Java Collections Framework which has +lib/io.netty-netty-codec-4.1.121.Final.jar contains the extensions to Java Collections Framework which has been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * LICENSE: @@ -341,7 +341,7 @@ been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified version of Robert Harder's Public Domain +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified version of Robert Harder's Public Domain Base64 Encoder and Decoder, which can be obtained at: * LICENSE: @@ -349,7 +349,7 @@ Base64 Encoder and Decoder, which can be obtained at: * HOMEPAGE: * http://iharder.sourceforge.net/current/java/base64/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'Webbit', an event based +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'Webbit', an event based WebSocket and HTTP server, which can be obtained at: * LICENSE: @@ -357,7 +357,7 @@ WebSocket and HTTP server, which can be obtained at: * HOMEPAGE: * https://github.com/joewalnes/webbit -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'SLF4J', a simple logging +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'SLF4J', a simple logging facade for Java, which can be obtained at: * LICENSE: @@ -365,7 +365,7 @@ facade for Java, which can be obtained at: * HOMEPAGE: * http://www.slf4j.org/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'Apache Harmony', an open source +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'Apache Harmony', an open source Java SE, which can be obtained at: * NOTICE: @@ -375,7 +375,7 @@ Java SE, which can be obtained at: * HOMEPAGE: * http://archive.apache.org/dist/harmony/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression and decompression library written by Matthew J. Francis. It can be obtained at: * LICENSE: @@ -383,7 +383,7 @@ and decompression library written by Matthew J. Francis. It can be obtained at: * HOMEPAGE: * https://code.google.com/p/jbzip2/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct the suffix array and the Burrows-Wheeler transformed string for any input string of a constant-size alphabet written by Yuta Mori. It can be obtained at: @@ -392,7 +392,7 @@ a constant-size alphabet written by Yuta Mori. It can be obtained at: * HOMEPAGE: * https://github.com/y-256/libdivsufsort -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, which can be obtained at: * LICENSE: @@ -400,7 +400,7 @@ Java Concurrency Tools for the JVM, which can be obtained at: * HOMEPAGE: * https://github.com/JCTools/JCTools -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in pure Java, which can be obtained at: * LICENSE: @@ -408,7 +408,7 @@ pure Java, which can be obtained at: * HOMEPAGE: * http://www.jcraft.com/jzlib/ -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * LICENSE: @@ -416,15 +416,15 @@ decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * HOMEPAGE: * https://github.com/ning/compress -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'lz4', a LZ4 Java compression +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'lz4-java', a LZ4 Java compression and decompression library written by Adrien Grand. It can be obtained at: * LICENSE: * deps/netty/LICENSE.lz4.txt (Apache License 2.0) * HOMEPAGE: - * https://github.com/jpountz/lz4-java + * https://github.com/yawkat/lz4-java -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'lzma-java', a LZMA Java compression +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'lzma-java', a LZMA Java compression and decompression library, which can be obtained at: * LICENSE: @@ -432,7 +432,7 @@ and decompression library, which can be obtained at: * HOMEPAGE: * https://github.com/jponge/lzma-java -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression and decompression library written by William Kinney. It can be obtained at: * LICENSE: @@ -440,7 +440,7 @@ and decompression library written by William Kinney. It can be obtained at: * HOMEPAGE: * https://code.google.com/p/jfastlz/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data interchange format, which can be obtained at: * LICENSE: @@ -448,7 +448,7 @@ Google's data interchange format, which can be obtained at: * HOMEPAGE: * https://github.com/google/protobuf -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate a temporary self-signed X.509 certificate when the JVM does not provide the equivalent functionality. It can be obtained at: @@ -457,7 +457,7 @@ equivalent functionality. It can be obtained at: * HOMEPAGE: * http://www.bouncycastle.org/ -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Snappy', a compression library produced +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Snappy', a compression library produced by Google Inc, which can be obtained at: * LICENSE: @@ -465,7 +465,7 @@ by Google Inc, which can be obtained at: * HOMEPAGE: * https://github.com/google/snappy -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java serialization API, which can be obtained at: * LICENSE: @@ -473,7 +473,7 @@ serialization API, which can be obtained at: * HOMEPAGE: * https://github.com/jboss-remoting/jboss-marshalling -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Caliper', Google's micro- +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Caliper', Google's micro- benchmarking framework, which can be obtained at: * LICENSE: @@ -481,7 +481,7 @@ benchmarking framework, which can be obtained at: * HOMEPAGE: * https://github.com/google/caliper -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Apache Commons Logging', a logging +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Apache Commons Logging', a logging framework, which can be obtained at: * LICENSE: @@ -489,7 +489,7 @@ framework, which can be obtained at: * HOMEPAGE: * http://commons.apache.org/logging/ -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Apache Log4J', a logging framework, which +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Apache Log4J', a logging framework, which can be obtained at: * LICENSE: @@ -497,7 +497,7 @@ can be obtained at: * HOMEPAGE: * http://logging.apache.org/log4j/ -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Aalto XML', an ultra-high performance +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Aalto XML', an ultra-high performance non-blocking XML processor, which can be obtained at: * LICENSE: @@ -505,7 +505,7 @@ non-blocking XML processor, which can be obtained at: * HOMEPAGE: * http://wiki.fasterxml.com/AaltoHome -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * LICENSE: @@ -513,7 +513,7 @@ the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * HOMEPAGE: * https://github.com/twitter/hpack -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * LICENSE: @@ -521,7 +521,7 @@ the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * HOMEPAGE: * https://github.com/python-hyper/hpack/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at: * LICENSE: @@ -529,7 +529,7 @@ the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at * HOMEPAGE: * https://github.com/nghttp2/nghttp2/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library provides utilities for the java.lang API, which can be obtained at: * LICENSE: @@ -538,7 +538,7 @@ provides utilities for the java.lang API, which can be obtained at: * https://commons.apache.org/proper/commons-lang/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', +lib/io.netty-netty-codec-4.1.121.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. * LICENSE: @@ -546,7 +546,7 @@ that provides an easy way to ensure a user has everything necessary to run the M * HOMEPAGE: * https://github.com/takari/maven-wrapper -lib/io.netty-netty-codec-4.1.111.Final.jar contains the dnsinfo.h header file, +lib/io.netty-netty-codec-4.1.121.Final.jar contains the dnsinfo.h header file, that provides a way to retrieve the system DNS configuration on MacOS. This private header is also used by Apple's open source mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/). @@ -561,13 +561,13 @@ This product bundles Google Protocol Buffers, which is available under a "3-clau license. Bundled as - - lib/com.google.protobuf-protobuf-java-3.21.9.jar -Source available at https://github.com/google/protobuf/tree/v3.21.9 + - lib/com.google.protobuf-protobuf-java-3.25.5.jar +Source available at https://github.com/google/protobuf/tree/v3.25.5 For details, see deps/protobuf-3.14.0/LICENSE. Bundled as - - lib/com.google.protobuf-protobuf-java-util-3.21.9.jar -Source available at https://github.com/protocolbuffers/protobuf/tree/v3.21.9 + - lib/com.google.protobuf-protobuf-java-util-3.25.5.jar +Source available at https://github.com/protocolbuffers/protobuf/tree/v3.25.5 For details, see deps/protobuf-3.12.0/LICENSE. ------------------------------------------------------------------------------------ This product bundles Simple Logging Facade for Java, which is available under a diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt index fea213f758c..a6fa25a9cbe 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt @@ -211,38 +211,38 @@ Apache Software License, Version 2. - lib/com.google.guava-guava-32.0.1-jre.jar [4] - lib/com.google.guava-failureaccess-1.0.1.jar [4] - lib/com.google.guava-listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar [4] -- lib/commons-cli-commons-cli-1.2.jar [5] -- lib/commons-codec-commons-codec-1.6.jar [6] +- lib/commons-cli-commons-cli-1.9.0.jar [5] +- lib/commons-codec-commons-codec-1.18.0.jar [6] - lib/commons-configuration-commons-configuration-1.10.jar [7] -- lib/commons-io-commons-io-2.7.jar [8] +- lib/commons-io-commons-io-2.19.0.jar [8] - lib/commons-lang-commons-lang-2.6.jar [9] - lib/commons-logging-commons-logging-1.1.1.jar [10] -- lib/io.netty-netty-buffer-4.1.111.Final.jar [11] -- lib/io.netty-netty-codec-4.1.111.Final.jar [11] -- lib/io.netty-netty-codec-dns-4.1.111.Final.jar [11] -- lib/io.netty-netty-codec-http-4.1.111.Final.jar [11] -- lib/io.netty-netty-codec-http2-4.1.111.Final.jar [11] -- lib/io.netty-netty-codec-socks-4.1.111.Final.jar [11] -- lib/io.netty-netty-common-4.1.111.Final.jar [11] -- lib/io.netty-netty-handler-4.1.111.Final.jar [11] -- lib/io.netty-netty-handler-proxy-4.1.111.Final.jar [11] -- lib/io.netty-netty-resolver-4.1.111.Final.jar [11] -- lib/io.netty-netty-resolver-dns-4.1.111.Final.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-linux-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-osx-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-osx-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-windows-x86_64.jar [11] -- lib/io.netty-netty-tcnative-classes-2.0.65.Final.jar [11] -- lib/io.netty-netty-transport-4.1.111.Final.jar [11] -- lib/io.netty-netty-transport-classes-epoll-4.1.111.Final.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.111.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.111.Final-linux-x86_64.jar [11] -- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-x86_64.jar [11] -- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-aarch_64.jar [11] -- lib/io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.25.Final.jar [11] -- lib/io.netty-netty-transport-native-unix-common-4.1.111.Final.jar [11] +- lib/io.netty-netty-buffer-4.1.121.Final.jar [11] +- lib/io.netty-netty-codec-4.1.121.Final.jar [11] +- lib/io.netty-netty-codec-dns-4.1.121.Final.jar [11] +- lib/io.netty-netty-codec-http-4.1.121.Final.jar [11] +- lib/io.netty-netty-codec-http2-4.1.121.Final.jar [11] +- lib/io.netty-netty-codec-socks-4.1.121.Final.jar [11] +- lib/io.netty-netty-common-4.1.121.Final.jar [11] +- lib/io.netty-netty-handler-4.1.121.Final.jar [11] +- lib/io.netty-netty-handler-proxy-4.1.121.Final.jar [11] +- lib/io.netty-netty-resolver-4.1.121.Final.jar [11] +- lib/io.netty-netty-resolver-dns-4.1.121.Final.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-windows-x86_64.jar [11] +- lib/io.netty-netty-tcnative-classes-2.0.70.Final.jar [11] +- lib/io.netty-netty-transport-4.1.121.Final.jar [11] +- lib/io.netty-netty-transport-classes-epoll-4.1.121.Final.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.121.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.121.Final-linux-x86_64.jar [11] +- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar [11] +- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar [11] +- lib/io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.26.Final.jar [11] +- lib/io.netty-netty-transport-native-unix-common-4.1.121.Final.jar [11] - lib/io.prometheus-simpleclient-0.15.0.jar [12] - lib/io.prometheus-simpleclient_common-0.15.0.jar [12] - lib/io.prometheus-simpleclient_hotspot-0.15.0.jar [12] @@ -251,31 +251,31 @@ Apache Software License, Version 2. - lib/io.prometheus-simpleclient_tracer_common-0.15.0.jar [12] - lib/io.prometheus-simpleclient_tracer_otel-0.15.0.jar [12] - lib/io.prometheus-simpleclient_tracer_otel_agent-0.15.0.jar [12] -- lib/io.vertx-vertx-auth-common-4.5.7.jar [13] -- lib/io.vertx-vertx-bridge-common-4.5.7.jar [14] -- lib/io.vertx-vertx-core-4.5.7.jar [15] -- lib/io.vertx-vertx-web-4.5.7.jar [16] -- lib/io.vertx-vertx-web-common-4.5.7.jar [16] +- lib/io.vertx-vertx-auth-common-4.5.11.jar [13] +- lib/io.vertx-vertx-bridge-common-4.5.11.jar [14] +- lib/io.vertx-vertx-core-4.5.11.jar [15] +- lib/io.vertx-vertx-web-4.5.11.jar [16] +- lib/io.vertx-vertx-web-common-4.5.11.jar [16] - lib/org.apache.logging.log4j-log4j-api-2.18.0.jar [17] - lib/org.apache.logging.log4j-log4j-core-2.18.0.jar [17] - lib/org.apache.logging.log4j-log4j-slf4j-impl-2.18.0.jar [17] - lib/org.apache.commons-commons-collections4-4.1.jar [19] -- lib/org.apache.commons-commons-lang3-3.6.jar [20] -- lib/org.apache.zookeeper-zookeeper-3.8.3.jar [21] -- lib/org.apache.zookeeper-zookeeper-jute-3.8.3.jar [21] -- lib/org.apache.zookeeper-zookeeper-3.8.3-tests.jar [21] -- lib/org.eclipse.jetty-jetty-http-9.4.53.v20231009.jar [22] -- lib/org.eclipse.jetty-jetty-io-9.4.53.v20231009.jar [22] -- lib/org.eclipse.jetty-jetty-security-9.4.53.v20231009.jar [22] -- lib/org.eclipse.jetty-jetty-server-9.4.53.v20231009.jar [22] -- lib/org.eclipse.jetty-jetty-servlet-9.4.53.v20231009.jar [22] -- lib/org.eclipse.jetty-jetty-util-9.4.53.v20231009.jar [22] -- lib/org.eclipse.jetty-jetty-util-ajax-9.4.53.v20231009.jar [22] +- lib/org.apache.commons-commons-lang3-3.17.0.jar [57] +- lib/org.apache.zookeeper-zookeeper-3.9.3.jar [21] +- lib/org.apache.zookeeper-zookeeper-jute-3.9.3.jar [21] +- lib/org.apache.zookeeper-zookeeper-3.9.3-tests.jar [21] +- lib/org.eclipse.jetty-jetty-http-9.4.57.v20241219.jar [22] +- lib/org.eclipse.jetty-jetty-io-9.4.57.v20241219.jar [22] +- lib/org.eclipse.jetty-jetty-security-9.4.57.v20241219.jar [22] +- lib/org.eclipse.jetty-jetty-server-9.4.57.v20241219.jar [22] +- lib/org.eclipse.jetty-jetty-servlet-9.4.57.v20241219.jar [22] +- lib/org.eclipse.jetty-jetty-util-9.4.57.v20241219.jar [22] +- lib/org.eclipse.jetty-jetty-util-ajax-9.4.57.v20241219.jar [22] - lib/org.rocksdb-rocksdbjni-7.9.2.jar [23] - lib/com.beust-jcommander-1.82.jar [24] - lib/com.yahoo.datasketches-memory-0.8.3.jar [25] - lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25] -- lib/net.jpountz.lz4-lz4-1.3.0.jar [26] +- lib/at.yawk.lz4-lz4-java-1.10.2.jar [26] - lib/com.google.api.grpc-proto-google-common-protos-2.9.0.jar [28] - lib/com.google.code.gson-gson-2.9.0.jar [29] - lib/io.opencensus-opencensus-api-0.28.0.jar [30] @@ -296,9 +296,9 @@ Apache Software License, Version 2. - lib/io.grpc-grpc-testing-1.54.1.jar [33] - lib/io.grpc-grpc-xds-1.54.1.jar [33] - lib/io.grpc-grpc-rls-1.54.1.jar[33] -- lib/org.apache.curator-curator-client-5.1.0.jar [34] -- lib/org.apache.curator-curator-framework-5.1.0.jar [34] -- lib/org.apache.curator-curator-recipes-5.1.0.jar [34] +- lib/org.apache.curator-curator-client-5.7.1.jar [34] +- lib/org.apache.curator-curator-framework-5.7.1.jar [34] +- lib/org.apache.curator-curator-recipes-5.7.1.jar [34] - lib/com.google.errorprone-error_prone_annotations-2.9.0.jar [36] - lib/org.apache.yetus-audience-annotations-0.12.0.jar [37] - lib/org.jctools-jctools-core-2.1.2.jar [38] @@ -322,13 +322,13 @@ Apache Software License, Version 2. [2] Source available at https://github.com/FasterXML/jackson-core/tree/jackson-core-2.17.1 [3] Source available at https://github.com/FasterXML/jackson-databind/tree/jackson-databind-2.17.1 [4] Source available at https://github.com/google/guava/tree/v32.0.1 -[5] Source available at https://github.com/apache/commons-cli/tree/cli-1.2 -[6] Source available at https://github.com/apache/commons-codec/tree/commons-codec-1.6-RC2 +[5] Source available at https://github.com/apache/commons-cli/tree/rel/commons-cli-1.9.0 +[6] Source available at https://github.com/apache/commons-codec/tree/rel/commons-codec-1.18.0 [7] Source available at https://github.com/apache/commons-configuration/tree/CONFIGURATION_1_10 -[8] Source available at https://github.com/apache/commons-io/tree/rel/commons-io-2.7 +[8] Source available at https://github.com/apache/commons-io/tree/rel/commons-io-2.19.0 [9] Source available at https://github.com/apache/commons-lang/tree/LANG_2_6 [10] Source available at https://github.com/apache/commons-logging/tree/commons-logging-1.1.1 -[11] Source available at https://github.com/netty/netty/tree/netty-4.1.111.Final +[11] Source available at https://github.com/netty/netty/tree/netty-4.1.121.Final [12] Source available at https://github.com/prometheus/client_java/tree/parent-0.15.0 [13] Source available at https://github.com/vert-x3/vertx-auth/tree/4.3.2 [14] Source available at https://github.com/vert-x3/vertx-bridge-common/tree/4.3.2 @@ -342,12 +342,12 @@ Apache Software License, Version 2. [23] Source available at https://github.com/facebook/rocksdb/tree/v6.29.4.1 [24] Source available at https://github.com/cbeust/jcommander/tree/1.82 [25] Source available at https://github.com/DataSketches/sketches-core/tree/sketches-0.8.3 -[26] Source available at https://github.com/lz4/lz4-java/tree/1.3.0 +[26] Source available at https://github.com/yawkat/lz4-java/tree/v1.10.2 [28] Source available at https://github.com/googleapis/java-common-protos/tree/v2.17.0 [29] Source available at https://github.com/google/gson/tree/gson-parent-2.10.1 [30] Source available at https://github.com/census-instrumentation/opencensus-java/tree/v0.28.0 [33] Source available at https://github.com/grpc/grpc-java/tree/v1.56.0 -[34] Source available at https://github.com/apache/curator/releases/tag/apache.curator-5.1.0 +[34] Source available at https://github.com/apache/curator/releases/tag/apache.curator-5.7.1 [36] Source available at https://github.com/google/error-prone/tree/v2.9.0 [37] Source available at https://github.com/apache/yetus/tree/rel/0.12.0 [38] Source available at https://github.com/JCTools/JCTools/tree/v2.1.2 @@ -365,11 +365,12 @@ Apache Software License, Version 2. [50] Source available at https://github.com/xerial/snappy-java/releases/tag/v1.1.10.5 [51] Source available at https://github.com/ReactiveX/RxJava/tree/v3.0.1 [52] Source available at https://github.com/carrotsearch/hppc/tree/0.9.1 +[57] Source available at https://github.com/apache/commons-lang/tree/rel/commons-lang-3.17.0 ------------------------------------------------------------------------------------ -lib/io.netty-netty-codec-4.1.111.Final.jar bundles some 3rd party dependencies +lib/io.netty-netty-codec-4.1.121.Final.jar bundles some 3rd party dependencies -lib/io.netty-netty-codec-4.1.111.Final.jar contains the extensions to Java Collections Framework which has +lib/io.netty-netty-codec-4.1.121.Final.jar contains the extensions to Java Collections Framework which has been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * LICENSE: @@ -378,7 +379,7 @@ been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified version of Robert Harder's Public Domain +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified version of Robert Harder's Public Domain Base64 Encoder and Decoder, which can be obtained at: * LICENSE: @@ -386,7 +387,7 @@ Base64 Encoder and Decoder, which can be obtained at: * HOMEPAGE: * http://iharder.sourceforge.net/current/java/base64/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'Webbit', an event based +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'Webbit', an event based WebSocket and HTTP server, which can be obtained at: * LICENSE: @@ -394,7 +395,7 @@ WebSocket and HTTP server, which can be obtained at: * HOMEPAGE: * https://github.com/joewalnes/webbit -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'SLF4J', a simple logging +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'SLF4J', a simple logging facade for Java, which can be obtained at: * LICENSE: @@ -402,7 +403,7 @@ facade for Java, which can be obtained at: * HOMEPAGE: * http://www.slf4j.org/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'Apache Harmony', an open source +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'Apache Harmony', an open source Java SE, which can be obtained at: * NOTICE: @@ -412,7 +413,7 @@ Java SE, which can be obtained at: * HOMEPAGE: * http://archive.apache.org/dist/harmony/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression and decompression library written by Matthew J. Francis. It can be obtained at: * LICENSE: @@ -420,7 +421,7 @@ and decompression library written by Matthew J. Francis. It can be obtained at: * HOMEPAGE: * https://code.google.com/p/jbzip2/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct the suffix array and the Burrows-Wheeler transformed string for any input string of a constant-size alphabet written by Yuta Mori. It can be obtained at: @@ -429,7 +430,7 @@ a constant-size alphabet written by Yuta Mori. It can be obtained at: * HOMEPAGE: * https://github.com/y-256/libdivsufsort -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, which can be obtained at: * LICENSE: @@ -437,7 +438,7 @@ Java Concurrency Tools for the JVM, which can be obtained at: * HOMEPAGE: * https://github.com/JCTools/JCTools -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in pure Java, which can be obtained at: * LICENSE: @@ -445,7 +446,7 @@ pure Java, which can be obtained at: * HOMEPAGE: * http://www.jcraft.com/jzlib/ -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * LICENSE: @@ -453,15 +454,15 @@ decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * HOMEPAGE: * https://github.com/ning/compress -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'lz4', a LZ4 Java compression +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'lz4-java', a LZ4 Java compression and decompression library written by Adrien Grand. It can be obtained at: * LICENSE: * deps/netty/LICENSE.lz4.txt (Apache License 2.0) * HOMEPAGE: - * https://github.com/jpountz/lz4-java + * https://github.com/yawkat/lz4-java -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'lzma-java', a LZMA Java compression +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'lzma-java', a LZMA Java compression and decompression library, which can be obtained at: * LICENSE: @@ -469,7 +470,7 @@ and decompression library, which can be obtained at: * HOMEPAGE: * https://github.com/jponge/lzma-java -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression and decompression library written by William Kinney. It can be obtained at: * LICENSE: @@ -477,7 +478,7 @@ and decompression library written by William Kinney. It can be obtained at: * HOMEPAGE: * https://code.google.com/p/jfastlz/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data interchange format, which can be obtained at: * LICENSE: @@ -485,7 +486,7 @@ Google's data interchange format, which can be obtained at: * HOMEPAGE: * https://github.com/google/protobuf -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate a temporary self-signed X.509 certificate when the JVM does not provide the equivalent functionality. It can be obtained at: @@ -494,7 +495,7 @@ equivalent functionality. It can be obtained at: * HOMEPAGE: * http://www.bouncycastle.org/ -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Snappy', a compression library produced +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Snappy', a compression library produced by Google Inc, which can be obtained at: * LICENSE: @@ -502,7 +503,7 @@ by Google Inc, which can be obtained at: * HOMEPAGE: * https://github.com/google/snappy -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java serialization API, which can be obtained at: * LICENSE: @@ -510,7 +511,7 @@ serialization API, which can be obtained at: * HOMEPAGE: * https://github.com/jboss-remoting/jboss-marshalling -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Caliper', Google's micro- +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Caliper', Google's micro- benchmarking framework, which can be obtained at: * LICENSE: @@ -518,7 +519,7 @@ benchmarking framework, which can be obtained at: * HOMEPAGE: * https://github.com/google/caliper -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Apache Commons Logging', a logging +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Apache Commons Logging', a logging framework, which can be obtained at: * LICENSE: @@ -526,7 +527,7 @@ framework, which can be obtained at: * HOMEPAGE: * http://commons.apache.org/logging/ -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Apache Log4J', a logging framework, which +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Apache Log4J', a logging framework, which can be obtained at: * LICENSE: @@ -534,7 +535,7 @@ can be obtained at: * HOMEPAGE: * http://logging.apache.org/log4j/ -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Aalto XML', an ultra-high performance +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Aalto XML', an ultra-high performance non-blocking XML processor, which can be obtained at: * LICENSE: @@ -542,7 +543,7 @@ non-blocking XML processor, which can be obtained at: * HOMEPAGE: * http://wiki.fasterxml.com/AaltoHome -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * LICENSE: @@ -550,7 +551,7 @@ the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * HOMEPAGE: * https://github.com/twitter/hpack -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * LICENSE: @@ -558,7 +559,7 @@ the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * HOMEPAGE: * https://github.com/python-hyper/hpack/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at: * LICENSE: @@ -566,7 +567,7 @@ the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at * HOMEPAGE: * https://github.com/nghttp2/nghttp2/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library provides utilities for the java.lang API, which can be obtained at: * LICENSE: @@ -575,7 +576,7 @@ provides utilities for the java.lang API, which can be obtained at: * https://commons.apache.org/proper/commons-lang/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', +lib/io.netty-netty-codec-4.1.121.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. * LICENSE: @@ -583,7 +584,7 @@ that provides an easy way to ensure a user has everything necessary to run the M * HOMEPAGE: * https://github.com/takari/maven-wrapper -lib/io.netty-netty-codec-4.1.111.Final.jar contains the dnsinfo.h header file, +lib/io.netty-netty-codec-4.1.121.Final.jar contains the dnsinfo.h header file, that provides a way to retrieve the system DNS configuration on MacOS. This private header is also used by Apple's open source mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/). @@ -629,13 +630,13 @@ This product bundles Google Protocol Buffers, which is available under a "3-clau license. Bundled as - - lib/com.google.protobuf-protobuf-java-3.21.9.jar -Source available at https://github.com/google/protobuf/tree/v3.21.9 + - lib/com.google.protobuf-protobuf-java-3.25.5.jar +Source available at https://github.com/google/protobuf/tree/v3.25.5 For details, see deps/protobuf-3.14.0/LICENSE. Bundled as - - lib/com.google.protobuf-protobuf-java-util-3.21.9.jar -Source available at https://github.com/protocolbuffers/protobuf/tree/v3.21.9 + - lib/com.google.protobuf-protobuf-java-util-3.25.5.jar +Source available at https://github.com/protocolbuffers/protobuf/tree/v3.25.5 For details, see deps/protobuf-3.12.0/LICENSE. ------------------------------------------------------------------------------------ This product bundles the JCP Standard Java Servlet API, which is available under a diff --git a/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt index af368d1d948..168d4a1ff59 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt @@ -23,31 +23,31 @@ LongAdder), which was released with the following comments: http://creativecommons.org/publicdomain/zero/1.0/ ------------------------------------------------------------------------------------ -- lib/io.netty-netty-buffer-4.1.111.Final.jar -- lib/io.netty-netty-codec-4.1.111.Final.jar -- lib/io.netty-netty-codec-dns-4.1.111.Final.jar -- lib/io.netty-netty-codec-http-4.1.111.Final.jar -- lib/io.netty-netty-codec-http2-4.1.111.Final.jar -- lib/io.netty-netty-codec-socks-4.1.111.Final.jar -- lib/io.netty-netty-common-4.1.111.Final.jar -- lib/io.netty-netty-handler-4.1.111.Final.jar -- lib/io.netty-netty-handler-proxy-4.1.111.Final.jar -- lib/io.netty-netty-resolver-4.1.111.Final.jar -- lib/io.netty-netty-resolver-dns-4.1.111.Final.jar -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final.jar -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-linux-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-osx-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-osx-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-windows-x86_64.jar [11] -- lib/io.netty-netty-tcnative-classes-2.0.65.Final.jar -- lib/io.netty-netty-transport-4.1.111.Final.jar -- lib/io.netty-netty-transport-classes-epoll-4.1.111.Final.jar -- lib/io.netty-netty-transport-native-epoll-4.1.111.Final-linux-aarch_64.jar -- lib/io.netty-netty-transport-native-epoll-4.1.111.Final-linux-x86_64.jar -- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-x86_64.jar -- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-aarch_64.jar -- lib/io.netty-netty-transport-native-unix-common-4.1.111.Final.jar +- lib/io.netty-netty-buffer-4.1.121.Final.jar +- lib/io.netty-netty-codec-4.1.121.Final.jar +- lib/io.netty-netty-codec-dns-4.1.121.Final.jar +- lib/io.netty-netty-codec-http-4.1.121.Final.jar +- lib/io.netty-netty-codec-http2-4.1.121.Final.jar +- lib/io.netty-netty-codec-socks-4.1.121.Final.jar +- lib/io.netty-netty-common-4.1.121.Final.jar +- lib/io.netty-netty-handler-4.1.121.Final.jar +- lib/io.netty-netty-handler-proxy-4.1.121.Final.jar +- lib/io.netty-netty-resolver-4.1.121.Final.jar +- lib/io.netty-netty-resolver-dns-4.1.121.Final.jar +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final.jar +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-windows-x86_64.jar [11] +- lib/io.netty-netty-tcnative-classes-2.0.70.Final.jar +- lib/io.netty-netty-transport-4.1.121.Final.jar +- lib/io.netty-netty-transport-classes-epoll-4.1.121.Final.jar +- lib/io.netty-netty-transport-native-epoll-4.1.121.Final-linux-aarch_64.jar +- lib/io.netty-netty-transport-native-epoll-4.1.121.Final-linux-x86_64.jar +- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar +- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar +- lib/io.netty-netty-transport-native-unix-common-4.1.121.Final.jar The Netty Project @@ -93,13 +93,13 @@ SoundCloud Ltd. (http://soundcloud.com/). This product includes software developed as part of the Ocelli project by Netflix Inc. (https://github.com/Netflix/ocelli/). ------------------------------------------------------------------------------------ -- lib/org.eclipse.jetty-jetty-http-9.4.53.v20231009.jar -- lib/org.eclipse.jetty-jetty-io-9.4.53.v20231009.jar -- lib/org.eclipse.jetty-jetty-security-9.4.53.v20231009.jar -- lib/org.eclipse.jetty-jetty-server-9.4.53.v20231009.jar -- lib/org.eclipse.jetty-jetty-servlet-9.4.53.v20231009.jar -- lib/org.eclipse.jetty-jetty-util-9.4.53.v20231009.jar -- lib/org.eclipse.jetty-jetty-util-ajax-9.4.53.v20231009.jar +- lib/org.eclipse.jetty-jetty-http-9.4.57.v20241219.jar +- lib/org.eclipse.jetty-jetty-io-9.4.57.v20241219.jar +- lib/org.eclipse.jetty-jetty-security-9.4.57.v20241219.jar +- lib/org.eclipse.jetty-jetty-server-9.4.57.v20241219.jar +- lib/org.eclipse.jetty-jetty-servlet-9.4.57.v20241219.jar +- lib/org.eclipse.jetty-jetty-util-9.4.57.v20241219.jar +- lib/org.eclipse.jetty-jetty-util-ajax-9.4.57.v20241219.jar ============================================================== Jetty Web Container @@ -121,7 +121,7 @@ Jetty is dual licensed under both Jetty may be distributed under either license. -lib/org.eclipse.jetty-jetty-util-9.4.53.v20231009.jar bundles UnixCrypt +lib/org.eclipse.jetty-jetty-util-9.4.57.v20241219.jar bundles UnixCrypt The UnixCrypt.java code implements the one way cryptography used by Unix systems for simple password protection. Copyright 1996 Aki Yoshida, diff --git a/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt index 88eaaf8a5d0..47d5c6a522e 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt +++ b/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt @@ -5,25 +5,25 @@ This product includes software developed at The Apache Software Foundation (http://www.apache.org/). ------------------------------------------------------------------------------------ -- lib/io.netty-netty-buffer-4.1.111.Final.jar -- lib/io.netty-netty-codec-4.1.111.Final.jar -- lib/io.netty-netty-common-4.1.111.Final.jar -- lib/io.netty-netty-handler-4.1.111.Final.jar -- lib/io.netty-netty-resolver-4.1.111.Final.jar -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final.jar -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-linux-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-osx-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-osx-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-windows-x86_64.jar [11] -- lib/io.netty-netty-tcnative-classes-2.0.65.Final.jar -- lib/io.netty-netty-transport-4.1.111.Final.jar -- lib/io.netty-netty-transport-classes-epoll-4.1.111.Final.jar -- lib/io.netty-netty-transport-native-epoll-4.1.111.Final-linux-aarch_64.jar -- lib/io.netty-netty-transport-native-epoll-4.1.111.Final-linux-x86_64.jar -- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-x86_64.jar -- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-aarch_64.jar -- lib/io.netty-netty-transport-native-unix-common-4.1.111.Final.jar +- lib/io.netty-netty-buffer-4.1.121.Final.jar +- lib/io.netty-netty-codec-4.1.121.Final.jar +- lib/io.netty-netty-common-4.1.121.Final.jar +- lib/io.netty-netty-handler-4.1.121.Final.jar +- lib/io.netty-netty-resolver-4.1.121.Final.jar +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final.jar +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-windows-x86_64.jar [11] +- lib/io.netty-netty-tcnative-classes-2.0.70.Final.jar +- lib/io.netty-netty-transport-4.1.121.Final.jar +- lib/io.netty-netty-transport-classes-epoll-4.1.121.Final.jar +- lib/io.netty-netty-transport-native-epoll-4.1.121.Final-linux-aarch_64.jar +- lib/io.netty-netty-transport-native-epoll-4.1.121.Final-linux-x86_64.jar +- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar +- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar +- lib/io.netty-netty-transport-native-unix-common-4.1.121.Final.jar The Netty Project diff --git a/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt index 2873f19dedb..210718ee9fe 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt @@ -5,31 +5,31 @@ This product includes software developed at The Apache Software Foundation (http://www.apache.org/). ------------------------------------------------------------------------------------ -- lib/io.netty-netty-buffer-4.1.111.Final.jar -- lib/io.netty-netty-codec-4.1.111.Final.jar -- lib/io.netty-netty-codec-dns-4.1.111.Final.jar -- lib/io.netty-netty-codec-http-4.1.111.Final.jar -- lib/io.netty-netty-codec-http2-4.1.111.Final.jar -- lib/io.netty-netty-codec-socks-4.1.111.Final.jar -- lib/io.netty-netty-common-4.1.111.Final.jar -- lib/io.netty-netty-handler-4.1.111.Final.jar -- lib/io.netty-netty-handler-proxy-4.1.111.Final.jar -- lib/io.netty-netty-resolver-4.1.111.Final.jar -- lib/io.netty-netty-resolver-dns-4.1.111.Final.jar -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final.jar -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-linux-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-osx-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-osx-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-windows-x86_64.jar [11] -- lib/io.netty-netty-tcnative-classes-2.0.65.Final.jar -- lib/io.netty-netty-transport-4.1.111.Final.jar -- lib/io.netty-netty-transport-classes-epoll-4.1.111.Final.jar -- lib/io.netty-netty-transport-native-epoll-4.1.111.Final-linux-aarch_64.jar -- lib/io.netty-netty-transport-native-epoll-4.1.111.Final-linux-x86_64.jar -- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-x86_64.jar -- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-aarch_64.jar -- lib/io.netty-netty-transport-native-unix-common-4.1.111.Final.jar +- lib/io.netty-netty-buffer-4.1.121.Final.jar +- lib/io.netty-netty-codec-4.1.121.Final.jar +- lib/io.netty-netty-codec-dns-4.1.121.Final.jar +- lib/io.netty-netty-codec-http-4.1.121.Final.jar +- lib/io.netty-netty-codec-http2-4.1.121.Final.jar +- lib/io.netty-netty-codec-socks-4.1.121.Final.jar +- lib/io.netty-netty-common-4.1.121.Final.jar +- lib/io.netty-netty-handler-4.1.121.Final.jar +- lib/io.netty-netty-handler-proxy-4.1.121.Final.jar +- lib/io.netty-netty-resolver-4.1.121.Final.jar +- lib/io.netty-netty-resolver-dns-4.1.121.Final.jar +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final.jar +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-windows-x86_64.jar [11] +- lib/io.netty-netty-tcnative-classes-2.0.70.Final.jar +- lib/io.netty-netty-transport-4.1.121.Final.jar +- lib/io.netty-netty-transport-classes-epoll-4.1.121.Final.jar +- lib/io.netty-netty-transport-native-epoll-4.1.121.Final-linux-aarch_64.jar +- lib/io.netty-netty-transport-native-epoll-4.1.121.Final-linux-x86_64.jar +- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar +- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar +- lib/io.netty-netty-transport-native-unix-common-4.1.121.Final.jar The Netty Project @@ -75,13 +75,13 @@ SoundCloud Ltd. (http://soundcloud.com/). This product includes software developed as part of the Ocelli project by Netflix Inc. (https://github.com/Netflix/ocelli/). ------------------------------------------------------------------------------------ -- lib/org.eclipse.jetty-jetty-http-9.4.53.v20231009.jar -- lib/org.eclipse.jetty-jetty-io-9.4.53.v20231009.jar -- lib/org.eclipse.jetty-jetty-security-9.4.53.v20231009.jar -- lib/org.eclipse.jetty-jetty-server-9.4.53.v20231009.jar -- lib/org.eclipse.jetty-jetty-servlet-9.4.53.v20231009.jar -- lib/org.eclipse.jetty-jetty-util-9.4.53.v20231009.jar -- lib/org.eclipse.jetty-jetty-util-ajax-9.4.53.v20231009.jar +- lib/org.eclipse.jetty-jetty-http-9.4.57.v20241219.jar +- lib/org.eclipse.jetty-jetty-io-9.4.57.v20241219.jar +- lib/org.eclipse.jetty-jetty-security-9.4.57.v20241219.jar +- lib/org.eclipse.jetty-jetty-server-9.4.57.v20241219.jar +- lib/org.eclipse.jetty-jetty-servlet-9.4.57.v20241219.jar +- lib/org.eclipse.jetty-jetty-util-9.4.57.v20241219.jar +- lib/org.eclipse.jetty-jetty-util-ajax-9.4.57.v20241219.jar ============================================================== Jetty Web Container @@ -103,7 +103,7 @@ Jetty is dual licensed under both Jetty may be distributed under either license. -lib/org.eclipse.jetty-jetty-util-9.4.53.v20231009.jar bundles UnixCrypt +lib/org.eclipse.jetty-jetty-util-9.4.57.v20241219.jar bundles UnixCrypt The UnixCrypt.java code implements the one way cryptography used by Unix systems for simple password protection. Copyright 1996 Aki Yoshida, diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookKeeperServerStats.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookKeeperServerStats.java index d4657d20362..361861cda14 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookKeeperServerStats.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookKeeperServerStats.java @@ -150,6 +150,7 @@ public interface BookKeeperServerStats { String THREAD_RUNTIME = "THREAD_RUNTIME"; String MAJOR_COMPACTION_COUNT = "MAJOR_COMPACTION_TOTAL"; String MINOR_COMPACTION_COUNT = "MINOR_COMPACTION_TOTAL"; + String ENTRY_LOCATION_COMPACTION_COUNT = "ENTRY_LOCATION_COMPACTION_TOTAL"; String ACTIVE_LEDGER_COUNT = "ACTIVE_LEDGER_TOTAL"; String DELETED_LEDGER_COUNT = "DELETED_LEDGER_TOTAL"; diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieException.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieException.java index 2b85961cf4b..d88673d0076 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieException.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieException.java @@ -69,6 +69,8 @@ public static BookieException create(int code) { return new MetadataStoreException(); case Code.UnknownBookieIdException: return new UnknownBookieIdException(); + case Code.LedgerFencedAndDeletedException: + return new LedgerFencedAndDeletedException(); case Code.DataUnknownException: return new DataUnknownException(); default: @@ -95,6 +97,7 @@ public interface Code { int CookieExistsException = -109; int EntryLogMetadataMapException = -110; int DataUnknownException = -111; + int LedgerFencedAndDeletedException = -112; } public int getCode() { @@ -199,6 +202,15 @@ public LedgerFencedException() { } } + /** + * Signals that a ledger has been fenced in a bookie. No more entries can be appended to that ledger. + */ + public static class LedgerFencedAndDeletedException extends BookieException { + public LedgerFencedAndDeletedException() { + super(Code.LedgerFencedException); + } + } + /** * Signals that a ledger's operation has been rejected by an internal component because of the resource saturation. */ diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java index d0874ffa165..968977ecdc3 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java @@ -527,7 +527,7 @@ public void process(int journalVersion, long offset, ByteBuffer recBuff) throws masterKeyCache.put(ledgerId, masterKey); // Force to re-insert the master key in ledger storage - handles.getHandle(ledgerId, masterKey); + handles.getHandle(ledgerId, masterKey, true); } else { throw new IOException("Invalid journal. Contains journalKey " + " but layout version (" + journalVersion @@ -539,7 +539,7 @@ public void process(int journalVersion, long offset, ByteBuffer recBuff) throws if (key == null) { key = ledgerStorage.readMasterKey(ledgerId); } - LedgerDescriptor handle = handles.getHandle(ledgerId, key); + LedgerDescriptor handle = handles.getHandle(ledgerId, key, true); handle.setFenced(); } else { throw new IOException("Invalid journal. Contains fenceKey " @@ -557,7 +557,7 @@ public void process(int journalVersion, long offset, ByteBuffer recBuff) throws if (key == null) { key = ledgerStorage.readMasterKey(ledgerId); } - LedgerDescriptor handle = handles.getHandle(ledgerId, key); + LedgerDescriptor handle = handles.getHandle(ledgerId, key, true); handle.setExplicitLac(explicitLacBuf); } else { throw new IOException("Invalid journal. Contains explicitLAC " + " but layout version (" @@ -580,7 +580,7 @@ public void process(int journalVersion, long offset, ByteBuffer recBuff) throws if (key == null) { key = ledgerStorage.readMasterKey(ledgerId); } - LedgerDescriptor handle = handles.getHandle(ledgerId, key); + LedgerDescriptor handle = handles.getHandle(ledgerId, key, true); recBuff.rewind(); handle.addEntry(Unpooled.wrappedBuffer(recBuff)); @@ -902,7 +902,7 @@ LedgerDescriptor getLedgerForEntry(ByteBuf entry, final byte[] masterKey) throws IOException, BookieException { final long ledgerId = entry.getLong(entry.readerIndex()); - return handles.getHandle(ledgerId, masterKey); + return handles.getHandle(ledgerId, masterKey, false); } private Journal getJournal(long ledgerId) { @@ -1011,7 +1011,7 @@ public void setExplicitLac(ByteBuf entry, WriteCallback writeCallback, Object ct ByteBuf explicitLACEntry = null; try { long ledgerId = entry.getLong(entry.readerIndex()); - LedgerDescriptor handle = handles.getHandle(ledgerId, masterKey); + LedgerDescriptor handle = handles.getHandle(ledgerId, masterKey, false); synchronized (handle) { entry.markReaderIndex(); handle.setExplicitLac(entry); @@ -1100,7 +1100,7 @@ public void addEntry(ByteBuf entry, boolean ackBeforeSync, WriteCallback cb, Obj */ public CompletableFuture fenceLedger(long ledgerId, byte[] masterKey) throws IOException, BookieException { - LedgerDescriptor handle = handles.getHandle(ledgerId, masterKey); + LedgerDescriptor handle = handles.getHandle(ledgerId, masterKey, false); return handle.fenceAndLogInJournal(getJournal(ledgerId)); } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieResources.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieResources.java index c9b71b9968d..755efd5be02 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieResources.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieResources.java @@ -70,6 +70,7 @@ public static ByteBufAllocatorWithOomHandler createAllocator(ServerConfiguration .poolingConcurrency(conf.getAllocatorPoolingConcurrency()) .outOfMemoryPolicy(conf.getAllocatorOutOfMemoryPolicy()) .leakDetectionPolicy(conf.getAllocatorLeakDetectionPolicy()) + .exitOnOutOfMemory(conf.exitOnOutOfMemory()) .build(); } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java index 944b97e79dc..8cf34f24898 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java @@ -488,7 +488,7 @@ public RecoverCmd() { opts.addOption("sk", "skipOpenLedgers", false, "Skip recovering open ledgers"); opts.addOption("d", "deleteCookie", false, "Delete cookie node for the bookie."); opts.addOption("sku", "skipUnrecoverableLedgers", false, "Skip unrecoverable ledgers."); - opts.addOption("rate", "replicationRate", false, "Replication rate by bytes"); + opts.addOption("rate", "replicationRate", true, "Replication rate by bytes"); } @Override diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStateManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStateManager.java index 620a1c2aaa4..070897b9473 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStateManager.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStateManager.java @@ -165,7 +165,7 @@ private boolean isRegistrationManagerDisabled() { @Override public void initState(){ if (forceReadOnly.get()) { - this.bookieStatus.setToReadOnlyMode(); + this.bookieStatus.setToReadOnlyMode(false); } else if (conf.isPersistBookieStatusEnabled()) { this.bookieStatus.readFromDirectories(statusDirs); } @@ -256,22 +256,22 @@ public Void call() throws IOException { } @Override - public Future transitionToWritableMode() { + public Future transitionToWritableMode(boolean isManuallyModify) { return stateService.submit(new Callable() { @Override public Void call() throws Exception{ - doTransitionToWritableMode(); + doTransitionToWritableMode(isManuallyModify); return null; } }); } @Override - public Future transitionToReadOnlyMode() { + public Future transitionToReadOnlyMode(boolean isManuallyModify) { return stateService.submit(new Callable() { @Override public Void call() throws Exception{ - doTransitionToReadOnlyMode(); + doTransitionToReadOnlyMode(isManuallyModify); return null; } }); @@ -297,11 +297,20 @@ private void doRegisterBookie(boolean isReadOnly) throws IOException { } } - @VisibleForTesting public void doTransitionToWritableMode() { + doTransitionToWritableMode(false); + } + + @VisibleForTesting + public void doTransitionToWritableMode(boolean isManuallyModify) { if (shuttingdown || forceReadOnly.get()) { return; } + if (!isManuallyModify && bookieStatus.isInReadOnlyMode() && bookieStatus.isManuallyModifiedToReadOnly()) { + LOG.info("Skip to transition Bookie to Writable mode automatically because it is manually set to read-only" + + " mode, which can only be changed manually."); + return; + } if (!bookieStatus.setToWritableMode()) { // do nothing if already in writable mode @@ -333,12 +342,16 @@ public void doTransitionToWritableMode() { } } - @VisibleForTesting public void doTransitionToReadOnlyMode() { + doTransitionToReadOnlyMode(false); + } + + @VisibleForTesting + public void doTransitionToReadOnlyMode(boolean isManuallyModify) { if (shuttingdown) { return; } - if (!bookieStatus.setToReadOnlyMode()) { + if (!bookieStatus.setToReadOnlyMode(isManuallyModify)) { return; } if (!conf.isReadOnlyModeEnabled()) { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStatus.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStatus.java index 7b2d0aad990..ccbba36b532 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStatus.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStatus.java @@ -31,6 +31,7 @@ import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.util.List; +import lombok.Getter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,6 +54,8 @@ enum BookieMode { private int layoutVersion; private long lastUpdateTime; private volatile BookieMode bookieMode; + @Getter + private boolean isManuallyModifiedToReadOnly; BookieStatus() { this.bookieMode = BookieMode.READ_WRITE; @@ -72,6 +75,7 @@ synchronized boolean setToWritableMode() { if (!bookieMode.equals(BookieMode.READ_WRITE)) { bookieMode = BookieMode.READ_WRITE; this.lastUpdateTime = System.currentTimeMillis(); + this.isManuallyModifiedToReadOnly = false; return true; } return false; @@ -81,10 +85,11 @@ boolean isInReadOnlyMode() { return bookieMode.equals(BookieMode.READ_ONLY); } - synchronized boolean setToReadOnlyMode() { + synchronized boolean setToReadOnlyMode(boolean isManuallyModify) { if (!bookieMode.equals(BookieMode.READ_ONLY)) { bookieMode = BookieMode.READ_ONLY; this.lastUpdateTime = System.currentTimeMillis(); + this.isManuallyModifiedToReadOnly = isManuallyModify; return true; } return false; @@ -147,6 +152,7 @@ void readFromDirectories(List directories) { this.lastUpdateTime = status.lastUpdateTime; this.layoutVersion = status.layoutVersion; this.bookieMode = status.bookieMode; + this.isManuallyModifiedToReadOnly = status.isManuallyModifiedToReadOnly; success = true; } } @@ -216,6 +222,15 @@ public BookieStatus parse(BufferedReader reader) if (status.layoutVersion == 1 && parts.length == 3) { status.bookieMode = BookieMode.valueOf(parts[1]); status.lastUpdateTime = Long.parseLong(parts[2].trim()); + status.isManuallyModifiedToReadOnly = true; + return status; + } + // Since we should guarantee the compatibility of downgrade. We do not change the layoutVersion, otherwise, + // the string can not be parsed by the lower version. + if (status.layoutVersion == 1 && parts.length == 4) { + status.bookieMode = BookieMode.valueOf(parts[1]); + status.lastUpdateTime = Long.parseLong(parts[2].trim()); + status.isManuallyModifiedToReadOnly = Boolean.parseBoolean(parts[3].trim()); return status; } } @@ -231,6 +246,8 @@ public String toString() { builder.append(getBookieMode()); builder.append(","); builder.append(System.currentTimeMillis()); + builder.append(","); + builder.append(isManuallyModifiedToReadOnly); builder.append("\n"); return builder.toString(); } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/DefaultEntryLogger.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/DefaultEntryLogger.java index ab200b7ca0d..a8065411eed 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/DefaultEntryLogger.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/DefaultEntryLogger.java @@ -1062,7 +1062,7 @@ public EntryLogMetadata getEntryLogMetadata(long entryLogId, AbstractLogCompacto } catch (FileNotFoundException fne) { LOG.warn("Cannot find entry log file {}.log : {}", Long.toHexString(entryLogId), fne.getMessage()); throw fne; - } catch (Exception e) { + } catch (Throwable e) { LOG.info("Failed to get ledgers map index from: {}.log : {}", entryLogId, e.getMessage()); // Fall-back to scanning @@ -1152,6 +1152,10 @@ EntryLogMetadata extractEntryLogMetadataFromIndex(long entryLogId) throws IOExce + " -- found: " + meta.getLedgersMap().size() + " -- entryLogId: " + entryLogId); } + if (header.ledgersCount == 0) { + throw new IOException("No ledgers map found in entryLogId " + entryLogId + ", do scan to double confirm"); + } + return meta; } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLoggerAllocator.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLoggerAllocator.java index 68fc1eb3caf..e9ff5030d18 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLoggerAllocator.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLoggerAllocator.java @@ -81,6 +81,7 @@ class EntryLoggerAllocator { // within the same JVM. All of these Bookie instances access this header // so there can be race conditions when entry logs are rolled over and // this header buffer is cleared before writing it into the new logChannel. + logfileHeader.setZero(0, DefaultEntryLogger.LOGFILE_HEADER_SIZE); logfileHeader.writeBytes("BKLO".getBytes(UTF_8)); logfileHeader.writeInt(DefaultEntryLogger.HEADER_CURRENT_VERSION); logfileHeader.writerIndex(DefaultEntryLogger.LOGFILE_HEADER_SIZE); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectionStatus.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectionStatus.java index 3f872092f01..4ad450a64f1 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectionStatus.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectionStatus.java @@ -42,6 +42,8 @@ public class GarbageCollectionStatus { private long lastMajorCompactionTime; private long lastMinorCompactionTime; + private long lastEntryLocationCompactionTime; private long majorCompactionCounter; private long minorCompactionCounter; + private long entryLocationCompactionCounter; } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java index 4a1c0289879..b1c4b0ab2d8 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java @@ -33,6 +33,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -85,6 +86,10 @@ public class GarbageCollectorThread implements Runnable { long majorCompactionMaxTimeMillis; long lastMajorCompactionTime; + final long entryLocationCompactionInterval; + long randomCompactionDelay; + long lastEntryLocationCompactionTime; + @Getter final boolean isForceGCAllowWhenNoSpace; @@ -196,6 +201,10 @@ public GarbageCollectorThread(ServerConfiguration conf, isForceGCAllowWhenNoSpace = conf.getIsForceGCAllowWhenNoSpace(); majorCompactionMaxTimeMillis = conf.getMajorCompactionMaxTimeMillis(); minorCompactionMaxTimeMillis = conf.getMinorCompactionMaxTimeMillis(); + entryLocationCompactionInterval = conf.getEntryLocationCompactionInterval() * SECOND; + if (entryLocationCompactionInterval > 0) { + randomCompactionDelay = ThreadLocalRandom.current().nextLong(entryLocationCompactionInterval); + } boolean isForceAllowCompaction = conf.isForceAllowCompaction(); @@ -262,12 +271,22 @@ public void removeEntryLog(long logToRemove) { } } + if (entryLocationCompactionInterval > 0) { + if (entryLocationCompactionInterval < gcWaitTime) { + throw new IOException( + "Too short entry location compaction interval : " + entryLocationCompactionInterval); + } + } + LOG.info("Minor Compaction : enabled=" + enableMinorCompaction + ", threshold=" + minorCompactionThreshold + ", interval=" + minorCompactionInterval); LOG.info("Major Compaction : enabled=" + enableMajorCompaction + ", threshold=" + majorCompactionThreshold + ", interval=" + majorCompactionInterval); + LOG.info("Entry Location Compaction : interval=" + entryLocationCompactionInterval + ", randomCompactionDelay=" + + randomCompactionDelay); - lastMinorCompactionTime = lastMajorCompactionTime = System.currentTimeMillis(); + lastMinorCompactionTime = lastMajorCompactionTime = + lastEntryLocationCompactionTime = System.currentTimeMillis(); } private EntryLogMetadataMap createEntryLogMetadataMap() throws IOException { @@ -445,6 +464,7 @@ public void runWithFlags(boolean force, boolean suspendMajor, boolean suspendMin gcStats.getMajorCompactionCounter().inc(); majorCompacting.set(false); } + } else if (((isForceMinorCompactionAllow && force) || (enableMinorCompaction && (force || curTime - lastMinorCompactionTime > minorCompactionInterval))) && (!suspendMinor)) { @@ -459,6 +479,20 @@ public void runWithFlags(boolean force, boolean suspendMajor, boolean suspendMin minorCompacting.set(false); } } + if (entryLocationCompactionInterval > 0 && (curTime - lastEntryLocationCompactionTime > ( + entryLocationCompactionInterval + randomCompactionDelay))) { + // enter entry location compaction + LOG.info( + "Enter entry location compaction, entryLocationCompactionInterval {}, randomCompactionDelay " + + "{}, lastEntryLocationCompactionTime {}", + entryLocationCompactionInterval, randomCompactionDelay, lastEntryLocationCompactionTime); + ledgerStorage.entryLocationCompact(); + lastEntryLocationCompactionTime = System.currentTimeMillis(); + randomCompactionDelay = ThreadLocalRandom.current().nextLong(entryLocationCompactionInterval); + LOG.info("Next entry location compaction interval {}", + entryLocationCompactionInterval + randomCompactionDelay); + gcStats.getEntryLocationCompactionCounter().inc(); + } gcStats.getGcThreadRuntime().registerSuccessfulEvent( MathUtils.nowInNano() - threadStart, TimeUnit.NANOSECONDS); } catch (EntryLogMetadataMapException e) { @@ -466,6 +500,10 @@ public void runWithFlags(boolean force, boolean suspendMajor, boolean suspendMin e.getMessage(), e); gcStats.getGcThreadRuntime().registerFailedEvent( MathUtils.nowInNano() - threadStart, TimeUnit.NANOSECONDS); + } catch (Throwable e) { + LOG.error("Error in garbage collector thread, Failed to complete GC/Compaction due to {}", + e.getMessage(), e); + gcStats.getGcThreadRuntime().registerFailedEvent(MathUtils.elapsedNanos(threadStart), TimeUnit.NANOSECONDS); } finally { if (force && forceGarbageCollection.compareAndSet(true, false)) { LOG.info("{} Set forceGarbageCollection to false after force GC to make it forceGC-able again.", @@ -757,11 +795,12 @@ protected void extractMetaFromEntryLogs() throws EntryLogMetadataMapException { continue; } - LOG.info("Extracting entry log meta from entryLogId: {}", entryLogId); try { // Read through the entry log file and extract the entry log meta EntryLogMetadata entryLogMeta = entryLogger.getEntryLogMetadata(entryLogId, throttler); + LOG.info("Extracted entry log meta from entryLogId: {}, ledgers {}", + entryLogId, entryLogMeta.getLedgersMap().keys()); removeIfLedgerNotExists(entryLogMeta); if (entryLogMeta.isEmpty()) { // This means the entry log is not associated with any active @@ -774,8 +813,13 @@ protected void extractMetaFromEntryLogs() throws EntryLogMetadataMapException { entryLogMetaMap.put(entryLogId, entryLogMeta); } } catch (IOException | RuntimeException e) { - LOG.warn("Premature exception when processing " + entryLogId - + " recovery will take care of the problem", e); + LOG.warn("Premature exception when processing {} recovery will take care of the problem", + entryLogId, e); + } catch (OutOfMemoryError oome) { + // somewhat similar to https://github.com/apache/bookkeeper/pull/3901 + // entrylog file can be corrupted but instead having a negative entry size + // it ends up with very large value for the entry size causing OODME + LOG.warn("OutOfMemoryError when processing {} - skipping the entry log", entryLogId, oome); } } } @@ -796,8 +840,10 @@ public GarbageCollectionStatus getGarbageCollectionStatus() { .minorCompacting(minorCompacting.get()) .lastMajorCompactionTime(lastMajorCompactionTime) .lastMinorCompactionTime(lastMinorCompactionTime) + .lastEntryLocationCompactionTime(lastEntryLocationCompactionTime) .majorCompactionCounter(gcStats.getMajorCompactionCounter().get()) .minorCompactionCounter(gcStats.getMinorCompactionCounter().get()) + .entryLocationCompactionCounter(gcStats.getEntryLocationCompactionCounter().get()) .build(); } } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/HandleFactory.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/HandleFactory.java index 22500b74cb6..c81294d4dbe 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/HandleFactory.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/HandleFactory.java @@ -24,7 +24,7 @@ import java.io.IOException; interface HandleFactory { - LedgerDescriptor getHandle(long ledgerId, byte[] masterKey) + LedgerDescriptor getHandle(long ledgerId, byte[] masterKey, boolean journalReplay) throws IOException, BookieException; LedgerDescriptor getReadOnlyHandle(long ledgerId) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/HandleFactoryImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/HandleFactoryImpl.java index 3f643019c9a..ac87c3aed45 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/HandleFactoryImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/HandleFactoryImpl.java @@ -21,7 +21,10 @@ package org.apache.bookkeeper.bookie; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import java.io.IOException; +import java.time.Duration; import org.apache.bookkeeper.bookie.LedgerStorage.LedgerDeletionListener; import org.apache.bookkeeper.util.collections.ConcurrentLongHashMap; @@ -29,6 +32,14 @@ class HandleFactoryImpl implements HandleFactory, LedgerDeletionListener { private final ConcurrentLongHashMap ledgers; private final ConcurrentLongHashMap readOnlyLedgers; + /** + * Once the ledger was marked "fenced" before, the ledger was accessed by multi clients. One client is calling + * "delete" now, and other clients may call "write" continuously later. We mark these ledgers can not be written + * anymore. And maintains the state for 7 days is safety. + */ + private final Cache recentlyFencedAndDeletedLedgers = CacheBuilder.newBuilder() + .expireAfterAccess(Duration.ofDays(7)).build(); + final LedgerStorage ledgerStorage; HandleFactoryImpl(LedgerStorage ledgerStorage) { @@ -40,10 +51,14 @@ class HandleFactoryImpl implements HandleFactory, LedgerDeletionListener { } @Override - public LedgerDescriptor getHandle(final long ledgerId, final byte[] masterKey) throws IOException, BookieException { + public LedgerDescriptor getHandle(final long ledgerId, final byte[] masterKey, boolean journalReplay) + throws IOException, BookieException { LedgerDescriptor handle = ledgers.get(ledgerId); if (handle == null) { + if (!journalReplay && recentlyFencedAndDeletedLedgers.getIfPresent(ledgerId) != null) { + throw BookieException.create(BookieException.Code.LedgerFencedAndDeletedException); + } handle = LedgerDescriptor.create(masterKey, ledgerId, ledgerStorage); ledgers.putIfAbsent(ledgerId, handle); } @@ -64,8 +79,22 @@ public LedgerDescriptor getReadOnlyHandle(final long ledgerId) throws IOExceptio return handle; } + private void markIfConflictWritingOccurs(long ledgerId) { + LedgerDescriptor ledgerDescriptor = ledgers.get(ledgerId); + try { + if (ledgerDescriptor != null && ledgerDescriptor.isFenced()) { + recentlyFencedAndDeletedLedgers.put(ledgerId, true); + } + } catch (IOException | BookieException ex) { + // The ledger is in limbo state. + recentlyFencedAndDeletedLedgers.put(ledgerId, true); + } + } + @Override public void ledgerDeleted(long ledgerId) { + markIfConflictWritingOccurs(ledgerId); + // Do delete. ledgers.remove(ledgerId); readOnlyLedgers.remove(ledgerId); } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsMonitor.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsMonitor.java index 7ad8ba1e0c5..73e2ba18712 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsMonitor.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsMonitor.java @@ -21,6 +21,7 @@ package org.apache.bookkeeper.bookie; +import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.io.File; import java.io.IOException; @@ -54,6 +55,7 @@ class LedgerDirsMonitor { private final long minUsableSizeForHighPriorityWrites; private ScheduledExecutorService executor; private ScheduledFuture checkTask; + private boolean isFirstLoopOfCheckTask = true; public LedgerDirsMonitor(final ServerConfiguration conf, final DiskChecker diskChecker, @@ -67,6 +69,10 @@ public LedgerDirsMonitor(final ServerConfiguration conf, } private void check(final LedgerDirsManager ldm) { + final boolean isFirstLoopOfCheckTaskLocalValue = this.isFirstLoopOfCheckTask; + if (isFirstLoopOfCheckTaskLocalValue) { + this.isFirstLoopOfCheckTask = false; + } final ConcurrentMap diskUsages = ldm.getDiskUsages(); boolean someDiskFulled = false; boolean highPriorityWritesAllowed = true; @@ -175,6 +181,14 @@ private void check(final LedgerDirsManager ldm) { } } + if (isFirstLoopOfCheckTaskLocalValue && ldm.getFullFilledLedgerDirs().isEmpty()) { + // notify no disk full. + for (LedgerDirsListener listener : ldm.getListeners()) { + listener.allDisksWritable(); + } + return; + } + if (conf.isReadOnlyModeOnAnyDiskFullEnabled()) { if (someDiskFulled && !ldm.getFullFilledLedgerDirs().isEmpty()) { // notify any disk full. @@ -192,7 +206,8 @@ private void check(final LedgerDirsManager ldm) { } } - private void check() { + @VisibleForTesting + void check() { dirsManagers.forEach(this::check); } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/StateManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/StateManager.java index 7ed3f0b6572..9ef1f384e77 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/StateManager.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/StateManager.java @@ -100,12 +100,20 @@ public interface StateManager extends AutoCloseable { /** * Change the state of bookie to Writable mode. */ - Future transitionToWritableMode(); + Future transitionToWritableMode(boolean isManuallyModify); + + default Future transitionToWritableMode() { + return transitionToWritableMode(false); + } /** * Change the state of bookie to ReadOnly mode. */ - Future transitionToReadOnlyMode(); + Future transitionToReadOnlyMode(boolean isManuallyModify); + + default Future transitionToReadOnlyMode() { + return transitionToReadOnlyMode(false); + } /** * ShutdownHandler used to shutdown bookie. diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/stats/GarbageCollectorStats.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/stats/GarbageCollectorStats.java index 1c9475608f4..3029e7d55b7 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/stats/GarbageCollectorStats.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/stats/GarbageCollectorStats.java @@ -25,6 +25,7 @@ import static org.apache.bookkeeper.bookie.BookKeeperServerStats.BOOKIE_SCOPE; import static org.apache.bookkeeper.bookie.BookKeeperServerStats.CATEGORY_SERVER; import static org.apache.bookkeeper.bookie.BookKeeperServerStats.DELETED_LEDGER_COUNT; +import static org.apache.bookkeeper.bookie.BookKeeperServerStats.ENTRY_LOCATION_COMPACTION_COUNT; import static org.apache.bookkeeper.bookie.BookKeeperServerStats.MAJOR_COMPACTION_COUNT; import static org.apache.bookkeeper.bookie.BookKeeperServerStats.MINOR_COMPACTION_COUNT; import static org.apache.bookkeeper.bookie.BookKeeperServerStats.RECLAIMED_COMPACTION_SPACE_BYTES; @@ -61,6 +62,11 @@ public class GarbageCollectorStats { help = "Number of major compactions" ) private final Counter majorCompactionCounter; + @StatsDoc( + name = ENTRY_LOCATION_COMPACTION_COUNT, + help = "Number of entry location compactions" + ) + private final Counter entryLocationCompactionCounter; @StatsDoc( name = RECLAIMED_DELETION_SPACE_BYTES, help = "Number of disk space bytes reclaimed via deleting entry log files" @@ -105,6 +111,7 @@ public GarbageCollectorStats(StatsLogger statsLogger, this.minorCompactionCounter = statsLogger.getCounter(MINOR_COMPACTION_COUNT); this.majorCompactionCounter = statsLogger.getCounter(MAJOR_COMPACTION_COUNT); + this.entryLocationCompactionCounter = statsLogger.getCounter(ENTRY_LOCATION_COMPACTION_COUNT); this.reclaimedSpaceViaCompaction = statsLogger.getCounter(RECLAIMED_COMPACTION_SPACE_BYTES); this.reclaimedSpaceViaDeletes = statsLogger.getCounter(RECLAIMED_DELETION_SPACE_BYTES); this.gcThreadRuntime = statsLogger.getOpStatsLogger(THREAD_RUNTIME); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java index a77a0a18f7c..b870fb5939d 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java @@ -38,8 +38,10 @@ import java.util.List; import java.util.Map.Entry; import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.bookkeeper.bookie.storage.ldb.KeyValueStorageFactory.DbConfigType; import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.commons.lang3.StringUtils; import org.rocksdb.BlockBasedTableConfig; import org.rocksdb.BloomFilter; import org.rocksdb.Cache; @@ -73,6 +75,8 @@ public class KeyValueStorageRocksDB implements KeyValueStorage { static KeyValueStorageFactory factory = (defaultBasePath, subPath, dbConfigType, conf) -> new KeyValueStorageRocksDB(defaultBasePath, subPath, dbConfigType, conf); + private volatile boolean closed = true; + private final ReentrantReadWriteLock closedLock = new ReentrantReadWriteLock(); private final RocksDB db; private RocksObject options; private List columnFamilyDescriptors; @@ -131,7 +135,7 @@ public KeyValueStorageRocksDB(String basePath, String subPath, DbConfigType dbCo dbFilePath = conf.getDefaultRocksDBConf(); } log.info("Searching for a RocksDB configuration file in {}", dbFilePath); - if (Paths.get(dbFilePath).toFile().exists()) { + if (StringUtils.isNotBlank(dbFilePath) && Paths.get(dbFilePath).toFile().exists()) { log.info("Found a RocksDB configuration file and using it to initialize the RocksDB"); db = initializeRocksDBWithConfFile(basePath, subPath, dbConfigType, conf, readOnly, dbFilePath); } else { @@ -146,6 +150,7 @@ public KeyValueStorageRocksDB(String basePath, String subPath, DbConfigType dbCo optionDontCache.setFillCache(false); this.writeBatchMaxSize = conf.getMaxOperationNumbersInSingleRocksDBBatch(); + this.closed = false; } private RocksDB initializeRocksDBWithConfFile(String basePath, String subPath, DbConfigType dbConfigType, @@ -182,6 +187,7 @@ private RocksDB initializeRocksDBWithBookieConf(String basePath, String subPath, Options options = new Options(); options.setCreateIfMissing(true); ChecksumType checksumType = ChecksumType.valueOf(conf.getString(ROCKSDB_CHECKSUM_TYPE, "kxxHash")); + int formatVersion = conf.getInt(ROCKSDB_FORMAT_VERSION, 5); if (dbConfigType == DbConfigType.EntryLocation) { /* Set default RocksDB block-cache size to 10% / numberOfLedgers of direct memory, unless override */ @@ -198,7 +204,6 @@ private RocksDB initializeRocksDBWithBookieConf(String basePath, String subPath, int blockSize = conf.getInt(ROCKSDB_BLOCK_SIZE, 64 * 1024); int bloomFilterBitsPerKey = conf.getInt(ROCKSDB_BLOOM_FILTERS_BITS_PER_KEY, 10); boolean lz4CompressionEnabled = conf.getBoolean(ROCKSDB_LZ4_COMPRESSION_ENABLED, true); - int formatVersion = conf.getInt(ROCKSDB_FORMAT_VERSION, 2); if (lz4CompressionEnabled) { options.setCompressionType(CompressionType.LZ4_COMPRESSION); @@ -235,6 +240,7 @@ private RocksDB initializeRocksDBWithBookieConf(String basePath, String subPath, } else { this.cache = null; BlockBasedTableConfig tableOptions = new BlockBasedTableConfig(); + tableOptions.setFormatVersion(formatVersion); tableOptions.setChecksumType(checksumType); options.setTableFormatConfig(tableOptions); } @@ -285,7 +291,13 @@ private RocksDB initializeRocksDBWithBookieConf(String basePath, String subPath, @Override public void close() throws IOException { - db.close(); + try { + closedLock.writeLock().lock(); + closed = true; + db.close(); + } finally { + closedLock.writeLock().unlock(); + } if (cache != null) { cache.close(); } @@ -511,7 +523,18 @@ public void close() { @Override public long count() throws IOException { try { - return db.getLongProperty("rocksdb.estimate-num-keys"); + if (closed) { + throw new IOException("RocksDB is closed"); + } + try { + closedLock.readLock().lock(); + if (!closed) { + return db.getLongProperty("rocksdb.estimate-num-keys"); + } + throw new IOException("RocksDB is closed"); + } finally { + closedLock.readLock().unlock(); + } } catch (RocksDBException e) { throw new IOException("Error in getting records count", e); } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LedgerMetadataIndex.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LedgerMetadataIndex.java index 9b50479b7d5..edc816ce4b9 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LedgerMetadataIndex.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LedgerMetadataIndex.java @@ -336,6 +336,7 @@ public void removeDeletedLedgers() throws IOException { key.set(ledgerId); ledgersDb.delete(key.array); ++deletedLedgers; + pendingDeletedLedgers.remove(ledgerId); } if (log.isDebugEnabled()) { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java index 61aebd8e1a1..867ba905ffe 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java @@ -313,6 +313,8 @@ public boolean isMinorGcSuspended() { public void entryLocationCompact() { if (entryLocationIndex.isCompacting()) { // RocksDB already running compact. + log.info("Compacting directory {}, skipping this entryLocationCompaction this time.", + entryLocationIndex.getEntryLocationDBPath()); return; } cleanupExecutor.execute(() -> { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeper.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeper.java index d7043dc8c9a..0362aadcbaa 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeper.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeper.java @@ -478,6 +478,7 @@ public BookKeeper(ClientConfiguration conf, ZooKeeper zk, EventLoopGroup eventLo .poolingConcurrency(conf.getAllocatorPoolingConcurrency()) .outOfMemoryPolicy(conf.getAllocatorOutOfMemoryPolicy()) .leakDetectionPolicy(conf.getAllocatorLeakDetectionPolicy()) + .exitOnOutOfMemory(conf.exitOnOutOfMemory()) .build(); } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookieWatcherImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookieWatcherImpl.java index 978842b114a..f7af1549286 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookieWatcherImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookieWatcherImpl.java @@ -273,7 +273,7 @@ public List newEnsemble(int ensembleSize, int writeQuorumSize, if (isEnsembleAdheringToPlacementPolicy == PlacementPolicyAdherence.FAIL) { ensembleNotAdheringToPlacementPolicy.inc(); if (ensembleSize > 1) { - log.warn("New ensemble: {} is not adhering to Placement Policy. quarantinedBookies: {}", + log.info("New ensemble: {} is not adhering to Placement Policy. quarantinedBookies: {}", socketAddresses, quarantinedBookiesSet); } } @@ -289,7 +289,7 @@ public List newEnsemble(int ensembleSize, int writeQuorumSize, isEnsembleAdheringToPlacementPolicy = newEnsembleResponse.getAdheringToPolicy(); if (isEnsembleAdheringToPlacementPolicy == PlacementPolicyAdherence.FAIL) { ensembleNotAdheringToPlacementPolicy.inc(); - log.warn("New ensemble: {} is not adhering to Placement Policy", socketAddresses); + log.info("New ensemble: {} is not adhering to Placement Policy", socketAddresses); } newEnsembleTimer.registerFailedEvent(MathUtils.nowInNano() - startTime, TimeUnit.NANOSECONDS); } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerHandle.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerHandle.java index 0ea5f8fec14..e99d8ba91df 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerHandle.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerHandle.java @@ -1849,19 +1849,24 @@ void notifyWriteFailed(int index, BookieId addr) { } void maybeHandleDelayedWriteBookieFailure() { + Map toReplace = null; synchronized (metadataLock) { if (delayedWriteFailedBookies.isEmpty()) { return; } - Map toReplace = new HashMap<>(delayedWriteFailedBookies); + toReplace = new HashMap<>(delayedWriteFailedBookies); delayedWriteFailedBookies.clear(); + } - // Original intent of this change is to do a best-effort ensemble change. - // But this is not possible until the local metadata is completely immutable. - // Until the feature "Make LedgerMetadata Immutable #610" Is complete we will use - // handleBookieFailure() to handle delayed writes as regular bookie failures. - handleBookieFailure(toReplace); + if (toReplace.isEmpty()) { + return; } + + // Original intent of this change is to do a best-effort ensemble change. + // But this is not possible until the local metadata is completely immutable. + // Until the feature "Make LedgerMetadata Immutable #610" Is complete we will use + // handleBookieFailure() to handle delayed writes as regular bookie failures. + handleBookieFailure(toReplace); } void handleBookieFailure(final Map failedBookies) { @@ -1980,12 +1985,12 @@ void ensembleChangeLoop(List origEnsemble, Map fail List newEnsemble = null; Set replaced = null; + + Map toReplace = null; synchronized (metadataLock) { if (!delayedWriteFailedBookies.isEmpty()) { - Map toReplace = new HashMap<>(delayedWriteFailedBookies); + toReplace = new HashMap<>(delayedWriteFailedBookies); delayedWriteFailedBookies.clear(); - - ensembleChangeLoop(origEnsemble, toReplace); } else { newEnsemble = getCurrentEnsemble(); replaced = EnsembleUtils.diffEnsemble(origEnsemble, newEnsemble); @@ -1994,6 +1999,11 @@ void ensembleChangeLoop(List origEnsemble, Map fail changingEnsemble = false; } } + + if (toReplace != null && !toReplace.isEmpty()) { + ensembleChangeLoop(origEnsemble, toReplace); + } + if (newEnsemble != null) { // unsetSuccess outside of lock unsetSuccessAndSendWriteRequest(newEnsemble, replaced); } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java index 4f9ed29f15e..f71302e3d1d 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java @@ -241,6 +241,7 @@ public RackawareEnsemblePlacementPolicyImpl withDefaultRack(String rack) { return this; } + @Override public String getDefaultRack() { return defaultRack; } @@ -466,13 +467,26 @@ public PlacementResult replaceBookie(int ensembleSize, int writeQuorum try { excludeBookies = addDefaultRackBookiesIfMinNumRacksIsEnforced(excludeBookies); excludeBookies.addAll(currentEnsemble); + + Set ensembleNodes = new HashSet<>(); + Set excludeNodes = new HashSet<>(); BookieNode bn = knownBookies.get(bookieToReplace); if (null == bn) { bn = createBookieNode(bookieToReplace); } - - Set ensembleNodes = convertBookiesToNodes(currentEnsemble); - Set excludeNodes = convertBookiesToNodes(excludeBookies); + for (BookieId bookieId : currentEnsemble) { + if (bookieId.equals(bookieToReplace)) { + continue; + } + ensembleNodes.add(convertBookieToNode(bookieId)); + } + for (BookieId bookieId : excludeBookies) { + if (bookieId.equals(bookieToReplace)) { + excludeNodes.add(bn); + continue; + } + excludeNodes.add(convertBookieToNode(bookieId)); + } excludeNodes.addAll(ensembleNodes); excludeNodes.add(bn); @@ -535,7 +549,7 @@ public BookieNode selectFromNetworkLocation( LOG.warn("Failed to choose a bookie from network location {}, " + "the bookies in the network location are {}, excluded bookies {}, " + "current ensemble {}, fallback to choose bookie randomly from the cluster.", - networkLoc, topology.getLeaves(networkLoc), excludeBookies, ensemble); + networkLoc, topology.getLeaves(networkLoc), excludeBookies, ensemble.toList()); // randomly choose one from whole cluster, ignore the provided predicate. return selectRandom(1, excludeBookies, predicate, ensemble).get(0); } @@ -561,7 +575,7 @@ public BookieNode selectFromNetworkLocation(String networkLoc, LOG.warn("Failed to choose a bookie node from network location {}, " + "the bookies in the network location are {}, excluded bookies {}, " + "current ensemble {}, fallback to choose bookie randomly from the cluster.", - networkLoc, topology.getLeaves(networkLoc), excludeBookies, ensemble); + networkLoc, topology.getLeaves(networkLoc), excludeBookies, ensemble.toList()); return selectFromNetworkLocation(excludeRacks, excludeBookies, predicate, ensemble, fallbackToRandom); } } @@ -865,8 +879,9 @@ DistributionSchedule.WriteSet reorderReadSequenceWithRegion( if (useRegionAware || reorderReadsRandom) { isAnyBookieUnavailable = true; } else { - for (int i = 0; i < ensemble.size(); i++) { - BookieId bookieAddr = ensemble.get(i); + for (int i = 0; i < writeSet.size(); i++) { + int idx = writeSet.get(i); + BookieId bookieAddr = ensemble.get(idx); if ((!knownBookies.containsKey(bookieAddr) && !readOnlyBookies.contains(bookieAddr)) || slowBookies.getIfPresent(bookieAddr) != null) { // Found at least one bookie not available in the ensemble, or in slowBookies diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RegionAwareEnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RegionAwareEnsemblePlacementPolicy.java index ef68e024672..bc882589b36 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RegionAwareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RegionAwareEnsemblePlacementPolicy.java @@ -117,6 +117,23 @@ protected String parseBookieRegion(BookieId addr) { @Override public void handleBookiesThatLeft(Set leftBookies) { + // case 1: (In some situation, eg, Broker and bookie restart concurrently.) + //1. Bookie X join cluster for the first time, encounters a region exception, and `address2Region` record X's + // region as default-region. + //2. Bookie X left cluster and is removed from knownBookies, but address2Region retains the information of + // bookie X. + //3. update Bookie X's rack info, and calling `onBookieRackChange` will only update address2Region for + // addresses present in knownBookies; therefore, bookie X's region info is not updated. + //4. Bookie X join cluster again, since address2Region contains the previous default-region information, + // getRegion will directly use cached data, resulting of an incorrect region. + + // The bookie region is initialized to "default-region" in address2Region. + // We should ensure that when a bookie leaves the cluster, + // we also clean up the corresponding region information for that bookie in address2Region, + // so that it can update the correct region for the bookie during onBookieRackChange and + // handleBookiesThatJoined. + // to avoid traffic skew in ensemble selection. + leftBookies.forEach(address2Region::remove); super.handleBookiesThatLeft(leftBookies); for (TopologyAwareEnsemblePlacementPolicy policy: perRegionPlacement.values()) { @@ -649,4 +666,12 @@ public PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy(List bookieInfoMap) { + super.updateBookieInfo(bookieInfoMap); + for (TopologyAwareEnsemblePlacementPolicy policy : perRegionPlacement.values()) { + policy.updateBookieInfo(bookieInfoMap); + } + } } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java index 463d9599de2..9eae79438fd 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java @@ -824,13 +824,17 @@ protected String resolveNetworkLocation(BookieId addr) { if (null != historyBookie) { return historyBookie.getNetworkLocation(); } - - LOG.error("Cannot resolve bookieId {} to a network address, resolving as {}", addr, - NetworkTopology.DEFAULT_REGION_AND_RACK, err); - return NetworkTopology.DEFAULT_REGION_AND_RACK; + String defaultRack = getDefaultRack(); + LOG.error("Cannot resolve bookieId {} to a network address, resolving as {}. {}", addr, + defaultRack, err.getMessage()); + return defaultRack; } } + protected String getDefaultRack() { + return NetworkTopology.DEFAULT_REGION_AND_RACK; + } + protected Set convertBookiesToNodes(Collection bookies) { Set nodes = new HashSet(); for (BookieId addr : bookies) { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/AbstractConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/AbstractConfiguration.java index 438dc40983e..6d2a82bb551 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/AbstractConfiguration.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/AbstractConfiguration.java @@ -183,6 +183,7 @@ public abstract class AbstractConfiguration protected static final String ALLOCATOR_POOLING_CONCURRENCY = "allocatorPoolingConcurrency"; protected static final String ALLOCATOR_OOM_POLICY = "allocatorOutOfMemoryPolicy"; protected static final String ALLOCATOR_LEAK_DETECTION_POLICY = "allocatorLeakDetectionPolicy"; + protected static final String ALLOCATOR_EXIT_ON_OUT_OF_MEMORY = "allocatorExitOnOutOfMemory"; // option to limit stats logging public static final String LIMIT_STATS_LOGGING = "limitStatsLogging"; @@ -1156,6 +1157,15 @@ public T setAllocatorLeakDetectionPolicy(LeakDetectionPolicy leakDetectionPolicy return getThis(); } + public T setExitOnOutOfMemory(boolean exitOnOutOfMemory) { + this.setProperty(ALLOCATOR_EXIT_ON_OUT_OF_MEMORY, exitOnOutOfMemory); + return getThis(); + } + + public boolean exitOnOutOfMemory() { + return getBoolean(ALLOCATOR_EXIT_ON_OUT_OF_MEMORY, false); + } + /** * Return whether the busy-wait is enabled for BookKeeper and Netty IO threads. * diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java index 019144e9942..9a6f0e5b9e3 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java @@ -25,6 +25,7 @@ import java.io.File; import java.net.URL; import java.util.concurrent.TimeUnit; +import lombok.SneakyThrows; import org.apache.bookkeeper.bookie.FileChannelProvider; import org.apache.bookkeeper.bookie.InterleavedLedgerStorage; import org.apache.bookkeeper.bookie.LedgerStorage; @@ -105,6 +106,7 @@ public class ServerConfiguration extends AbstractConfigurationIf it is set to less than zero, the entry location compaction is disabled. + * + * @return high water mark. + */ + public long getEntryLocationCompactionInterval() { + return getLong(ENTRY_LOCATION_COMPACTION_INTERVAL, -1); + } + + /** + * Set interval to run entry location compaction. + * + * @see #getMajorCompactionInterval() + * + * @param interval + * Interval to run entry location compaction + * @return server configuration + */ + public ServerConfiguration setEntryLocationCompactionInterval(long interval) { + setProperty(ENTRY_LOCATION_COMPACTION_INTERVAL, interval); + return this; + } + /** * Should we remove pages from page cache after force write. * @@ -3187,6 +3214,10 @@ public void validate() throws ConfigurationException { if (getMajorCompactionInterval() > 0 && getMajorCompactionInterval() * SECOND < getGcWaitTime()) { throw new ConfigurationException("majorCompactionInterval should be >= gcWaitTime."); } + if (getEntryLocationCompactionInterval() > 0 + && getEntryLocationCompactionInterval() * SECOND < getGcWaitTime()) { + throw new ConfigurationException("entryLocationCompactionInterval should be >= gcWaitTime."); + } } /** @@ -4017,12 +4048,7 @@ public boolean isSkipReplayJournalInvalidRecord() { * @return String configured default rocksdb conf. */ public String getDefaultRocksDBConf() { - String defaultPath = "conf/default_rocksdb.conf"; - URL defURL = getClass().getClassLoader().getResource(defaultPath); - if (defURL != null) { - defaultPath = defURL.getPath(); - } - return getString(DEFAULT_ROCKSDB_CONF, defaultPath); + return getString(DEFAULT_ROCKSDB_CONF, getDefaultFilePath("conf/default_rocksdb.conf")); } /** @@ -4041,12 +4067,7 @@ public ServerConfiguration setDefaultRocksDBConf(String defaultRocksdbConf) { * @return String configured entry Location rocksdb conf. */ public String getEntryLocationRocksdbConf() { - String defaultPath = "conf/entry_location_rocksdb.conf"; - URL defURL = getClass().getClassLoader().getResource(defaultPath); - if (defURL != null) { - defaultPath = defURL.getPath(); - } - return getString(ENTRY_LOCATION_ROCKSDB_CONF, defaultPath); + return getString(ENTRY_LOCATION_ROCKSDB_CONF, getDefaultFilePath("conf/entry_location_rocksdb.conf")); } /** @@ -4065,12 +4086,7 @@ public ServerConfiguration setEntryLocationRocksdbConf(String entryLocationRocks * @return String configured ledger metadata rocksdb conf. */ public String getLedgerMetadataRocksdbConf() { - String defaultPath = "conf/ledger_metadata_rocksdb.conf"; - URL defURL = getClass().getClassLoader().getResource(defaultPath); - if (defURL != null) { - defaultPath = defURL.getPath(); - } - return getString(LEDGER_METADATA_ROCKSDB_CONF, defaultPath); + return getString(LEDGER_METADATA_ROCKSDB_CONF, getDefaultFilePath("conf/ledger_metadata_rocksdb.conf")); } /** @@ -4103,4 +4119,28 @@ public ServerConfiguration setOperationMaxNumbersInSingleRocksDBWriteBatch(int m public int getMaxOperationNumbersInSingleRocksDBBatch() { return getInt(MAX_OPERATION_NUMBERS_IN_SINGLE_ROCKSDB_WRITE_BATCH, 100000); } + + /** + * Retrieves the default file path for the specified file name. + * This method prioritizes a file available in the classpath, which is often used in testing scenarios. + * If the file is not found in the classpath, the original file name is returned. + * + * @param fileName the name of the file for which to retrieve the path. + * @return the path of the file if found in the classpath, otherwise the input file name. + */ + @SneakyThrows + private String getDefaultFilePath(String fileName) { + // Attempt to locate the file in the classpath, used mainly for testing purposes. + URL resourceURL = getClass().getClassLoader().getResource(fileName); + if (resourceURL != null && "file".equals(resourceURL.getProtocol())) { + // Convert the URL to a File object using toURI() for proper URL decoding + // and platform specific file path handling (such as on Windows OS) + File file = new File(resourceURL.toURI()); + if (file.exists()) { + return file.getAbsolutePath(); + } + } + // Return the original file name if no path was found in the classpath + return fileName; + } } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieClientImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieClientImpl.java index 5dde52bc3f5..1e5184d07e8 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieClientImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieClientImpl.java @@ -22,6 +22,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import com.google.protobuf.ExtensionRegistry; import io.netty.buffer.ByteBuf; @@ -355,7 +356,9 @@ private void completeRead(final int rc, } } - private static class ChannelReadyForAddEntryCallback + // Without test, this class should be modifier with "private". + @VisibleForTesting + static class ChannelReadyForAddEntryCallback implements GenericCallback { private final Handle recyclerHandle; @@ -365,7 +368,9 @@ private static class ChannelReadyForAddEntryCallback private long entryId; private BookieId addr; private Object ctx; - private WriteCallback cb; + // Without test, this class should be modifier with "private". + @VisibleForTesting + WriteCallback cb; private int options; private byte[] masterKey; private boolean allowFastFail; @@ -394,17 +399,24 @@ static ChannelReadyForAddEntryCallback create( @Override public void operationComplete(final int rc, PerChannelBookieClient pcbc) { - try { - if (rc != BKException.Code.OK) { - bookieClient.completeAdd(rc, ledgerId, entryId, addr, cb, ctx); - } else { + if (rc != BKException.Code.OK) { + bookieClient.executor.executeOrdered(ledgerId, () -> { + try { + bookieClient.completeAdd(rc, ledgerId, entryId, addr, cb, ctx); + } finally { + ReferenceCountUtil.release(toSend); + } + recycle(); + }); + } else { + try { pcbc.addEntry(ledgerId, masterKey, entryId, toSend, cb, ctx, options, allowFastFail, writeFlags); + } finally { + ReferenceCountUtil.release(toSend); } - } finally { - ReferenceCountUtil.release(toSend); + recycle(); } - recycle(); } private ChannelReadyForAddEntryCallback( diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java index 149f97fc28b..7bce38aae2c 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java @@ -615,11 +615,15 @@ protected void initChannel(Channel ch) throws Exception { } ChannelFuture future = bootstrap.connect(bookieAddr); - future.addListener(contextPreservingListener(new ConnectionFutureListener(startTime))); - future.addListener(x -> makeWritable()); + addChannelListeners(future, startTime); return future; } + protected void addChannelListeners(ChannelFuture future, long connectStartTime) { + future.addListener(contextPreservingListener(new ConnectionFutureListener(connectStartTime))); + future.addListener(x -> makeWritable()); + } + void cleanDisconnectAndClose() { disconnect(); close(); @@ -1161,7 +1165,13 @@ private void writeAndFlush(final Channel channel, completion.setOutstanding(); } } else { + try { + future.get(); + } catch (Exception ex) { + LOG.warn("Failed to request to the bookie: {}", bookieId, ex); + } nettyOpLogger.registerFailedEvent(MathUtils.elapsedNanos(startTime), TimeUnit.NANOSECONDS); + errorOut(key); } } finally { if (cleanupActionAfterWrite != null) { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/ReadEntryProcessor.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/ReadEntryProcessor.java index 04efd9634b2..9c6d672f480 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/ReadEntryProcessor.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/ReadEntryProcessor.java @@ -93,7 +93,7 @@ protected void processPacket() { handleReadResultForFenceRead(fenceResult, data, startTimeNanos); return; } - } catch (Bookie.NoLedgerException e) { + } catch (Bookie.NoLedgerException | BookieException.LedgerFencedAndDeletedException e) { if (LOG.isDebugEnabled()) { LOG.debug("Error reading {}", request, e); } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/ReadEntryProcessorV3.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/ReadEntryProcessorV3.java index 999b8095db6..05f9de0ea3a 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/ReadEntryProcessorV3.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/ReadEntryProcessorV3.java @@ -215,9 +215,10 @@ protected ReadResponse getReadResponse() { } } return readEntry(readResponse, entryId, startTimeSw); - } catch (Bookie.NoLedgerException e) { + } catch (Bookie.NoLedgerException | BookieException.LedgerFencedAndDeletedException e) { if (RequestUtils.isFenceRequest(readRequest)) { - LOG.info("No ledger found reading entry {} when fencing ledger {}", entryId, ledgerId); + LOG.info("No ledger found(or it has been deleted) reading entry {} when fencing ledger {}", + entryId, ledgerId); } else if (entryId != BookieProtocol.LAST_ADD_CONFIRMED) { LOG.info("No ledger found while reading entry: {} from ledger: {}", entryId, ledgerId); } else if (LOG.isDebugEnabled()) { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteEntryProcessor.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteEntryProcessor.java index d611ab963af..665e2d3a1c1 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteEntryProcessor.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteEntryProcessor.java @@ -90,8 +90,8 @@ protected void processPacket() { } catch (IOException e) { LOG.error("Error writing {}", request, e); rc = BookieProtocol.EIO; - } catch (BookieException.LedgerFencedException lfe) { - LOG.warn("Write attempt on fenced ledger {} by client {}", request.getLedgerId(), + } catch (BookieException.LedgerFencedException | BookieException.LedgerFencedAndDeletedException lfe) { + LOG.warn("Write attempt on fenced/deleted ledger {} by client {}", request.getLedgerId(), requestHandler.ctx().channel().remoteAddress()); rc = BookieProtocol.EFENCED; } catch (BookieException e) { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteEntryProcessorV3.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteEntryProcessorV3.java index 36aff7ad924..79d9f41d5c3 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteEntryProcessorV3.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteEntryProcessorV3.java @@ -136,8 +136,8 @@ public void writeComplete(int rc, long ledgerId, long entryId, logger.error("Error writing entry:{} to ledger:{}", entryId, ledgerId, e); status = StatusCode.EIO; - } catch (BookieException.LedgerFencedException e) { - logger.error("Ledger fenced while writing entry:{} to ledger:{}", + } catch (BookieException.LedgerFencedException | BookieException.LedgerFencedAndDeletedException e) { + logger.error("Ledger fenced/deleted while writing entry:{} to ledger:{}", entryId, ledgerId, e); status = StatusCode.EFENCED; } catch (BookieException e) { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteLacProcessorV3.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteLacProcessorV3.java index 293cea3bb0c..81d1bc66050 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteLacProcessorV3.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteLacProcessorV3.java @@ -105,6 +105,10 @@ public void writeComplete(int rc, long ledgerId, long entryId, BookieId addr, Ob requestProcessor.bookie.setExplicitLac(Unpooled.wrappedBuffer(lacToAdd), writeCallback, requestHandler, masterKey); status = StatusCode.EOK; + } catch (BookieException.LedgerFencedAndDeletedException e) { + logger.error("Error saving lac {} for ledger:{}, which has been deleted", + lac, ledgerId, e); + status = StatusCode.ENOLEDGER; } catch (IOException e) { logger.error("Error saving lac {} for ledger:{}", lac, ledgerId, e); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java index cf1f2f2f598..05807455dc4 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java @@ -61,6 +61,7 @@ import org.apache.bookkeeper.client.BookKeeperAdmin; import org.apache.bookkeeper.client.EnsemblePlacementPolicy; import org.apache.bookkeeper.client.LedgerChecker; +import org.apache.bookkeeper.client.LedgerEntry; import org.apache.bookkeeper.client.LedgerFragment; import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.client.api.LedgerMetadata; @@ -351,6 +352,10 @@ private boolean tryReadingFaultyEntries(LedgerHandle lh, LedgerFragment ledgerFr lh.asyncReadEntries(entryIdToRead, entryIdToRead, (rc, ledHan, seq, ctx) -> { long thisEntryId = (Long) ctx; if (rc == BKException.Code.OK) { + while (seq.hasMoreElements()) { + LedgerEntry entry = seq.nextElement(); + entry.getEntryBuffer().release(); + } entriesUnableToReadForThisLedger.remove(thisEntryId); if (numOfResponsesToWaitFor.decrementAndGet() == 0) { multiReadComplete.countDown(); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/BookieStateReadOnlyService.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/BookieStateReadOnlyService.java index d32074e2bb3..73f57c19598 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/BookieStateReadOnlyService.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/BookieStateReadOnlyService.java @@ -58,14 +58,14 @@ public HttpServiceResponse handle(HttpServiceRequest request) throws Exception { response.setBody("Bookie is in forceReadOnly mode, cannot transit to writable mode"); return response; } - stateManager.transitionToWritableMode().get(); + stateManager.transitionToWritableMode(true).get(); } else if (!stateManager.isReadOnly() && inState.isReadOnly()) { if (!stateManager.isReadOnlyModeEnabled()) { response.setCode(HttpServer.StatusCode.BAD_REQUEST); response.setBody("Bookie is disabled ReadOnly mode, cannot transit to readOnly mode"); return response; } - stateManager.transitionToReadOnlyMode().get(); + stateManager.transitionToReadOnlyMode(true).get(); } } else if (!HttpServer.Method.GET.equals(request.getMethod())) { response.setCode(HttpServer.StatusCode.NOT_FOUND); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/DeleteLedgerService.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/DeleteLedgerService.java index b4d465650c3..57cc127908d 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/DeleteLedgerService.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/DeleteLedgerService.java @@ -56,19 +56,20 @@ public HttpServiceResponse handle(HttpServiceRequest request) throws Exception { if (params != null && params.containsKey("ledger_id")) { ClientConfiguration clientConf = new ClientConfiguration(); clientConf.addConfiguration(conf); - BookKeeper bk = new BookKeeper(clientConf); - Long ledgerId = Long.parseLong(params.get("ledger_id")); + try (BookKeeper bk = new BookKeeper(clientConf)) { + Long ledgerId = Long.parseLong(params.get("ledger_id")); - bk.deleteLedger(ledgerId); + bk.deleteLedger(ledgerId); - String output = "Deleted ledger: " + ledgerId; - String jsonResponse = JsonUtil.toJson(output); - if (LOG.isDebugEnabled()) { - LOG.debug("output body:" + jsonResponse); + String output = "Deleted ledger: " + ledgerId; + String jsonResponse = JsonUtil.toJson(output); + if (LOG.isDebugEnabled()) { + LOG.debug("output body:" + jsonResponse); + } + response.setBody(jsonResponse); + response.setCode(HttpServer.StatusCode.OK); + return response; } - response.setBody(jsonResponse); - response.setCode(HttpServer.StatusCode.OK); - return response; } else { response.setCode(HttpServer.StatusCode.NOT_FOUND); response.setBody("Not ledger found. Should provide ledger_id="); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/zookeeper/ZooKeeperClient.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/zookeeper/ZooKeeperClient.java index c742f829e0f..3bddcb2f2ba 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/zookeeper/ZooKeeperClient.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/zookeeper/ZooKeeperClient.java @@ -241,9 +241,9 @@ public ZooKeeperClient build() throws IOException, KeeperException, InterruptedE // Create a watcher manager StatsLogger watcherStatsLogger = statsLogger.scope("watcher"); - ZooKeeperWatcherBase watcherManager = - null == watchers ? new ZooKeeperWatcherBase(sessionTimeoutMs, watcherStatsLogger) : - new ZooKeeperWatcherBase(sessionTimeoutMs, watchers, watcherStatsLogger); + ZooKeeperWatcherBase watcherManager = (null == watchers) + ? new ZooKeeperWatcherBase(sessionTimeoutMs, allowReadOnlyMode, watcherStatsLogger) + : new ZooKeeperWatcherBase(sessionTimeoutMs, allowReadOnlyMode, watchers, watcherStatsLogger); ZooKeeperClient client = new ZooKeeperClient( connectString, sessionTimeoutMs, diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/zookeeper/ZooKeeperWatcherBase.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/zookeeper/ZooKeeperWatcherBase.java index 758f079d0da..e44a5f364cd 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/zookeeper/ZooKeeperWatcherBase.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/zookeeper/ZooKeeperWatcherBase.java @@ -44,6 +44,7 @@ public class ZooKeeperWatcherBase implements Watcher { .getLogger(ZooKeeperWatcherBase.class); private final int zkSessionTimeOut; + private final boolean allowReadOnlyMode; private volatile CountDownLatch clientConnectLatch = new CountDownLatch(1); private final CopyOnWriteArraySet childWatchers = new CopyOnWriteArraySet(); @@ -53,18 +54,20 @@ public class ZooKeeperWatcherBase implements Watcher { private final ConcurrentHashMap eventCounters = new ConcurrentHashMap(); - public ZooKeeperWatcherBase(int zkSessionTimeOut) { - this(zkSessionTimeOut, NullStatsLogger.INSTANCE); + public ZooKeeperWatcherBase(int zkSessionTimeOut, boolean allowReadOnlyMode) { + this(zkSessionTimeOut, allowReadOnlyMode, NullStatsLogger.INSTANCE); } - public ZooKeeperWatcherBase(int zkSessionTimeOut, StatsLogger statsLogger) { - this(zkSessionTimeOut, new HashSet(), statsLogger); + public ZooKeeperWatcherBase(int zkSessionTimeOut, boolean allowReadOnlyMode, StatsLogger statsLogger) { + this(zkSessionTimeOut, allowReadOnlyMode, new HashSet(), statsLogger); } public ZooKeeperWatcherBase(int zkSessionTimeOut, + boolean allowReadOnlyMode, Set childWatchers, StatsLogger statsLogger) { this.zkSessionTimeOut = zkSessionTimeOut; + this.allowReadOnlyMode = allowReadOnlyMode; this.childWatchers.addAll(childWatchers); this.statsLogger = statsLogger; } @@ -130,6 +133,14 @@ public void process(WatchedEvent event) { LOG.info("ZooKeeper client is connected now."); clientConnectLatch.countDown(); break; + case ConnectedReadOnly: + if (allowReadOnlyMode) { + LOG.info("ZooKeeper client is connected in read-only mode now."); + clientConnectLatch.countDown(); + } else { + LOG.warn("ZooKeeper client is connected in read-only mode, which is not allowed."); + } + break; case Disconnected: LOG.info("ZooKeeper client is disconnected from zookeeper now," + " but it is OK unless we received EXPIRED event."); diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java index 5188eb816bb..bd0a12ca6cb 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java @@ -77,6 +77,7 @@ import org.apache.bookkeeper.common.component.ComponentStarter; import org.apache.bookkeeper.common.component.Lifecycle; import org.apache.bookkeeper.common.component.LifecycleComponent; +import org.apache.bookkeeper.common.util.JsonUtil; import org.apache.bookkeeper.conf.ClientConfiguration; import org.apache.bookkeeper.conf.ServerConfiguration; import org.apache.bookkeeper.conf.TestBKConfiguration; @@ -84,7 +85,10 @@ import org.apache.bookkeeper.discover.BookieServiceInfo.Endpoint; import org.apache.bookkeeper.discover.RegistrationManager; import org.apache.bookkeeper.http.HttpRouter; +import org.apache.bookkeeper.http.HttpServer; import org.apache.bookkeeper.http.HttpServerLoader; +import org.apache.bookkeeper.http.service.HttpServiceRequest; +import org.apache.bookkeeper.http.service.HttpServiceResponse; import org.apache.bookkeeper.meta.MetadataBookieDriver; import org.apache.bookkeeper.meta.exceptions.MetadataException; import org.apache.bookkeeper.meta.zk.ZKMetadataBookieDriver; @@ -96,6 +100,7 @@ import org.apache.bookkeeper.replication.ReplicationStats; import org.apache.bookkeeper.server.Main; import org.apache.bookkeeper.server.conf.BookieConfiguration; +import org.apache.bookkeeper.server.http.service.BookieStateReadOnlyService; import org.apache.bookkeeper.server.service.AutoRecoveryService; import org.apache.bookkeeper.server.service.BookieService; import org.apache.bookkeeper.stats.NullStatsLogger; @@ -107,8 +112,10 @@ import org.apache.bookkeeper.versioning.Version; import org.apache.bookkeeper.versioning.Versioned; import org.apache.bookkeeper.zookeeper.ZooKeeperClient; +import org.apache.commons.io.FileUtils; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.data.Stat; +import org.awaitility.Awaitility; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestName; @@ -1445,30 +1452,150 @@ public void testRetrieveBookieStatusWhenStatusFileIsCorrupted() throws Exception .setPersistBookieStatusEnabled(true) .setMetadataServiceUri(metadataServiceUri); // start a new bookie - BookieServer bookieServer = new BookieServer( + BookieServer bookieServer1 = new BookieServer( conf, new TestBookieImpl(conf), NullStatsLogger.INSTANCE, UnpooledByteBufAllocator.DEFAULT, new MockUncleanShutdownDetection()); - bookieServer.start(); + bookieServer1.start(); // transition in to read only and persist the status on disk - Bookie bookie = (BookieImpl) bookieServer.getBookie(); - assertFalse(bookie.isReadOnly()); - bookie.getStateManager().transitionToReadOnlyMode().get(); - assertTrue(bookie.isReadOnly()); + Bookie bookie1 = (BookieImpl) bookieServer1.getBookie(); + assertFalse(bookie1.isReadOnly()); + bookie1.getStateManager().transitionToReadOnlyMode().get(); + assertTrue(bookie1.isReadOnly()); // corrupt status file - List ledgerDirs = ((BookieImpl) bookie).getLedgerDirsManager().getAllLedgerDirs(); + List ledgerDirs = ((BookieImpl) bookie1).getLedgerDirsManager().getAllLedgerDirs(); corruptFile(new File(ledgerDirs.get(0), BOOKIE_STATUS_FILENAME)); corruptFile(new File(ledgerDirs.get(1), BOOKIE_STATUS_FILENAME)); // restart the bookie should be in read only mode - bookieServer.shutdown(); - bookieServer = new BookieServer( + bookieServer1.shutdown(); + BookieServer bookieServer2 = new BookieServer( conf, new TestBookieImpl(conf), NullStatsLogger.INSTANCE, UnpooledByteBufAllocator.DEFAULT, new MockUncleanShutdownDetection()); - bookieServer.start(); - bookie = bookieServer.getBookie(); - assertTrue(bookie.isReadOnly()); - bookieServer.shutdown(); + bookieServer2.start(); + BookieImpl bookie2 = (BookieImpl) bookieServer2.getBookie(); + assertTrue(bookie2.isReadOnly()); + // After a disk check, the bookie should switch to writable mode because the disk usage is fine. + bookie2.dirsMonitor.check(); + Awaitility.await().untilAsserted(() -> { + assertFalse(bookie2.isReadOnly()); + }); + bookieServer2.shutdown(); + } + + + private void setBookieToReadOnly(Bookie bookie, boolean readOnly) throws Exception { + BookieStateReadOnlyService.ReadOnlyState state = new BookieStateReadOnlyService.ReadOnlyState(); + state.setReadOnly(readOnly); + HttpServiceRequest request = new HttpServiceRequest(JsonUtil.toJson(state), HttpServer.Method.PUT, null); + BookieStateReadOnlyService service = new BookieStateReadOnlyService(bookie); + HttpServiceResponse response1 = service.handle(request); + assertEquals(HttpServer.StatusCode.OK.getValue(), response1.getStatusCode()); + } + + /** + * Verify: once the state is manually modified to read-only by Admin API, it should not be changed to writable + * by the monitor task. + * But it can be changed to read-only by monitor task if it was manually set to writable by Admin API. + */ + @Test(timeout = 1000 * 30) + public void testRetrieveBookieStatusAdminModifiedWhenStatusFileIsCorrupted() throws Exception { + File[] tmpLedgerDirs = new File[3]; + String[] filePath = new String[tmpLedgerDirs.length]; + for (int i = 0; i < tmpLedgerDirs.length; i++) { + tmpLedgerDirs[i] = tmpDirs.createNew("bookie", "test" + i); + filePath[i] = tmpLedgerDirs[i].getPath(); + } + final ServerConfiguration conf = TestBKConfiguration.newServerConfiguration(); + conf.setJournalDirName(filePath[0]) + .setLedgerDirNames(filePath) + .setReadOnlyModeEnabled(true) + .setPersistBookieStatusEnabled(true) + .setMetadataServiceUri(metadataServiceUri); + // start a new bookie + BookieServer bookieServer1 = new BookieServer( + conf, new TestBookieImpl(conf), + NullStatsLogger.INSTANCE, UnpooledByteBufAllocator.DEFAULT, + new MockUncleanShutdownDetection()); + bookieServer1.start(); + // transition in to read only and persist the status on disk + Bookie bookie1 = (BookieImpl) bookieServer1.getBookie(); + assertFalse(bookie1.isReadOnly()); + setBookieToReadOnly(bookie1, true); + assertTrue(bookie1.isReadOnly()); + // corrupt status file + List ledgerDirs = ((BookieImpl) bookie1).getLedgerDirsManager().getAllLedgerDirs(); + corruptFile(new File(ledgerDirs.get(0), BOOKIE_STATUS_FILENAME)); + corruptFile(new File(ledgerDirs.get(1), BOOKIE_STATUS_FILENAME)); + // restart the bookie should be in read only mode + bookieServer1.shutdown(); + BookieServer bookieServer2 = new BookieServer( + conf, new TestBookieImpl(conf), + NullStatsLogger.INSTANCE, UnpooledByteBufAllocator.DEFAULT, + new MockUncleanShutdownDetection()); + bookieServer2.start(); + BookieImpl bookie2 = (BookieImpl) bookieServer2.getBookie(); + assertTrue(bookie2.isReadOnly()); + // After a disk check, the bookie should not switch to writable mode because the state was set to read-only + // by admin api manually. + bookie2.dirsMonitor.check(); + Thread.sleep(2000); + Awaitility.await().untilAsserted(() -> { + assertTrue(bookie2.isReadOnly()); + }); + // We can use Admin Api to change the state back to writable. + setBookieToReadOnly(bookie2, false); + assertFalse(bookie2.isReadOnly()); + // The state can be changed to read-only by monitor task if it was manually set to writable by Admin API. + bookie2.getStateManager().transitionToReadOnlyMode().get(); + assertTrue(bookie2.isReadOnly()); + bookieServer2.shutdown(); + } + + /** + * Verify: the newest version can read the old version payload of persisted bookie status. + * old payload: "1,READ_ONLY,1752809349613" + * new payload: "1,READ_ONLY,1752809349613,false" + */ + @Test(timeout = 30000) + public void testPersistBookieStatusCompatibility() throws Exception { + File[] tmpLedgerDirs = new File[1]; + String[] filePath = new String[1]; + tmpLedgerDirs[0] = tmpDirs.createNew("bookie", "test"); + filePath[0] = tmpLedgerDirs[0].getPath(); + final ServerConfiguration conf = TestBKConfiguration.newServerConfiguration(); + conf.setJournalDirName(filePath[0]) + .setLedgerDirNames(filePath) + .setReadOnlyModeEnabled(true) + .setPersistBookieStatusEnabled(true) + .setMetadataServiceUri(metadataServiceUri); + // start a new bookie + BookieServer bookieServer1 = new BookieServer( + conf, new TestBookieImpl(conf), + NullStatsLogger.INSTANCE, UnpooledByteBufAllocator.DEFAULT, + new MockUncleanShutdownDetection()); + bookieServer1.start(); + Bookie bookie1 = (BookieImpl) bookieServer1.getBookie(); + // restart the bookie should be in read only mode + bookieServer1.shutdown(); + // Rewrite file to the old version payload. + List ledgerDirs = ((BookieImpl) bookie1).getLedgerDirsManager().getAllLedgerDirs(); + FileUtils.writeStringToFile(new File(ledgerDirs.get(0), BOOKIE_STATUS_FILENAME), + "1,READ_ONLY,1752809349613"); + BookieServer bookieServer2 = new BookieServer( + conf, new TestBookieImpl(conf), + NullStatsLogger.INSTANCE, UnpooledByteBufAllocator.DEFAULT, + new MockUncleanShutdownDetection()); + bookieServer2.start(); + BookieImpl bookie2 = (BookieImpl) bookieServer2.getBookie(); + assertTrue(bookie2.isReadOnly()); + // After a disk check, the bookie should not switch to writable mode because the state was set to read-only + // by admin api manually(the default value of previous version is true). + bookie2.dirsMonitor.check(); + Awaitility.await().untilAsserted(() -> { + assertTrue(bookie2.isReadOnly()); + }); + bookieServer2.shutdown(); } /** diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieJournalTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieJournalTest.java index a048bffc8a0..f489f3ffb93 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieJournalTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieJournalTest.java @@ -453,7 +453,7 @@ public void testV4Journal() throws Exception { } catch (Bookie.NoEntryException e) { // correct behaviour } - assertTrue(b.handles.getHandle(1, "testPasswd".getBytes()).isFenced()); + assertTrue(b.handles.getHandle(1, "testPasswd".getBytes(), false).isFenced()); b.shutdown(); } @@ -485,7 +485,7 @@ public void testV5Journal() throws Exception { } catch (Bookie.NoEntryException e) { // correct behavior } - assertTrue(b.handles.getHandle(1, "testV5Journal".getBytes()).isFenced()); + assertTrue(b.handles.getHandle(1, "testV5Journal".getBytes(), false).isFenced()); b.shutdown(); } diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/DefaultEntryLogTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/DefaultEntryLogTest.java index 38a9ebaf213..cbf12511126 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/DefaultEntryLogTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/DefaultEntryLogTest.java @@ -366,6 +366,46 @@ public void testRecoverFromLedgersMap() throws Exception { assertEquals(120, meta.getRemainingSize()); } + @Test + public void testLedgersMapIsEmpty() throws Exception { + // create some entries + entryLogger.addEntry(1L, generateEntry(1, 1).nioBuffer()); + entryLogger.addEntry(3L, generateEntry(3, 1).nioBuffer()); + entryLogger.addEntry(2L, generateEntry(2, 1).nioBuffer()); + entryLogger.addEntry(1L, generateEntry(1, 2).nioBuffer()); + ((EntryLogManagerBase) entryLogger.getEntryLogManager()).createNewLog(DefaultEntryLogger.UNASSIGNED_LEDGERID); + entryLogger.close(); + + // Rewrite the entry log header to be on V0 format + File f = new File(curDir, "0.log"); + RandomAccessFile raf = new RandomAccessFile(f, "rw"); + raf.seek(8); + // Mock that there is a ledgers map offset but the ledgers count is 0 + raf.writeLong(40); + raf.writeInt(0); + raf.close(); + + // now see which ledgers are in the log + entryLogger = new DefaultEntryLogger(conf, dirsMgr); + + try { + entryLogger.extractEntryLogMetadataFromIndex(0L); + fail("Should not be possible to recover from ledgers map index"); + } catch (IOException e) { + assertEquals("No ledgers map found in entryLogId 0, do scan to double confirm", e.getMessage()); + } + + // Public method should succeed by falling back to scanning the file + EntryLogMetadata meta = entryLogger.getEntryLogMetadata(0L); + LOG.info("Extracted Meta From Entry Log {}", meta); + assertEquals(60, meta.getLedgersMap().get(1L)); + assertEquals(30, meta.getLedgersMap().get(2L)); + assertEquals(30, meta.getLedgersMap().get(3L)); + assertFalse(meta.getLedgersMap().containsKey(4L)); + assertEquals(120, meta.getTotalSize()); + assertEquals(120, meta.getRemainingSize()); + } + /** * Explicitly try to recover using the ledgers map index at the end of the entry log. */ diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java index 4e8f06fe7f4..9e2090d7975 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java @@ -711,7 +711,7 @@ public void onSizeLimitReached(final CheckpointSource.Checkpoint cp) throws IOEx LOG.info("Started flushing mem table."); memTable.flush(FlushTestSortedLedgerStorage.this); } catch (IOException e) { - getStateManager().doTransitionToReadOnlyMode(); + getStateManager().doTransitionToReadOnlyMode(false); LOG.error("Exception thrown while flushing skip list cache.", e); } } diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageTest.java index 65f11e5d6a3..dfc2459678b 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageTest.java @@ -33,8 +33,10 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.util.List; +import java.util.Set; import org.apache.bookkeeper.bookie.Bookie; import org.apache.bookkeeper.bookie.Bookie.NoEntryException; import org.apache.bookkeeper.bookie.BookieException; @@ -52,6 +54,7 @@ import org.apache.bookkeeper.conf.TestBKConfiguration; import org.apache.bookkeeper.proto.BookieProtocol; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; @@ -822,4 +825,60 @@ public void testSingleLedgerDirectoryCheckpoint() throws Exception { assertEquals(7, logMark.getLogFileId()); assertEquals(8, logMark.getLogFileOffset()); } + + @Test + public void testSingleLedgerDirectoryCheckpointTriggerRemovePendingDeletedLedgers() + throws Exception { + int gcWaitTime = 1000; + File ledgerDir = new File(tmpDir, "dir"); + ServerConfiguration conf = TestBKConfiguration.newServerConfiguration(); + conf.setGcWaitTime(gcWaitTime); + conf.setProperty(DbLedgerStorage.WRITE_CACHE_MAX_SIZE_MB, 4); + conf.setProperty(DbLedgerStorage.READ_AHEAD_CACHE_MAX_SIZE_MB, 4); + conf.setLedgerStorageClass(DbLedgerStorage.class.getName()); + conf.setLedgerDirNames(new String[] { ledgerDir.getCanonicalPath() }); + + BookieImpl bookie = new TestBookieImpl(conf); + ByteBuf entry1 = Unpooled.buffer(1024); + entry1.writeLong(1); // ledger id + entry1.writeLong(2); // entry id + entry1.writeBytes("entry-1".getBytes()); + bookie.getLedgerStorage().addEntry(entry1); + + bookie.getJournals().get(0).getLastLogMark().getCurMark().setLogMark(1, 2); + ((DbLedgerStorage) bookie.getLedgerStorage()).getLedgerStorageList().get(0).flush(); + + File ledgerDirMark = new File(ledgerDir + "/current", "lastMark"); + try { + LogMark logMark = readLogMark(ledgerDirMark); + assertEquals(1, logMark.getLogFileId()); + assertEquals(2, logMark.getLogFileOffset()); + } catch (Exception e) { + fail(); + } + + ByteBuf entry2 = Unpooled.buffer(1024); + entry2.writeLong(2); // ledger id + entry2.writeLong(1); // entry id + entry2.writeBytes("entry-2".getBytes()); + + bookie.getLedgerStorage().addEntry(entry2); + // write one entry to first ledger directory and flush with logMark(1, 2), + // only the first ledger directory should have lastMark + bookie.getJournals().get(0).getLastLogMark().getCurMark().setLogMark(4, 5); + + SingleDirectoryDbLedgerStorage storage1 = + ((DbLedgerStorage) bookie.getLedgerStorage()).getLedgerStorageList().get(0); + Field field = SingleDirectoryDbLedgerStorage.class.getDeclaredField("ledgerIndex"); + field.setAccessible(true); + LedgerMetadataIndex ledgerMetadataIndex = (LedgerMetadataIndex) field.get(storage1); + Field field1 = LedgerMetadataIndex.class.getDeclaredField("pendingDeletedLedgers"); + field1.setAccessible(true); + Set pendingDeletedLedgers = (Set) field1.get(ledgerMetadataIndex); + + Assert.assertEquals(pendingDeletedLedgers.size(), 0); + pendingDeletedLedgers.add(2L); + bookie.getLedgerStorage().flush(); + Assert.assertEquals(pendingDeletedLedgers.size(), 0); + } } diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDBTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDBTest.java index 2ef3e010f8b..97be7ae7c78 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDBTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDBTest.java @@ -21,9 +21,11 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import java.io.File; +import java.io.IOException; import java.net.URL; import java.nio.file.Files; import java.nio.file.Paths; @@ -131,4 +133,19 @@ public void testLevelCompactionDynamicLevelBytesFromConfigurationFile() throws E ColumnFamilyOptions familyOptions = columnFamilyDescriptorList.get(0).getOptions(); assertEquals(true, familyOptions.levelCompactionDynamicLevelBytes()); } + + @Test + public void testCallCountAfterClose() throws IOException { + ServerConfiguration configuration = new ServerConfiguration(); + URL url = getClass().getClassLoader().getResource("test_entry_location_rocksdb.conf"); + configuration.setEntryLocationRocksdbConf(url.getPath()); + File tmpDir = Files.createTempDirectory("bk-kv-rocksdbtest-file").toFile(); + Files.createDirectory(Paths.get(tmpDir.toString(), "subDir")); + KeyValueStorageRocksDB rocksDB = new KeyValueStorageRocksDB(tmpDir.toString(), "subDir", + KeyValueStorageFactory.DbConfigType.EntryLocation, configuration); + assertNotNull(rocksDB.getColumnFamilyDescriptors()); + rocksDB.close(); + IOException exception = assertThrows(IOException.class, rocksDB::count); + assertEquals("RocksDB is closed", exception.getMessage()); + } } diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookKeeperClientZKSessionExpiry.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookKeeperClientZKSessionExpiry.java index b1a8bb66dd7..c72834397e0 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookKeeperClientZKSessionExpiry.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookKeeperClientZKSessionExpiry.java @@ -51,7 +51,7 @@ public void run() { byte[] sessionPasswd = bkc.getZkHandle().getSessionPasswd(); try { - ZooKeeperWatcherBase watcher = new ZooKeeperWatcherBase(10000); + ZooKeeperWatcherBase watcher = new ZooKeeperWatcherBase(10000, false); ZooKeeper zk = new ZooKeeper(zkUtil.getZooKeeperConnectString(), 10000, watcher, sessionId, sessionPasswd); zk.close(); diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookKeeperTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookKeeperTest.java index 8f64c256a43..386a83709e3 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookKeeperTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookKeeperTest.java @@ -901,7 +901,7 @@ protected ZooKeeper createZooKeeper() throws IOException { public void testZKConnectionLossForLedgerCreation() throws Exception { int zkSessionTimeOut = 10000; AtomicLong ledgerIdToInjectFailure = new AtomicLong(INVALID_LEDGERID); - ZooKeeperWatcherBase zooKeeperWatcherBase = new ZooKeeperWatcherBase(zkSessionTimeOut, + ZooKeeperWatcherBase zooKeeperWatcherBase = new ZooKeeperWatcherBase(zkSessionTimeOut, false, NullStatsLogger.INSTANCE); MockZooKeeperClient zkFaultInjectionWrapper = new MockZooKeeperClient(zkUtil.getZooKeeperConnectString(), zkSessionTimeOut, zooKeeperWatcherBase, ledgerIdToInjectFailure); diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestFencing.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestFencing.java index 34751525aa6..9f7a81450bb 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestFencing.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestFencing.java @@ -24,12 +24,29 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.UnpooledByteBufAllocator; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.bookie.Bookie; +import org.apache.bookkeeper.bookie.InterleavedLedgerStorage; +import org.apache.bookkeeper.bookie.LedgerStorage; +import org.apache.bookkeeper.bookie.SortedLedgerStorage; +import org.apache.bookkeeper.bookie.storage.ldb.SingleDirectoryDbLedgerStorage; import org.apache.bookkeeper.client.BookKeeper.DigestType; +import org.apache.bookkeeper.client.api.WriteFlag; import org.apache.bookkeeper.conf.ClientConfiguration; import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.proto.BookieClientImpl; +import org.apache.bookkeeper.proto.BookieProtocol; +import org.apache.bookkeeper.proto.BookkeeperInternalCallbacks; +import org.apache.bookkeeper.proto.PerChannelBookieClient; import org.apache.bookkeeper.test.BookKeeperClusterTestCase; +import org.apache.bookkeeper.test.TestStatsProvider; +import org.apache.bookkeeper.util.ByteBufList; +import org.awaitility.reflect.WhiteboxImpl; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,6 +55,7 @@ * This unit test tests ledger fencing. * */ +@Slf4j public class TestFencing extends BookKeeperClusterTestCase { private static final Logger LOG = LoggerFactory.getLogger(TestFencing.class); @@ -77,6 +95,7 @@ public void testBasicFencing() throws Exception { fail("Should have thrown an exception when trying to write"); } catch (BKException.BKLedgerFencedException e) { // correct behaviour + log.info("expected a fenced error", e); } /* @@ -87,6 +106,165 @@ public void testBasicFencing() throws Exception { readlh.getLastAddConfirmed() == writelh.getLastAddConfirmed()); } + @Test + public void testWriteAfterDeleted() throws Exception { + LedgerHandle writeLedger; + writeLedger = bkc.createLedger(digestType, "password".getBytes()); + + String tmp = "BookKeeper is cool!"; + for (int i = 0; i < 10; i++) { + long entryId = writeLedger.addEntry(tmp.getBytes()); + LOG.info("entryId: {}", entryId); + } + + // Fence and delete. + BookKeeperTestClient bkc2 = new BookKeeperTestClient(baseClientConf, new TestStatsProvider()); + LedgerHandle readLedger = bkc2.openLedger(writeLedger.getId(), digestType, "password".getBytes()); + bkc2.deleteLedger(readLedger.ledgerId); + + // Waiting for GC. + for (ServerTester server : servers) { + triggerGC(server.getServer().getBookie()); + } + + try { + long entryId = writeLedger.addEntry(tmp.getBytes()); + LOG.info("Not expected: entryId: {}", entryId); + LOG.error("Should have thrown an exception"); + fail("Should have thrown an exception when trying to write"); + } catch (BKException.BKLedgerFencedException e) { + log.info("expected a fenced error", e); + // correct behaviour + } + + /* + * Check it has been recovered properly. + */ + assertTrue("Has not recovered correctly: " + readLedger.getLastAddConfirmed() + + " original " + writeLedger.getLastAddConfirmed(), + readLedger.getLastAddConfirmed() == writeLedger.getLastAddConfirmed()); + + // cleanup. + bkc2.close(); + } + + private void triggerGC(Bookie bookie) { + LedgerStorage ledgerStorage = bookie.getLedgerStorage(); + if (ledgerStorage instanceof InterleavedLedgerStorage + || ledgerStorage instanceof SingleDirectoryDbLedgerStorage) { + Runnable gcThread = WhiteboxImpl.getInternalState(ledgerStorage, "gcThread"); + gcThread.run(); + } else if (ledgerStorage instanceof SortedLedgerStorage) { + Object actLedgerStorage = WhiteboxImpl.getInternalState(ledgerStorage, "interleavedLedgerStorage"); + Runnable gcThread = WhiteboxImpl.getInternalState(actLedgerStorage, "gcThread"); + gcThread.run(); + } + } + + @Test(timeout = 3000 * 1000) + public void testConcurrentFenceAndDeleteLedger() throws Exception { + LedgerHandle writeLedger; + writeLedger = bkc.createLedger(digestType, "password".getBytes()); + + String tmp = "BookKeeper is cool!"; + long lac = 0; + for (int i = 0; i < 10; i++) { + long entryId = writeLedger.addEntry(tmp.getBytes()); + LOG.info("entryId: {}", entryId); + lac = entryId; + } + + // Fence and delete. + final BookieId bookieId = writeLedger.getLedgerMetadata().getEnsembleAt(0).get(0); + ClientConfiguration clientConfiguration2 = newClientConfiguration(); + clientConfiguration2.setUseV2WireProtocol(true); + ClientConfiguration clientConfiguration3 = newClientConfiguration(); + BookKeeperTestClient bkcV2 = new BookKeeperTestClient(clientConfiguration2, new TestStatsProvider()); + LedgerHandle writeLedgerV2 = bkcV2.createLedger(digestType, "password".getBytes()); + BookKeeperTestClient bkcV3 = new BookKeeperTestClient(clientConfiguration3, new TestStatsProvider()); + LedgerHandle writeLedgerV3 = bkcV3.createLedger(digestType, "password".getBytes()); + ReadOnlyLedgerHandle readLedgerV2 = + (ReadOnlyLedgerHandle) bkcV2.openLedger(writeLedger.getId(), digestType, "password".getBytes()); + ReadOnlyLedgerHandle readLedgerV3 = + (ReadOnlyLedgerHandle) bkcV3.openLedger(writeLedger.getId(), digestType, "password".getBytes()); + BookieClientImpl bookieClientV2 = (BookieClientImpl) readLedgerV2.clientCtx.getBookieClient(); + BookieClientImpl bookieClientV3 = (BookieClientImpl) readLedgerV3.clientCtx.getBookieClient(); + // Trigger opening connection. + CompletableFuture obtainV2 = new CompletableFuture<>(); + bookieClientV2.lookupClient(bookieId).obtain( + new BookkeeperInternalCallbacks.GenericCallback() { + @Override + public void operationComplete(int rc, PerChannelBookieClient result) { + obtainV2.complete(rc); + } + }, writeLedger.getId()); + assertEquals(obtainV2.get().intValue(), BKException.Code.OK); + CompletableFuture obtainV3 = new CompletableFuture<>(); + bookieClientV3.lookupClient(bookieId).obtain( + new BookkeeperInternalCallbacks.GenericCallback() { + @Override + public void operationComplete(int rc, PerChannelBookieClient result) { + obtainV3.complete(rc); + } + }, writeLedger.getId()); + assertEquals(obtainV3.get().intValue(), BKException.Code.OK); + bkcV3.deleteLedger(readLedgerV3.ledgerId); + + // Waiting for GC. + for (ServerTester server : servers) { + triggerGC(server.getServer().getBookie()); + } + + // Verify: read requests with V2 protocol will receive a NoSuchLedgerException. + final byte readEntryFlagFencing = 1; + CompletableFuture readResV2 = new CompletableFuture<>(); + bookieClientV2.readEntry(bookieId, + writeLedger.getId(), 0, (rc, ledgerId, entryId1, buffer, ctx) -> { + readResV2.complete(rc); + }, null, readEntryFlagFencing, readLedgerV2.ledgerKey); + assertEquals(BKException.Code.NoSuchLedgerExistsException, readResV2.get().intValue()); + // Verify: read requests with V3 protocol will receive a NoSuchLedgerException. + CompletableFuture readResV3 = new CompletableFuture<>(); + bookieClientV3.readEntry(bookieId, + writeLedger.getId(), 0, (rc, ledgerId, entryId1, buffer, ctx) -> { + readResV3.complete(rc); + }, null, readEntryFlagFencing, readLedgerV3.ledgerKey); + assertEquals(BKException.Code.NoSuchLedgerExistsException, readResV3.get().intValue()); + // Verify: add requests with V2 protocol will receive a NoSuchLedgerException. + log.info("Try to add the next entry: {}:{}", writeLedger.getId(), lac + 1); + final ByteBuf dataV2 = UnpooledByteBufAllocator.DEFAULT.heapBuffer(); + // Combine add request, and rewrite ledgerId of the request. + dataV2.writeByte(1); + final ByteBuf toSendV2 = (ByteBuf) writeLedgerV2.macManager.computeDigestAndPackageForSending( + lac + 1, lac, 1, dataV2, writeLedger.ledgerKey, BookieProtocol.FLAG_NONE); + toSendV2.setLong(28, writeLedger.getId()); + CompletableFuture addResV2 = new CompletableFuture<>(); + bookieClientV2.addEntry(bookieId, writeLedger.getId(), writeLedger.ledgerKey, lac + 1, toSendV2, + (rc, ledgerId, entryId1, addr, ctx) -> { + addResV2.complete(rc); + }, null, BookieProtocol.FLAG_NONE, false, WriteFlag.NONE); + assertEquals(BKException.Code.LedgerFencedException, addResV2.get().intValue()); + // Verify: read requests with V3 protocol will receive a NoSuchLedgerException. + final ByteBuf dataV3 = UnpooledByteBufAllocator.DEFAULT.heapBuffer(); + dataV3.writeByte(1); + // Combine add request, and rewrite ledgerId of the request. + final ByteBufList toSendV3 = (ByteBufList) writeLedgerV3.macManager.computeDigestAndPackageForSending( + lac + 1, lac, 1, dataV3, writeLedger.ledgerKey, BookieProtocol.FLAG_NONE); + toSendV3.getBuffer(0).setLong(0, writeLedger.getId()); + CompletableFuture addResV3 = new CompletableFuture<>(); + bookieClientV3.addEntry(bookieId, writeLedger.getId(), writeLedger.ledgerKey, lac + 1, toSendV3, + (rc, ledgerId, entryId1, addr, ctx) -> { + addResV3.complete(rc); + }, null, BookieProtocol.FLAG_NONE, false, WriteFlag.NONE); + assertEquals(BKException.Code.LedgerFencedException, addResV3.get().intValue()); + + // cleanup. + writeLedgerV2.close(); + writeLedgerV3.close(); + bkcV2.close(); + bkcV3.close(); + } + private static int threadCount = 0; class LedgerOpenThread extends Thread { diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java index 4d0606f9021..0bda0f33306 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java @@ -29,6 +29,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import io.netty.util.HashedWheelTimer; import java.net.InetAddress; +import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -40,6 +41,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import junit.framework.TestCase; import org.apache.bookkeeper.client.BKException.BKNotEnoughBookiesException; @@ -62,6 +64,7 @@ import org.apache.bookkeeper.stats.Gauge; import org.apache.bookkeeper.stats.NullStatsLogger; import org.apache.bookkeeper.test.TestStatsProvider; +import org.apache.bookkeeper.test.TestStatsProvider.TestOpStatsLogger; import org.apache.bookkeeper.test.TestStatsProvider.TestStatsLogger; import org.apache.bookkeeper.util.StaticDNSResolver; import org.apache.commons.collections4.CollectionUtils; @@ -2270,6 +2273,106 @@ public void testNodeWithFailures() throws Exception { assertEquals(ensemble.get(reoderSet.get(3)), addr2.toBookieId()); assertEquals(ensemble.get(reoderSet.get(0)), addr3.toBookieId()); assertEquals(ensemble.get(reoderSet.get(1)), addr4.toBookieId()); + StaticDNSResolver.reset(); + } + + @Test + public void testSlowBookieInEnsembleOnly() throws Exception { + repp.uninitalize(); + updateMyRack("/r1/rack1"); + + TestStatsProvider statsProvider = new TestStatsProvider(); + TestStatsLogger statsLogger = statsProvider.getStatsLogger(""); + + repp = new RackawareEnsemblePlacementPolicy(); + repp.initialize(conf, Optional.empty(), timer, + DISABLE_ALL, statsLogger, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); + repp.withDefaultRack(NetworkTopology.DEFAULT_REGION_AND_RACK); + + TestOpStatsLogger readRequestsReorderedCounter = (TestOpStatsLogger) statsLogger + .getOpStatsLogger(BookKeeperClientStats.READ_REQUESTS_REORDERED); + + // Update cluster + Set addrs = new HashSet(); + addrs.add(addr1.toBookieId()); + addrs.add(addr2.toBookieId()); + addrs.add(addr3.toBookieId()); + addrs.add(addr4.toBookieId()); + repp.onClusterChanged(addrs, new HashSet()); + repp.registerSlowBookie(addr1.toBookieId(), 0L); + Map bookiePendingMap = new HashMap<>(); + bookiePendingMap.put(addr1.toBookieId(), 1L); + repp.onClusterChanged(addrs, new HashSet<>()); + + DistributionSchedule.WriteSet writeSet = writeSetFromValues(1, 2, 3); + + DistributionSchedule.WriteSet reorderSet = repp.reorderReadSequence( + ensemble, getBookiesHealthInfo(new HashMap<>(), bookiePendingMap), writeSet); + + // If the slow bookie is only present in the ensemble, no reordering occurs. + assertEquals(writeSet, reorderSet); + assertEquals(0, readRequestsReorderedCounter.getSuccessCount()); + } + + @Test + public void testReplaceNotAvailableBookieWithDefaultRack() throws Exception { + repp.uninitalize(); + repp.withDefaultRack(NetworkTopology.DEFAULT_RACK); + AtomicInteger counter = new AtomicInteger(); + BookieAddressResolver mockResolver = new BookieAddressResolver() { + @Override + public BookieSocketAddress resolve(BookieId bookieId) throws BookieIdNotResolvedException { + if (bookieId.equals(addr1.toBookieId()) && counter.getAndIncrement() >= 1) { + throw new BookieIdNotResolvedException(bookieId, + new RuntimeException(addr1.toBookieId() + " shutdown")); + } + try { + return new BookieSocketAddress(bookieId.toString()); + } catch (UnknownHostException err) { + throw new BookieIdNotResolvedException(bookieId, err); + } + } + }; + + repp.initialize(conf, Optional.empty(), timer, DISABLE_ALL, NullStatsLogger.INSTANCE, + mockResolver); + BookieSocketAddress addr1 = new BookieSocketAddress("127.0.0.2", 3181); + BookieSocketAddress addr2 = new BookieSocketAddress("127.0.0.3", 3181); + BookieSocketAddress addr3 = new BookieSocketAddress("127.0.0.4", 3181); + BookieSocketAddress addr4 = new BookieSocketAddress("127.0.0.5", 3181); + // update dns mapping + StaticDNSResolver.addNodeToRack(addr1.getHostName(), NetworkTopology.DEFAULT_RACK); + StaticDNSResolver.addNodeToRack(addr2.getHostName(), "/r1"); + StaticDNSResolver.addNodeToRack(addr3.getHostName(), "/r1"); + StaticDNSResolver.addNodeToRack(addr4.getHostName(), NetworkTopology.DEFAULT_RACK); + + // Update cluster + Set addrs = new HashSet(); + addrs.add(addr1.toBookieId()); + addrs.add(addr2.toBookieId()); + addrs.add(addr3.toBookieId()); + addrs.add(addr4.toBookieId()); + repp.onClusterChanged(addrs, new HashSet()); + + // replace node under r1 + EnsemblePlacementPolicy.PlacementResult replaceBookieResponse = + repp.replaceBookie(1, 1, 1, null, new ArrayList<>(), addr1.toBookieId(), new HashSet<>()); + BookieId replacedBookie = replaceBookieResponse.getResult(); + assertEquals(addr4.toBookieId(), replacedBookie); + + //clear history bookies and make addr1 shutdown. + repp = new RackawareEnsemblePlacementPolicy(); + repp.initialize(conf, Optional.empty(), timer, DISABLE_ALL, NullStatsLogger.INSTANCE, + mockResolver); + + addrs.remove(addr1.toBookieId()); + repp.onClusterChanged(addrs, new HashSet()); + + // replace node under r1 again + replaceBookieResponse = + repp.replaceBookie(1, 1, 1, null, new ArrayList<>(), addr1.toBookieId(), new HashSet<>()); + replacedBookie = replaceBookieResponse.getResult(); + assertEquals(addr4.toBookieId(), replacedBookie); } @Test diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRegionAwareEnsemblePlacementPolicy.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRegionAwareEnsemblePlacementPolicy.java index 9d8e36a350d..ca6c074b33e 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRegionAwareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRegionAwareEnsemblePlacementPolicy.java @@ -1066,7 +1066,7 @@ public void testEnsembleWithThreeRegionsReplaceDisableDurability() throws Except testEnsembleWithThreeRegionsReplaceInternal(1, true, false); } - public void testEnsembleWithThreeRegionsReplaceInternal(int minDurability, boolean disableDurability, + private void testEnsembleWithThreeRegionsReplaceInternal(int minDurability, boolean disableDurability, boolean disableOneRegion) throws Exception { repp.uninitalize(); repp = new RegionAwareEnsemblePlacementPolicy(); @@ -1208,7 +1208,7 @@ public void testEnsembleDisableDurability() throws Exception { testEnsembleDurabilityDisabledInternal(2, true); } - public void testEnsembleDurabilityDisabledInternal(int minDurability, boolean disableDurability) throws Exception { + private void testEnsembleDurabilityDisabledInternal(int minDurability, boolean disableDurability) throws Exception { repp.uninitalize(); repp = new RegionAwareEnsemblePlacementPolicy(); conf.setProperty(REPP_REGIONS_TO_WRITE, "region1;region2;region3"); @@ -1780,6 +1780,103 @@ public void testRegionsWithDiskWeight() throws Exception { assertEquals(3, ensemble.size()); } + @Test + public void testRegionsWithDifferentDiskWeight() throws Exception { + repp.uninitalize(); + repp = new RegionAwareEnsemblePlacementPolicy(); + conf.setProperty(REPP_ENABLE_VALIDATION, false); + conf.setDiskWeightBasedPlacementEnabled(true); + repp.initialize(conf, Optional.empty(), timer, DISABLE_ALL, + NullStatsLogger.INSTANCE, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); + BookieSocketAddress addr1 = new BookieSocketAddress("127.0.0.2", 3181); + BookieSocketAddress addr2 = new BookieSocketAddress("127.0.0.3", 3181); + BookieSocketAddress addr3 = new BookieSocketAddress("127.0.0.4", 3181); + BookieSocketAddress addr4 = new BookieSocketAddress("127.0.0.5", 3181); + BookieSocketAddress addr5 = new BookieSocketAddress("127.0.0.6", 3181); + + // update dns mapping + StaticDNSResolver.addNodeToRack(addr1.getHostName(), "/region1/r1"); + StaticDNSResolver.addNodeToRack(addr2.getHostName(), "/region1/r1"); + StaticDNSResolver.addNodeToRack(addr3.getHostName(), "/region2/r2"); + StaticDNSResolver.addNodeToRack(addr4.getHostName(), "/region2/r2"); + StaticDNSResolver.addNodeToRack(addr5.getHostName(), "/region2/r2"); + // Update cluster + Set addrs = new HashSet<>(); + addrs.add(addr1.toBookieId()); + addrs.add(addr2.toBookieId()); + addrs.add(addr3.toBookieId()); + addrs.add(addr4.toBookieId()); + addrs.add(addr5.toBookieId()); + repp.onClusterChanged(addrs, new HashSet<>()); + + // update bookie weight + // due to default BookieMaxWeightMultipleForWeightBasedPlacement=3, the test cases need to be in the range + Map bookieInfoMap = new HashMap<>(); + bookieInfoMap.put(addr1.toBookieId(), new BookieInfoReader.BookieInfo(1000000, 800000)); + bookieInfoMap.put(addr2.toBookieId(), new BookieInfoReader.BookieInfo(1000000, 400000)); + bookieInfoMap.put(addr3.toBookieId(), new BookieInfoReader.BookieInfo(1000000, 200000)); + bookieInfoMap.put(addr4.toBookieId(), new BookieInfoReader.BookieInfo(1000000, 300000)); + bookieInfoMap.put(addr5.toBookieId(), new BookieInfoReader.BookieInfo(1000000, 400000)); + repp.updateBookieInfo(bookieInfoMap); + + List ensemble; + Map countMap = new HashMap<>(); + addrs.forEach(a -> countMap.put(a, 0)); + int loopTimes = 5000; + for (int i = 0; i < loopTimes; ++i) { + ensemble = repp.newEnsemble(2, 2, 2, null, + new HashSet<>()).getResult(); + for (BookieId bookieId : ensemble) { + countMap.put(bookieId, countMap.get(bookieId) + 1); + } + } + + // c1 should be 2x than c2 + // c4 should be 1.5x than c3 + // c5 should be 2x than c3 + // we allow a range of (-50%, 50%) deviation instead of the exact multiples + int c1, c2, c3, c4, c5; + c1 = countMap.get(addr1.toBookieId()); + c2 = countMap.get(addr2.toBookieId()); + c3 = countMap.get(addr3.toBookieId()); + c4 = countMap.get(addr4.toBookieId()); + c5 = countMap.get(addr5.toBookieId()); + assertTrue(Math.abs((double) c1 / c2 - 2.0) < 1.0); + assertTrue(Math.abs((double) c4 / c3 - 1.5) < 1.0); + assertTrue(Math.abs((double) c5 / c3 - 2.0) < 1.0); + + // update bookie weight + // due to default BookieMaxWeightMultipleForWeightBasedPlacement=3, the test cases need to be in the range + bookieInfoMap.put(addr1.toBookieId(), new BookieInfoReader.BookieInfo(1000000, 400000)); + bookieInfoMap.put(addr2.toBookieId(), new BookieInfoReader.BookieInfo(1000000, 800000)); + bookieInfoMap.put(addr3.toBookieId(), new BookieInfoReader.BookieInfo(1000000, 400000)); + bookieInfoMap.put(addr4.toBookieId(), new BookieInfoReader.BookieInfo(1000000, 300000)); + bookieInfoMap.put(addr5.toBookieId(), new BookieInfoReader.BookieInfo(1000000, 200000)); + repp.updateBookieInfo(bookieInfoMap); + + addrs.forEach(a -> countMap.put(a, 0)); + for (int i = 0; i < loopTimes; ++i) { + ensemble = repp.newEnsemble(2, 2, 2, null, + new HashSet<>()).getResult(); + for (BookieId bookieId : ensemble) { + countMap.put(bookieId, countMap.get(bookieId) + 1); + } + } + + // c2 should be 2x than c1 + // c4 should be 1.5x than c5 + // c3 should be 2x than c5 + // we allow a range of (-50%, 50%) deviation instead of the exact multiples + c1 = countMap.get(addr1.toBookieId()); + c2 = countMap.get(addr2.toBookieId()); + c3 = countMap.get(addr3.toBookieId()); + c4 = countMap.get(addr4.toBookieId()); + c5 = countMap.get(addr5.toBookieId()); + assertTrue(Math.abs((double) c2 / c1 - 2.0) < 1.0); + assertTrue(Math.abs((double) c4 / c5 - 1.5) < 1.0); + assertTrue(Math.abs((double) c3 / c5 - 2.0) < 1.0); + } + @Test public void testNotifyRackChangeWithOldRegion() throws Exception { BookieSocketAddress addr1 = new BookieSocketAddress("127.0.1.1", 3181); @@ -1930,4 +2027,121 @@ public void testNotifyRackChangeWithNewRegion() throws Exception { assertEquals("region2", repp.address2Region.get(addr3.toBookieId())); assertEquals("region3", repp.address2Region.get(addr4.toBookieId())); } + + @Test + public void testBookieLeftThenJoinWithDNSResolveFailed() throws Exception { + + BookieSocketAddress addr1 = new BookieSocketAddress("127.0.1.1", 3181); + BookieSocketAddress addr2 = new BookieSocketAddress("127.0.1.2", 3181); + BookieSocketAddress addr3 = new BookieSocketAddress("127.0.1.3", 3181); + BookieSocketAddress addr4 = new BookieSocketAddress("127.0.1.4", 3181); + + // init dns mapping + // 2. mock dns resolver failed, use default region and rack. + // addr1 rack info. /region-1/default-rack -> /default-region/default-rack. + + // 1. mock addr1 dns resolver failed and use default region and rack. + StaticDNSResolver.addNodeToRack(addr1.getHostName(), "/default-region/default-rack"); + StaticDNSResolver.addNodeToRack(addr2.getHostName(), "/region-1/default-rack"); + StaticDNSResolver.addNodeToRack(addr3.getHostName(), "/region-2/default-rack"); + StaticDNSResolver.addNodeToRack(addr4.getHostName(), "/region-3/default-rack"); + + // init cluster + Set addrs = Sets.newHashSet(addr1.toBookieId(), + addr2.toBookieId(), addr3.toBookieId(), addr4.toBookieId()); + repp.onClusterChanged(addrs, new HashSet<>()); + + assertEquals(4, repp.knownBookies.size()); + assertEquals("/default-region/default-rack", repp.knownBookies.get(addr1.toBookieId()).getNetworkLocation()); + assertEquals("/region-1/default-rack", repp.knownBookies.get(addr2.toBookieId()).getNetworkLocation()); + assertEquals("/region-2/default-rack", repp.knownBookies.get(addr3.toBookieId()).getNetworkLocation()); + assertEquals("/region-3/default-rack", repp.knownBookies.get(addr4.toBookieId()).getNetworkLocation()); + + assertEquals(4, repp.perRegionPlacement.size()); + TopologyAwareEnsemblePlacementPolicy unknownRegionPlacement = repp.perRegionPlacement.get("UnknownRegion"); + assertEquals(1, unknownRegionPlacement.knownBookies.keySet().size()); + assertEquals("/default-region/default-rack", + unknownRegionPlacement.knownBookies.get(addr1.toBookieId()).getNetworkLocation()); + + TopologyAwareEnsemblePlacementPolicy region1Placement = repp.perRegionPlacement.get("region-1"); + assertEquals(1, region1Placement.knownBookies.keySet().size()); + assertEquals("/region-1/default-rack", + region1Placement.knownBookies.get(addr2.toBookieId()).getNetworkLocation()); + + TopologyAwareEnsemblePlacementPolicy region2Placement = repp.perRegionPlacement.get("region-2"); + assertEquals(1, region2Placement.knownBookies.keySet().size()); + assertEquals("/region-2/default-rack", + region2Placement.knownBookies.get(addr3.toBookieId()).getNetworkLocation()); + + TopologyAwareEnsemblePlacementPolicy region3Placement = repp.perRegionPlacement.get("region-3"); + assertEquals(1, region3Placement.knownBookies.keySet().size()); + assertEquals("/region-3/default-rack", + region3Placement.knownBookies.get(addr4.toBookieId()).getNetworkLocation()); + + assertEquals("UnknownRegion", repp.address2Region.get(addr1.toBookieId())); + assertEquals("region-1", repp.address2Region.get(addr2.toBookieId())); + assertEquals("region-2", repp.address2Region.get(addr3.toBookieId())); + assertEquals("region-3", repp.address2Region.get(addr4.toBookieId())); + + // 2. addr1 bookie shutdown and decommission + addrs.remove(addr1.toBookieId()); + repp.onClusterChanged(addrs, new HashSet<>()); + + assertEquals(3, repp.knownBookies.size()); + assertNull(repp.knownBookies.get(addr1.toBookieId())); + assertEquals("/region-1/default-rack", repp.knownBookies.get(addr2.toBookieId()).getNetworkLocation()); + assertEquals("/region-2/default-rack", repp.knownBookies.get(addr3.toBookieId()).getNetworkLocation()); + assertEquals("/region-3/default-rack", repp.knownBookies.get(addr4.toBookieId()).getNetworkLocation()); + + // UnknownRegion,region-1,region-2,region-3 + assertEquals(4, repp.perRegionPlacement.size()); + // after addr1 bookie left, it should remove from locally address2Region + assertNull(repp.address2Region.get(addr1.toBookieId())); + assertEquals("region-1", repp.address2Region.get(addr2.toBookieId())); + assertEquals("region-2", repp.address2Region.get(addr3.toBookieId())); + assertEquals("region-3", repp.address2Region.get(addr4.toBookieId())); + + + // 3. addr1 bookie start and join + addrs.add(addr1.toBookieId()); + repp.onClusterChanged(addrs, new HashSet<>()); + + assertEquals(4, repp.knownBookies.size()); + assertEquals("/default-region/default-rack", repp.knownBookies.get(addr1.toBookieId()).getNetworkLocation()); + assertEquals("/region-1/default-rack", repp.knownBookies.get(addr2.toBookieId()).getNetworkLocation()); + assertEquals("/region-2/default-rack", repp.knownBookies.get(addr3.toBookieId()).getNetworkLocation()); + assertEquals("/region-3/default-rack", repp.knownBookies.get(addr4.toBookieId()).getNetworkLocation()); + + // UnknownRegion,region-1,region-2,region-3 + assertEquals(4, repp.perRegionPlacement.size()); + assertEquals("UnknownRegion", repp.address2Region.get(addr1.toBookieId())); + // addr1 bookie belongs to unknown region + unknownRegionPlacement = repp.perRegionPlacement.get("UnknownRegion"); + assertEquals(1, unknownRegionPlacement.knownBookies.keySet().size()); + assertEquals("/default-region/default-rack", + unknownRegionPlacement.knownBookies.get(addr1.toBookieId()).getNetworkLocation()); + + // 4. Update the correct rack. + // change addr1 rack info. /default-region/default-rack -> /region-1/default-rack. + List bookieAddressList = new ArrayList<>(); + List rackList = new ArrayList<>(); + bookieAddressList.add(addr1); + rackList.add("/region-1/default-rack"); + // onBookieRackChange + StaticDNSResolver.changeRack(bookieAddressList, rackList); + + assertEquals(4, repp.perRegionPlacement.size()); + // addr1 bookie, oldRegion=default-region, newRegion=region-1 + assertEquals("region-1", repp.address2Region.get(addr1.toBookieId())); + + unknownRegionPlacement = repp.perRegionPlacement.get("UnknownRegion"); + assertEquals(0, unknownRegionPlacement.knownBookies.keySet().size()); + assertNotNull(unknownRegionPlacement.historyBookies.get(addr1.toBookieId())); + + + region1Placement = repp.perRegionPlacement.get("region-1"); + assertEquals(2, region1Placement.knownBookies.keySet().size()); + assertEquals("/region-1/default-rack", + region1Placement.knownBookies.get(addr1.toBookieId()).getNetworkLocation()); + } } diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/conf/AbstractConfigurationTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/conf/AbstractConfigurationTest.java index a6333a47d32..194ab2c68d0 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/conf/AbstractConfigurationTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/conf/AbstractConfigurationTest.java @@ -19,6 +19,8 @@ package org.apache.bookkeeper.conf; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.CALLS_REAL_METHODS; import static org.mockito.Mockito.mock; @@ -179,4 +181,12 @@ public void testAllocatorLeakDetectionPolicy() { System.getProperties().put(nettyLevelKey, nettyLevelStr); } } + + @Test + public void testExitOnOutOfMemory() { + assertFalse(conf.exitOnOutOfMemory()); + conf.setExitOnOutOfMemory(true); + assertTrue(conf.exitOnOutOfMemory()); + } + } diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/conf/TestServerConfiguration.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/conf/TestServerConfiguration.java index 04ac87818f7..d8aa62d0d23 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/conf/TestServerConfiguration.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/conf/TestServerConfiguration.java @@ -158,6 +158,7 @@ public void testEntryLogSizeLimit() throws ConfigurationException { public void testCompactionSettings() throws ConfigurationException { ServerConfiguration conf = new ServerConfiguration(); long major, minor; + long entryLocationCompactionInterval; // Default Values major = conf.getMajorCompactionMaxTimeMillis(); @@ -239,5 +240,24 @@ public void testCompactionSettings() throws ConfigurationException { minorThreshold = conf.getMinorCompactionThreshold(); Assert.assertEquals(0.6, majorThreshold, 0.00001); Assert.assertEquals(0.3, minorThreshold, 0.00001); + + // Default Values + entryLocationCompactionInterval = conf.getEntryLocationCompactionInterval(); + Assert.assertEquals(-1, entryLocationCompactionInterval); + + // Set entry location compaction + conf.setEntryLocationCompactionInterval(3600); + entryLocationCompactionInterval = conf.getEntryLocationCompactionInterval(); + Assert.assertEquals(3600, entryLocationCompactionInterval); + + conf.setEntryLocationCompactionInterval(550); + try { + conf.validate(); + fail(); + } catch (ConfigurationException ignore) { + } + + conf.setEntryLocationCompactionInterval(650); + conf.validate(); } } diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/proto/ClientSocketDisconnectTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/proto/ClientSocketDisconnectTest.java new file mode 100644 index 00000000000..709ed771ddc --- /dev/null +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/proto/ClientSocketDisconnectTest.java @@ -0,0 +1,144 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.bookkeeper.proto; + +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; +import io.netty.util.concurrent.DefaultThreadFactory; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.client.api.BookKeeper; +import org.apache.bookkeeper.client.api.DigestType; +import org.apache.bookkeeper.conf.ClientConfiguration; +import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.test.BookKeeperClusterTestCase; +import org.apache.bookkeeper.tls.SecurityException; +import org.apache.bookkeeper.util.EventLoopUtil; +import org.junit.Assert; +import org.junit.Test; + +@Slf4j +public class ClientSocketDisconnectTest extends BookKeeperClusterTestCase { + + public ClientSocketDisconnectTest() { + super(1); + this.useUUIDasBookieId = true; + } + + public static class PerChannelBookieClientDecorator extends PerChannelBookieClient { + + private final ThreadCounter threadCounter; + private final AtomicInteger failurePredicate = new AtomicInteger(); + + public PerChannelBookieClientDecorator(PerChannelBookieClient client, BookieId addr, ThreadCounter tCounter) + throws SecurityException { + super(client.executor, client.eventLoopGroup, addr, client.bookieAddressResolver); + this.threadCounter = tCounter; + } + + // Inject a disconnection per two connections. + protected void addChannelListeners(ChannelFuture future, long connectStartTime) { + future.addListener((ChannelFutureListener) future1 -> { + if (failurePredicate.incrementAndGet() % 2 == 1) { + future1.channel().close(); + } + }); + super.addChannelListeners(future, connectStartTime); + } + + // Records the thread who running "PendingAddOp.writeComplete". + @Override + protected void connectIfNeededAndDoOp(BookkeeperInternalCallbacks.GenericCallback op) { + BookieClientImpl.ChannelReadyForAddEntryCallback callback = + (BookieClientImpl.ChannelReadyForAddEntryCallback) op; + BookkeeperInternalCallbacks.WriteCallback originalCallback = callback.cb; + callback.cb = (rc, ledgerId, entryId, addr, ctx) -> { + threadCounter.record(); + originalCallback.writeComplete(rc, ledgerId, entryId, addr, ctx); + }; + super.connectIfNeededAndDoOp(op); + } + } + + private static class ThreadCounter { + + private final Map records = new ConcurrentHashMap<>(); + + public void record() { + Thread currentThread = Thread.currentThread(); + records.computeIfAbsent(currentThread, k -> new AtomicInteger()); + records.get(currentThread).incrementAndGet(); + } + } + + @Test + public void testAddEntriesCallbackWithBKClientThread() throws Exception { + // Create BKC and a ledger handle. + ClientConfiguration conf = new ClientConfiguration(); + conf.setMetadataServiceUri(zkUtil.getMetadataServiceUri()); + org.apache.bookkeeper.client.BookKeeper bkc = + (org.apache.bookkeeper.client.BookKeeper) BookKeeper.newBuilder(conf) + .eventLoopGroup( + EventLoopUtil.getClientEventLoopGroup(conf, new DefaultThreadFactory("test-io"))) + .build(); + final BookieClientImpl bookieClient = (BookieClientImpl) bkc.getClientCtx().getBookieClient(); + LedgerHandle lh = (LedgerHandle) bkc.newCreateLedgerOp() + .withEnsembleSize(1) + .withWriteQuorumSize(1) + .withAckQuorumSize(1) + .withDigestType(DigestType.CRC32C) + .withPassword(new byte[0]) + .execute().join(); + + // Inject two operations. + // 1. Inject a disconnection when connecting successfully. + // 2. Records the thread who running "PendingAddOp.writeComplete". + final ThreadCounter callbackThreadRecorder = new ThreadCounter(); + List ensemble = lh.getLedgerMetadata() + .getAllEnsembles().entrySet().iterator().next().getValue(); + DefaultPerChannelBookieClientPool clientPool = + (DefaultPerChannelBookieClientPool) bookieClient.lookupClient(ensemble.get(0)); + PerChannelBookieClient[] clients = clientPool.clients; + + // Write 100 entries and wait for finishing. + for (int i = 0; i < clients.length; i++) { + clients[i] = new PerChannelBookieClientDecorator(clients[i], ensemble.get(0), callbackThreadRecorder); + } + int addCount = 1000; + CountDownLatch countDownLatch = new CountDownLatch(addCount); + for (int i = 0; i < addCount; i++) { + lh.asyncAddEntry(new byte[]{1}, (rc, lh1, entryId, ctx) -> { + countDownLatch.countDown(); + }, i); + } + countDownLatch.await(); + + // Verify: all callback will run in the "BookKeeperClientWorker" thread. + for (Thread callbackThread : callbackThreadRecorder.records.keySet()) { + Assert.assertTrue(callbackThread.getName(), callbackThread.getName().startsWith("BookKeeperClientWorker")); + } + } +} \ No newline at end of file diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java index 2e3e09012fb..bbeb7e758fd 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java @@ -326,7 +326,7 @@ public void testReadOnlyBookieExclusionFromURLedgersCheck() throws Exception { BookieServer bk = serverByIndex(bkIndex); bookieConf.setReadOnlyModeEnabled(true); - ((BookieImpl) bk.getBookie()).getStateManager().doTransitionToReadOnlyMode(); + ((BookieImpl) bk.getBookie()).getStateManager().doTransitionToReadOnlyMode(false); bkc.waitForReadOnlyBookie(BookieImpl.getBookieId(confByIndex(bkIndex))) .get(30, TimeUnit.SECONDS); @@ -361,7 +361,7 @@ public void testReadOnlyBookieShutdown() throws Exception { BookieServer bk = serverByIndex(bkIndex); bookieConf.setReadOnlyModeEnabled(true); - ((BookieImpl) bk.getBookie()).getStateManager().doTransitionToReadOnlyMode(); + ((BookieImpl) bk.getBookie()).getStateManager().doTransitionToReadOnlyMode(false); bkc.waitForReadOnlyBookie(BookieImpl.getBookieId(confByIndex(bkIndex))) .get(30, TimeUnit.SECONDS); diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java index a9914233899..28f9915cb2c 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java @@ -1040,7 +1040,7 @@ public void testRWShutDownInTheCaseOfZKOperationFailures() throws Exception { * create MockZooKeeperClient instance and wait for it to be connected. */ int zkSessionTimeOut = 10000; - ZooKeeperWatcherBase zooKeeperWatcherBase = new ZooKeeperWatcherBase(zkSessionTimeOut, + ZooKeeperWatcherBase zooKeeperWatcherBase = new ZooKeeperWatcherBase(zkSessionTimeOut, false, NullStatsLogger.INSTANCE); MockZooKeeperClient zkFaultInjectionWrapper = new MockZooKeeperClient(zkUtil.getZooKeeperConnectString(), zkSessionTimeOut, zooKeeperWatcherBase); diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/ZooKeeperCluster.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/ZooKeeperCluster.java index 08ecbd7cc12..b0e828bd5ca 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/ZooKeeperCluster.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/ZooKeeperCluster.java @@ -64,7 +64,7 @@ void sleepCluster(int time, TimeUnit timeUnit, CountDownLatch l) default void expireSession(ZooKeeper zk) throws Exception { long id = zk.getSessionId(); byte[] password = zk.getSessionPasswd(); - ZooKeeperWatcherBase w = new ZooKeeperWatcherBase(10000); + ZooKeeperWatcherBase w = new ZooKeeperWatcherBase(10000, false); ZooKeeper zk2 = new ZooKeeper(getZooKeeperConnectString(), zk.getSessionTimeout(), w, id, password); w.waitForConnection(); zk2.close(); diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/ZooKeeperClusterUtil.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/ZooKeeperClusterUtil.java index 3eace4a62c5..6dbf182110f 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/ZooKeeperClusterUtil.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/ZooKeeperClusterUtil.java @@ -139,4 +139,12 @@ public void killCluster() throws Exception { public void sleepCluster(int time, TimeUnit timeUnit, CountDownLatch l) throws InterruptedException, IOException { throw new UnsupportedOperationException("sleepServer operation is not supported for ZooKeeperClusterUtil"); } + + public void stopPeer(int id) throws Exception { + quorumUtil.shutdown(id); + } + + public void enableLocalSession(boolean localSessionEnabled) { + quorumUtil.enableLocalSession(localSessionEnabled); + } } diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/zookeeper/TestZooKeeperClient.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/zookeeper/TestZooKeeperClient.java index 3c1bd65a64b..9e2ccf16e87 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/zookeeper/TestZooKeeperClient.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/zookeeper/TestZooKeeperClient.java @@ -171,7 +171,7 @@ public void process(WatchedEvent event) { }; final int timeout = 2000; ZooKeeperWatcherBase watcherManager = - new ZooKeeperWatcherBase(timeout).addChildWatcher(testWatcher); + new ZooKeeperWatcherBase(timeout, false).addChildWatcher(testWatcher); List watchers = new ArrayList(1); watchers.add(testWatcher); ZooKeeperClient client = new ShutdownZkServerClient( @@ -895,4 +895,32 @@ public void processResult(int rc, String path, Object ctx) { logger.info("Delete children from znode " + path); } + @Test + public void testAllowReadOnlyMode() throws Exception { + if (zkUtil instanceof ZooKeeperClusterUtil) { + System.setProperty("readonlymode.enabled", "true"); + ((ZooKeeperClusterUtil) zkUtil).enableLocalSession(true); + zkUtil.restartCluster(); + Thread.sleep(2000); + ((ZooKeeperClusterUtil) zkUtil).stopPeer(2); + ((ZooKeeperClusterUtil) zkUtil).stopPeer(3); + } + + try (ZooKeeperClient client = ZooKeeperClient.newBuilder() + .connectString(zkUtil.getZooKeeperConnectString()) + .sessionTimeoutMs(30000) + .watchers(new HashSet()) + .operationRetryPolicy(retryPolicy) + .allowReadOnlyMode(true) + .build()) { + Assert.assertTrue("Client failed to connect a ZooKeeper in read-only mode.", + client.getState().isConnected()); + } finally { + if (zkUtil instanceof ZooKeeperClusterUtil) { + System.setProperty("readonlymode.enabled", "false"); + ((ZooKeeperClusterUtil) zkUtil).enableLocalSession(false); + } + } + } + } diff --git a/bookkeeper-server/src/test/resources/conf/default_rocksdb.conf b/bookkeeper-server/src/test/resources/conf/default_rocksdb.conf index 0f3a08779ed..b7b0ae677dd 100644 --- a/bookkeeper-server/src/test/resources/conf/default_rocksdb.conf +++ b/bookkeeper-server/src/test/resources/conf/default_rocksdb.conf @@ -23,7 +23,14 @@ info_log_level=INFO_LEVEL # set by jni: options.setKeepLogFileNum keep_log_file_num=30 - -[CFOptions "default"] # set by jni: options.setLogFileTimeToRoll - log_file_time_to_roll=86400 \ No newline at end of file + log_file_time_to_roll=86400 + + [CFOptions "default"] + #no default setting in CFOptions + +[TableOptions/BlockBasedTable "default"] + # set by jni: tableOptions.setFormatVersion + format_version=5 + # set by jni: tableOptions.setChecksumType + checksum=kxxHash \ No newline at end of file diff --git a/bookkeeper-server/src/test/resources/conf/entry_location_rocksdb.conf b/bookkeeper-server/src/test/resources/conf/entry_location_rocksdb.conf index 6f6c1b4d052..976e53b47b7 100644 --- a/bookkeeper-server/src/test/resources/conf/entry_location_rocksdb.conf +++ b/bookkeeper-server/src/test/resources/conf/entry_location_rocksdb.conf @@ -26,7 +26,7 @@ # set by jni: options.setLogFileTimeToRoll log_file_time_to_roll=86400 # set by jni: options.setMaxBackgroundJobs or options.setIncreaseParallelism - max_background_jobs=2 + max_background_jobs=32 # set by jni: options.setMaxSubcompactions max_subcompactions=1 # set by jni: options.setMaxTotalWalSize @@ -57,10 +57,10 @@ [TableOptions/BlockBasedTable "default"] # set by jni: tableOptions.setBlockSize block_size=65536 - # set by jni: tableOptions.setBlockCache +# set by jni: tableOptions.setBlockCache, default value is: maxDirectMemory() / ledgerDirsSize / 10; block_cache=206150041 # set by jni: tableOptions.setFormatVersion - format_version=2 + format_version=5 # set by jni: tableOptions.setChecksumType checksum=kxxHash # set by jni: tableOptions.setFilterPolicy, bloomfilter:[bits_per_key]:[use_block_based_builder] diff --git a/bookkeeper-server/src/test/resources/conf/ledger_metadata_rocksdb.conf b/bookkeeper-server/src/test/resources/conf/ledger_metadata_rocksdb.conf index 0f3a08779ed..b7b0ae677dd 100644 --- a/bookkeeper-server/src/test/resources/conf/ledger_metadata_rocksdb.conf +++ b/bookkeeper-server/src/test/resources/conf/ledger_metadata_rocksdb.conf @@ -23,7 +23,14 @@ info_log_level=INFO_LEVEL # set by jni: options.setKeepLogFileNum keep_log_file_num=30 - -[CFOptions "default"] # set by jni: options.setLogFileTimeToRoll - log_file_time_to_roll=86400 \ No newline at end of file + log_file_time_to_roll=86400 + + [CFOptions "default"] + #no default setting in CFOptions + +[TableOptions/BlockBasedTable "default"] + # set by jni: tableOptions.setFormatVersion + format_version=5 + # set by jni: tableOptions.setChecksumType + checksum=kxxHash \ No newline at end of file diff --git a/bookkeeper-server/src/test/resources/test_entry_location_rocksdb.conf b/bookkeeper-server/src/test/resources/test_entry_location_rocksdb.conf index 9d1c3e08c47..7640fd2c9e8 100644 --- a/bookkeeper-server/src/test/resources/test_entry_location_rocksdb.conf +++ b/bookkeeper-server/src/test/resources/test_entry_location_rocksdb.conf @@ -19,10 +19,22 @@ [DBOptions] # set by jni: options.setCreateIfMissing create_if_missing=true + # set by jni: options.setInfoLogLevel + info_log_level=INFO_LEVEL # set by jni: options.setKeepLogFileNum keep_log_file_num=1 + # set by jni: options.setLogFileTimeToRoll + log_file_time_to_roll=86400 + # set by jni: options.setMaxBackgroundJobs or options.setIncreaseParallelism + max_background_jobs=32 + # set by jni: options.setMaxSubcompactions + max_subcompactions=1 # set by jni: options.setMaxTotalWalSize max_total_wal_size=1000 + # set by jni: options.setMaxOpenFiles + max_open_files=-1 + # set by jni: options.setDeleteObsoleteFilesPeriodMicros + delete_obsolete_files_period_micros=3600000000 [CFOptions "default"] # set by jni: options.setCompressionType @@ -31,16 +43,24 @@ write_buffer_size=1024 # set by jni: options.setMaxWriteBufferNumber max_write_buffer_number=1 + # set by jni: options.setNumLevels + num_levels=7 + # set by jni: options.setLevelZeroFileNumCompactionTrigger + level0_file_num_compaction_trigger=4 + # set by jni: options.setMaxBytesForLevelBase + max_bytes_for_level_base=268435456 + # set by jni: options.setTargetFileSizeBase + target_file_size_base=67108864 # set by jni: options.setLevelCompactionDynamicLevelBytes level_compaction_dynamic_level_bytes=true [TableOptions/BlockBasedTable "default"] # set by jni: tableOptions.setBlockSize block_size=65536 - # set by jni: tableOptions.setBlockCache +# set by jni: tableOptions.setBlockCache, default value is: maxDirectMemory() / ledgerDirsSize / 10; block_cache=206150041 # set by jni: tableOptions.setFormatVersion - format_version=2 + format_version=5 # set by jni: tableOptions.setChecksumType checksum=kxxHash # set by jni: tableOptions.setFilterPolicy, bloomfilter:[bits_per_key]:[use_block_based_builder] diff --git a/conf/bk_server.conf b/conf/bk_server.conf index c0a021418d9..8a475c2fae2 100755 --- a/conf/bk_server.conf +++ b/conf/bk_server.conf @@ -581,6 +581,11 @@ ledgerDirectories=/tmp/bk-data # Set the rate at which compaction will readd entries. The unit is bytes added per second. # compactionRateByBytes=1000000 +# Interval to run entry location compaction, in seconds +# If it is set to less than zero, the entry location compaction is disabled. +# Note: should be greater than gcWaitTime. +# entryLocationCompactionInterval=-1 + # Flag to enable/disable transactional compaction. If it is set to true, it will use transactional compaction, # which it will use new entry log files to store compacted entries during compaction; if it is set to false, # it will use normal compaction, which it shares same entry log file with normal add operations. @@ -755,17 +760,31 @@ gcEntryLogMetadataCacheEnabled=false # How many entries to pre-fill in cache after a read cache miss # dbStorage_readAheadCacheBatchSize=100 +############################################################################# ## RocksDB specific configurations +############################################################################# ## DbLedgerStorage uses RocksDB to store the indexes from -## (ledgerId, entryId) -> (entryLog, offset) - +## (ledgerId, entryId) -> (entryLog, offset). This is called the entry location database. +## There's also a separate RocksDB database to to store the ledger metadata. +# +# RocksDB configuration can be applied in two ways: by using a configuration file +# or by setting individual properties. +# +# Most individual properties are applied only to the entry location database. +# To fine-tune the ledger metadata database, the configuration file method should be used. +# +# These properties apply to both entry location and ledger metadata databases +############################################################################# +# dbStorage_rocksDB_format_version=5 +# dbStorage_rocksDB_checksum_type=kxxHash +############################################################################# +# Entry location RocksDB database specific configurations: # Size of RocksDB block-cache. For best performance, this cache # should be big enough to hold a significant portion of the index # database which can reach ~2GB in some cases # Default is to use 10% / numberOfLedgers of the direct memory size # dbStorage_rocksDB_blockCacheSize= - -# Other RocksDB specific tunables +# Other RocksDB specific tunables for the entry location database # dbStorage_rocksDB_writeBufferSizeMB=64 # dbStorage_rocksDB_sstSizeInMB=64 # dbStorage_rocksDB_blockSize=65536 @@ -774,7 +793,12 @@ gcEntryLogMetadataCacheEnabled=false # dbStorage_rocksDB_numFilesInLevel0=4 # dbStorage_rocksDB_maxSizeInLevel1MB=256 # dbStorage_rocksDB_logPath= -# dbStorage_rocksDB_format_version=2 +############################################################################# +# Alternative RocksDB configuration by using configuration files. +############################################################################# +# entryLocationRocksdbConf=conf/entry_location_rocksdb.conf +# ledgerMetadataRocksdbConf=conf/ledger_metadata_rocksdb.conf +# defaultRocksdbConf=conf/default_rocksdb.conf ############################################################################# ## DirectIO entry logger configuration diff --git a/conf/default_rocksdb.conf.default b/conf/default_rocksdb.conf.default index e9b8e7c3ecd..ccedbb79ebb 100644 --- a/conf/default_rocksdb.conf.default +++ b/conf/default_rocksdb.conf.default @@ -16,6 +16,10 @@ # * limitations under the License. # */ +# When modifying this file, please also modify the configuration files(at +# bookkeeper-server/src/test/resources/conf) in the +# test case to ensure unit test coverage. + [DBOptions] # set by jni: options.setCreateIfMissing create_if_missing=true @@ -23,11 +27,14 @@ info_log_level=INFO_LEVEL # set by jni: options.setKeepLogFileNum keep_log_file_num=30 - -[CFOptions "default"] # set by jni: options.setLogFileTimeToRoll log_file_time_to_roll=86400 + [CFOptions "default"] + #no default setting in CFOptions + [TableOptions/BlockBasedTable "default"] + # set by jni: tableOptions.setFormatVersion + format_version=5 # set by jni: tableOptions.setChecksumType checksum=kxxHash \ No newline at end of file diff --git a/conf/entry_location_rocksdb.conf.default b/conf/entry_location_rocksdb.conf.default index 6f6c1b4d052..e4dc394243f 100644 --- a/conf/entry_location_rocksdb.conf.default +++ b/conf/entry_location_rocksdb.conf.default @@ -16,6 +16,10 @@ # * limitations under the License. # */ +# When modifying this file, please also modify the configuration files(at +# bookkeeper-server/src/test/resources/conf) in the +# test case to ensure unit test coverage. + [DBOptions] # set by jni: options.setCreateIfMissing create_if_missing=true @@ -26,7 +30,7 @@ # set by jni: options.setLogFileTimeToRoll log_file_time_to_roll=86400 # set by jni: options.setMaxBackgroundJobs or options.setIncreaseParallelism - max_background_jobs=2 + max_background_jobs=32 # set by jni: options.setMaxSubcompactions max_subcompactions=1 # set by jni: options.setMaxTotalWalSize @@ -57,10 +61,10 @@ [TableOptions/BlockBasedTable "default"] # set by jni: tableOptions.setBlockSize block_size=65536 - # set by jni: tableOptions.setBlockCache + # set by jni: tableOptions.setBlockCache, default value is: maxDirectMemory() / ledgerDirsSize / 10; block_cache=206150041 # set by jni: tableOptions.setFormatVersion - format_version=2 + format_version=5 # set by jni: tableOptions.setChecksumType checksum=kxxHash # set by jni: tableOptions.setFilterPolicy, bloomfilter:[bits_per_key]:[use_block_based_builder] diff --git a/conf/ledger_metadata_rocksdb.conf.default b/conf/ledger_metadata_rocksdb.conf.default index e9b8e7c3ecd..ccedbb79ebb 100644 --- a/conf/ledger_metadata_rocksdb.conf.default +++ b/conf/ledger_metadata_rocksdb.conf.default @@ -16,6 +16,10 @@ # * limitations under the License. # */ +# When modifying this file, please also modify the configuration files(at +# bookkeeper-server/src/test/resources/conf) in the +# test case to ensure unit test coverage. + [DBOptions] # set by jni: options.setCreateIfMissing create_if_missing=true @@ -23,11 +27,14 @@ info_log_level=INFO_LEVEL # set by jni: options.setKeepLogFileNum keep_log_file_num=30 - -[CFOptions "default"] # set by jni: options.setLogFileTimeToRoll log_file_time_to_roll=86400 + [CFOptions "default"] + #no default setting in CFOptions + [TableOptions/BlockBasedTable "default"] + # set by jni: tableOptions.setFormatVersion + format_version=5 # set by jni: tableOptions.setChecksumType checksum=kxxHash \ No newline at end of file diff --git a/dev/check-binary-license b/dev/check-binary-license index 5abc5b11a7b..af46f0b95fa 100755 --- a/dev/check-binary-license +++ b/dev/check-binary-license @@ -84,6 +84,11 @@ for J in $JARS; do continue fi + echo $J | grep -q "com.datastax.oss" + if [ $? == 0 ]; then + continue + fi + echo "$LICENSE" | grep -q $J if [ $? != 0 ]; then echo $J unaccounted for in LICENSE diff --git a/native-io/src/main/native-io-jni/cpp/native_io_jni.c b/native-io/src/main/native-io-jni/cpp/native_io_jni.c index d3bc164bec9..7be468518b0 100644 --- a/native-io/src/main/native-io-jni/cpp/native_io_jni.c +++ b/native-io/src/main/native-io-jni/cpp/native_io_jni.c @@ -20,11 +20,16 @@ */ #define _GNU_SOURCE +#include #include #include #include #include +#ifdef _WIN32 +#include +#else #include +#endif #include @@ -165,7 +170,14 @@ JNIEXPORT jint JNICALL Java_org_apache_bookkeeper_common_util_nativeio_NativeIOJni_fsync(JNIEnv * env, jclass clazz, jint fd) { - int res = fsync(fd); + int res; + + // Guarantee compatibility for winsows. + #ifdef _WIN32 + res = _commit((int)fd); + #else + res = fsync((int)fd); + #endif if (res == -1) { throwExceptionWithErrno(env, "Failed to fsync"); diff --git a/pom.xml b/pom.xml index c89621b85b1..f4de5687930 100644 --- a/pom.xml +++ b/pom.xml @@ -117,16 +117,16 @@ 1.18.2 1.8.0.Final 3.0.1 - 1.2 + 1.9.0 4.1 - 1.6 + 1.18.0 1.10 - 1.26.0 + 1.27.1 2.6 - 3.6 - 2.7 + 3.17.0 + 2.19.0 1.0.2.4 - 5.1.0 + 5.7.1 4.1.12.1 0.7.7 3.2.2 @@ -140,7 +140,7 @@ 2.1.10 2.17.1 1.82 - 9.4.53.v20231009 + 9.4.57.v20241219 1.37 2.8.2 1.14.3 @@ -152,18 +152,18 @@ 0.14.2 1.18.30 2.18.0 - 1.3.0 + 1.10.2 3.12.4 - 4.1.111.Final - 0.0.25.Final + 4.1.121.Final + 0.0.26.Final 9.1.3 2.0.9 0.15.0 0.8.3 4.5.13 4.4.15 - 3.21.9 - 3.21.9 + 3.25.5 + ${protobuf.version} ${grpc.version} 0.9.11 7.9.2 @@ -173,8 +173,8 @@ 4.6.0 1.3.2 1.19.4 - 4.5.7 - 3.8.3 + 4.5.11 + 3.9.3 1.1.10.5 2.1.2 0.9.1 @@ -192,7 +192,7 @@ 3.3.1 3.12.1 3.2.5 - 9.2.0 + 10.0.2 3.10.1 1.4.1.Final 0.6.1 @@ -319,9 +319,9 @@ - net.jpountz.lz4 - lz4 - ${lz4.version} + at.yawk.lz4 + lz4-java + ${lz4-java.version} @@ -980,7 +980,7 @@ org.apache.maven.plugins maven-surefire-plugin - -Xmx2G -Djava.net.preferIPv4Stack=true -Dio.netty.leakDetection.level=paranoid ${test.additional.args} + -Xmx2G -Dsun.net.inetaddr.ttl=1 -Dsun.net.inetaddr.negative.ttl=1 -Djava.net.preferIPv4Stack=true -Dio.netty.leakDetection.level=paranoid ${test.additional.args} ${redirectTestOutputToFile} ${forkCount.variable} false @@ -1209,6 +1209,10 @@ ${dependency-check-maven.version} false + + NIST_NVD_API_KEY + + src/owasp-dependency-check-suppressions.xml @@ -1293,7 +1297,7 @@ org.apache.maven.plugins maven-surefire-plugin - -Xmx2G -Djava.net.preferIPv4Stack=true + -Xmx2G -Dsun.net.inetaddr.ttl=1 -Dsun.net.inetaddr.negative.ttl=1 -Djava.net.preferIPv4Stack=true false ${forkCount.variable} false @@ -1313,7 +1317,7 @@ org.apache.maven.plugins maven-surefire-plugin - -Xmx2G -Djava.net.preferIPv4Stack=true -Dio.netty.leakDetection.level=paranoid -Dbookkeeper.log.root.level=INFO -Dbookkeeper.log.root.appender=CONSOLE + -Xmx2G -Dsun.net.inetaddr.ttl=1 -Dsun.net.inetaddr.negative.ttl=1 -Djava.net.preferIPv4Stack=true -Dio.netty.leakDetection.level=paranoid -Dbookkeeper.log.root.level=INFO -Dbookkeeper.log.root.appender=CONSOLE false ${forkCount.variable} false @@ -1443,12 +1447,12 @@ datastax-releases DataStax Local Releases - https://repo.sjc.dsinternal.org/artifactory/datastax-releases-local/ + https://repo.aws.dsinternal.org/artifactory/datastax-releases-local/ datastax-snapshots-local DataStax Local Snapshots - https://repo.sjc.dsinternal.org/artifactory/datastax-snapshots-local/ + https://repo.aws.dsinternal.org/artifactory/datastax-snapshots-local/ diff --git a/shaded/distributedlog-core-shaded/pom.xml b/shaded/distributedlog-core-shaded/pom.xml index 263b923ba84..4b35c10e1df 100644 --- a/shaded/distributedlog-core-shaded/pom.xml +++ b/shaded/distributedlog-core-shaded/pom.xml @@ -74,7 +74,7 @@ com.google.guava:failureaccess com.google.guava:guava com.google.protobuf:protobuf-java - net.jpountz.lz4:lz4 + at.yawk.lz4:lz4-java com.datastax.oss:bookkeeper-common com.datastax.oss:bookkeeper-common-allocator com.datastax.oss:cpu-affinity diff --git a/site3/website/docs/api/ledger-api.md b/site3/website/docs/api/ledger-api.md index 9b565088231..6364b138d42 100644 --- a/site3/website/docs/api/ledger-api.md +++ b/site3/website/docs/api/ledger-api.md @@ -49,7 +49,7 @@ shaded library, which relocate classes of protobuf and guava into a different na ### Gradle -If you're using [Gradle](https://gradle.org/), add this to your [`build.gradle`](https://spring.io/guides/gs/gradle/) build configuration file: +If you're using [Gradle](https://gradle.org/), add this to your [`build.gradle`] build configuration file: ```groovy dependencies { @@ -105,7 +105,7 @@ There are, however, other ways that you can create a client object: BookKeeper bkClient = new BookKeeper(config); ``` -* By specifying a `ClientConfiguration` and a [`ZooKeeper`](http://zookeeper.apache.org/doc/current/api/org/apache/zookeeper/ZooKeeper.html) client object: +* By specifying a `ClientConfiguration` and a [`ZooKeeper`] client object: ```java ClientConfiguration config = new ClientConfiguration(); diff --git a/stream/README.md b/stream/README.md index 1f57950cd36..4b828bc3c85 100644 --- a/stream/README.md +++ b/stream/README.md @@ -4,7 +4,7 @@ BookKeeper Table Service is a contrib module added to BookKeeper, providing a ta ## Detail Design -[BP-30](https://docs.google.com/document/d/155xAwWv5IdOitHh1NVMEwCMGgB28M3FyMiQSxEpjE-Y/edit#heading=h.56rbh52koe3f) +[BP-30] ## Build diff --git a/stream/clients/python/setup.py b/stream/clients/python/setup.py index c8c17ce200b..b5f214960cb 100644 --- a/stream/clients/python/setup.py +++ b/stream/clients/python/setup.py @@ -19,7 +19,7 @@ name = 'apache-bookkeeper-client' description = 'Apache BookKeeper client library' -version = '4.16.6' +version = '4.16.7' # Should be one of: # 'Development Status :: 3 - Alpha' # 'Development Status :: 4 - Beta' diff --git a/stream/distributedlog/common/pom.xml b/stream/distributedlog/common/pom.xml index 58885fbb8e0..46c1da617f1 100644 --- a/stream/distributedlog/common/pom.xml +++ b/stream/distributedlog/common/pom.xml @@ -74,8 +74,8 @@ netty-buffer - net.jpountz.lz4 - lz4 + at.yawk.lz4 + lz4-java org.jmock @@ -109,7 +109,7 @@ maven-surefire-plugin ${redirectTestOutputToFile} - -Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G ${test.additional.args} + -Xmx3G -Dsun.net.inetaddr.ttl=1 -Dsun.net.inetaddr.negative.ttl=1 -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G ${test.additional.args} 1 false 1800 diff --git a/stream/distributedlog/common/src/main/java/org/apache/distributedlog/io/LZ4CompressionCodec.java b/stream/distributedlog/common/src/main/java/org/apache/distributedlog/io/LZ4CompressionCodec.java index 48439d23a1a..a02322f4cc8 100644 --- a/stream/distributedlog/common/src/main/java/org/apache/distributedlog/io/LZ4CompressionCodec.java +++ b/stream/distributedlog/common/src/main/java/org/apache/distributedlog/io/LZ4CompressionCodec.java @@ -25,7 +25,7 @@ import java.nio.ByteBuffer; import net.jpountz.lz4.LZ4Compressor; import net.jpountz.lz4.LZ4Factory; -import net.jpountz.lz4.LZ4FastDecompressor; +import net.jpountz.lz4.LZ4SafeDecompressor; /** * An {@code lz4} based {@link CompressionCodec} implementation. @@ -44,7 +44,7 @@ public static LZ4CompressionCodec of() { // Used for compression private static final LZ4Compressor compressor = factory.fastCompressor(); // Used to decompress when the size of the output is known - private static final LZ4FastDecompressor decompressor = factory.fastDecompressor(); + private static final LZ4SafeDecompressor decompressor = factory.safeDecompressor(); @Override public ByteBuf compress(ByteBuf uncompressed, int headerLen) { @@ -69,7 +69,7 @@ public ByteBuf compress(ByteBuf uncompressed, int headerLen) { } @Override - // length parameter is ignored here because of the way the fastDecompressor works. + // length parameter is ignored here because of the way the safeDecompressor works. public ByteBuf decompress(ByteBuf compressed, int decompressedSize) { checkNotNull(compressed); checkArgument(compressed.readableBytes() >= 0); @@ -80,7 +80,7 @@ public ByteBuf decompress(ByteBuf compressed, int decompressedSize) { ByteBuffer compressedNio = compressed.nioBuffer(compressed.readerIndex(), compressed.readableBytes()); decompressor.decompress( - compressedNio, compressedNio.position(), + compressedNio, compressedNio.position(), compressedNio.remaining(), uncompressedNio, uncompressedNio.position(), uncompressedNio.remaining()); uncompressed.writerIndex(decompressedSize); return uncompressed; diff --git a/stream/distributedlog/core/pom.xml b/stream/distributedlog/core/pom.xml index c6c183c19f4..beae8e20bbb 100644 --- a/stream/distributedlog/core/pom.xml +++ b/stream/distributedlog/core/pom.xml @@ -110,7 +110,7 @@ false ${redirectTestOutputToFile} - -Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G ${test.additional.args} + -Xmx3G -Dsun.net.inetaddr.ttl=1 -Dsun.net.inetaddr.negative.ttl=1 -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G ${test.additional.args} 1 false 1800 diff --git a/stream/distributedlog/core/src/main/java/org/apache/distributedlog/BKDistributedLogNamespace.java b/stream/distributedlog/core/src/main/java/org/apache/distributedlog/BKDistributedLogNamespace.java index 0a3677d9560..8ed2cfeb362 100644 --- a/stream/distributedlog/core/src/main/java/org/apache/distributedlog/BKDistributedLogNamespace.java +++ b/stream/distributedlog/core/src/main/java/org/apache/distributedlog/BKDistributedLogNamespace.java @@ -158,12 +158,13 @@ public void deleteLog(String logName) if (!uri.isPresent()) { throw new LogNotFoundException("Log " + logName + " isn't found."); } - DistributedLogManager dlm = openLogInternal( + try (DistributedLogManager dlm = openLogInternal( uri.get(), logName, Optional.empty(), - Optional.empty()); - dlm.delete(); + Optional.empty())) { + dlm.delete(); + } } @Override diff --git a/stream/distributedlog/core/src/main/java/org/apache/distributedlog/impl/BKNamespaceDriver.java b/stream/distributedlog/core/src/main/java/org/apache/distributedlog/impl/BKNamespaceDriver.java index ea17ed76931..957995b3fd6 100644 --- a/stream/distributedlog/core/src/main/java/org/apache/distributedlog/impl/BKNamespaceDriver.java +++ b/stream/distributedlog/core/src/main/java/org/apache/distributedlog/impl/BKNamespaceDriver.java @@ -533,8 +533,7 @@ public Map enumerateLogsWithMetadataInNamespace() String namespaceRootPath = namespace.getPath(); HashMap result = new HashMap(); ZooKeeperClient zkc = writerZKC; - try { - ZooKeeper zk = Utils.sync(zkc, namespaceRootPath); + try (ZooKeeper zk = Utils.sync(zkc, namespaceRootPath)) { Stat currentStat = zk.exists(namespaceRootPath, false); if (currentStat == null) { return result; diff --git a/stream/distributedlog/pom.xml b/stream/distributedlog/pom.xml index 193476682a5..80cfa966381 100644 --- a/stream/distributedlog/pom.xml +++ b/stream/distributedlog/pom.xml @@ -75,7 +75,7 @@ maven-surefire-plugin ${redirectTestOutputToFile} - -Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G -Dio.netty.leakDetection.level=PARANOID ${test.additional.args} + -Xmx3G -Dsun.net.inetaddr.ttl=1 -Dsun.net.inetaddr.negative.ttl=1 -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G -Dio.netty.leakDetection.level=PARANOID ${test.additional.args} 1 false 1800 diff --git a/stream/pom.xml b/stream/pom.xml index 94af8ee3f85..192dff90fa8 100644 --- a/stream/pom.xml +++ b/stream/pom.xml @@ -57,7 +57,7 @@ true ${redirectTestOutputToFile} - -Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G -Dio.netty.leakDetection.level=PARANOID ${test.additional.args} + -Xmx3G -Dsun.net.inetaddr.ttl=1 -Dsun.net.inetaddr.negative.ttl=1 -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G -Dio.netty.leakDetection.level=PARANOID ${test.additional.args} 1 false 1800 diff --git a/tests/docker-images/all-released-versions-image/Dockerfile b/tests/docker-images/all-released-versions-image/Dockerfile index a8c761c7e88..b7fd4e2fd10 100644 --- a/tests/docker-images/all-released-versions-image/Dockerfile +++ b/tests/docker-images/all-released-versions-image/Dockerfile @@ -17,7 +17,7 @@ # under the License. # -FROM eclipse-temurin:8-jdk +FROM eclipse-temurin:8-jdk-jammy MAINTAINER Apache BookKeeper ENV BK_JOURNALDIR=/opt/bookkeeper/data/journal @@ -31,7 +31,7 @@ RUN sed -i -e "s|http://archive\.ubuntu\.com/ubuntu/|${UBUNTU_MIRROR:-http://arc -e "s|http://security\.ubuntu\.com/ubuntu/|${UBUNTU_SECURITY_MIRROR:-http://security.ubuntu.com/ubuntu/}|g" /etc/apt/sources.list \ && echo 'Acquire::http::Timeout "30";\nAcquire::http::ConnectionAttemptDelayMsec "2000";\nAcquire::https::Timeout "30";\nAcquire::https::ConnectionAttemptDelayMsec "2000";\nAcquire::ftp::Timeout "30";\nAcquire::ftp::ConnectionAttemptDelayMsec "2000";\nAcquire::Retries "15";' > /etc/apt/apt.conf.d/99timeout_and_retries \ && apt-get update && apt-get install -qy wget curl supervisor bash ca-certificates apt-transport-https \ - && apt-get -y install netcat dnsutils less procps iputils-ping \ + && apt-get -y install netcat-openbsd dnsutils less procps iputils-ping \ && apt-get install -y --no-install-recommends gpg gpg-agent sudo \ && echo "dash dash/sh boolean false" | debconf-set-selections \ && DEBIAN_FRONTEND=noninteractive dpkg-reconfigure dash \ diff --git a/tests/docker-images/current-version-image/Dockerfile b/tests/docker-images/current-version-image/Dockerfile index 0e489eb5ea6..ae171625f4f 100644 --- a/tests/docker-images/current-version-image/Dockerfile +++ b/tests/docker-images/current-version-image/Dockerfile @@ -45,7 +45,7 @@ RUN set -x \ && apt-get install -y --no-install-recommends python3 pip \ && ln -s /usr/bin/python3 /usr/bin/python \ && apt-get install -y --no-install-recommends gpg gpg-agent wget sudo \ - && apt-get -y install netcat dnsutils less procps iputils-ping \ + && apt-get -y install netcat-openbsd dnsutils less procps iputils-ping \ && apt-get -y --purge autoremove \ && apt-get autoclean \ && apt-get clean \ diff --git a/tests/docker-images/statestore-image/Dockerfile b/tests/docker-images/statestore-image/Dockerfile index 5a7a1688855..974d11b50c8 100644 --- a/tests/docker-images/statestore-image/Dockerfile +++ b/tests/docker-images/statestore-image/Dockerfile @@ -40,7 +40,7 @@ RUN set -x \ && apt-get install -y --no-install-recommends openjdk-17-jdk \ && apt-get install -y --no-install-recommends python3 pip \ && ln -s /usr/bin/python3 /usr/bin/python \ - && apt-get -y install netcat dnsutils less procps iputils-ping \ + && apt-get -y install netcat-openbsd dnsutils less procps iputils-ping \ && apt-get install -y --no-install-recommends gpg gpg-agent wget sudo \ && apt-get -y --purge autoremove \ && apt-get autoclean \ diff --git a/tests/integration-tests-base-groovy/pom.xml b/tests/integration-tests-base-groovy/pom.xml index 5e1919e236d..a56950b4213 100644 --- a/tests/integration-tests-base-groovy/pom.xml +++ b/tests/integration-tests-base-groovy/pom.xml @@ -68,7 +68,7 @@ org.apache.maven.plugins maven-surefire-plugin - -Xmx4G -Djava.net.preferIPv4Stack=true ${test.additional.args} + -Xmx4G -Dsun.net.inetaddr.ttl=1 -Dsun.net.inetaddr.negative.ttl=1 -Djava.net.preferIPv4Stack=true ${test.additional.args} 1 false diff --git a/tools/perf/src/main/java/org/apache/bookkeeper/tools/perf/journal/JournalWriter.java b/tools/perf/src/main/java/org/apache/bookkeeper/tools/perf/journal/JournalWriter.java index 383ccfb9825..e287fbd94af 100644 --- a/tools/perf/src/main/java/org/apache/bookkeeper/tools/perf/journal/JournalWriter.java +++ b/tools/perf/src/main/java/org/apache/bookkeeper/tools/perf/journal/JournalWriter.java @@ -495,6 +495,7 @@ private static ByteBufAllocator getAllocator(ServerConfiguration conf) { log.error("Unable to allocate memory, exiting bookie", ex); }) .leakDetectionPolicy(conf.getAllocatorLeakDetectionPolicy()) + .exitOnOutOfMemory(conf.exitOnOutOfMemory()) .build(); }