From ac9acf37dee0c6a11d85db362620870c0bc257ba Mon Sep 17 00:00:00 2001 From: mukesh-ctds Date: Thu, 6 Feb 2025 10:55:01 +0530 Subject: [PATCH 01/55] Migrate to the new Datastax internal artifactory (cherry picked from commit e02b9c27687afc811d8dfb4a6d054f6dac5e97ec) --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index c89621b85b1..a831f281c72 100644 --- a/pom.xml +++ b/pom.xml @@ -1443,12 +1443,12 @@ datastax-releases DataStax Local Releases - https://repo.sjc.dsinternal.org/artifactory/datastax-releases-local/ + https://repo.aws.dsinternal.org/artifactory/datastax-releases-local/ datastax-snapshots-local DataStax Local Snapshots - https://repo.sjc.dsinternal.org/artifactory/datastax-snapshots-local/ + https://repo.aws.dsinternal.org/artifactory/datastax-snapshots-local/ From 7514e4bcd9fda32464a7616b5e9fc2f510c6cdd4 Mon Sep 17 00:00:00 2001 From: mukesh-ctds Date: Thu, 6 Feb 2025 11:00:52 +0530 Subject: [PATCH 02/55] Enable Github CI for Datastax fork (cherry picked from commit 5ee217866642ab6532bd9e49bcf7b2eca41d97a6) --- .github/workflows/bk-ci.yml | 2 ++ .github/workflows/bk-streamstorage-python.yml | 2 ++ .github/workflows/dead-link-checker.yaml | 1 + .github/workflows/website-pr-validation.yml | 1 + 4 files changed, 6 insertions(+) diff --git a/.github/workflows/bk-ci.yml b/.github/workflows/bk-ci.yml index e04462f0f54..d5b258bd007 100644 --- a/.github/workflows/bk-ci.yml +++ b/.github/workflows/bk-ci.yml @@ -24,10 +24,12 @@ on: branches: - master - branch-* + - ds-* push: branches: - master - branch-* + - ds-* workflow_dispatch: env: diff --git a/.github/workflows/bk-streamstorage-python.yml b/.github/workflows/bk-streamstorage-python.yml index 71dbe610452..d785c5cb716 100644 --- a/.github/workflows/bk-streamstorage-python.yml +++ b/.github/workflows/bk-streamstorage-python.yml @@ -23,6 +23,7 @@ on: branches: - master - branch-* + - ds-* paths: - 'stream/**' - '.github/workflows/bk-streamstorage-python.yml' @@ -30,6 +31,7 @@ on: branches: - master - branch-* + - ds-* paths: - 'stream/**' - '.github/workflows/bk-streamstorage-python.yml' diff --git a/.github/workflows/dead-link-checker.yaml b/.github/workflows/dead-link-checker.yaml index 8dae314d409..944a5bf2032 100644 --- a/.github/workflows/dead-link-checker.yaml +++ b/.github/workflows/dead-link-checker.yaml @@ -22,6 +22,7 @@ on: branches: - master - branch-* + - ds-* paths: - '**.md' diff --git a/.github/workflows/website-pr-validation.yml b/.github/workflows/website-pr-validation.yml index 9d38cb59f28..6b813f7c5e9 100644 --- a/.github/workflows/website-pr-validation.yml +++ b/.github/workflows/website-pr-validation.yml @@ -25,6 +25,7 @@ on: branches: - master - branch-* + - ds-* paths: - 'site3/**' - '.github/workflows/website-pr-validation.yaml' From daaa7aff51b234f358c4f0e640ca26697bdd30bc Mon Sep 17 00:00:00 2001 From: Yan Zhao Date: Mon, 8 Jul 2024 17:17:08 +0800 Subject: [PATCH 03/55] Correct RackawareEnsemblePlacementPolicyImpl defaultRack when the bookie is not available. (#4439) When the bookie is not available, the RackawareEnsemblePlacementPolicyImpl default rack will be `/default-region/default-rack`, it should be `/default-rack` for RackawareEnsemblePlacementPolicyImpl. There are some logs. ``` 2024-06-17T05:22:46,591+0000 [ReplicationWorker] ERROR org.apache.bookkeeper.client.TopologyAwareEnsemblePlacementPolicy - Cannot resolve bookieId `test-bk-3:3181` to a network address, resolving as /default-region/default-rack org.apache.bookkeeper.proto.BookieAddressResolver$BookieIdNotResolvedException: Cannot resolve bookieId test-bk-3:3181, bookie does not exist or it is not running at org.apache.bookkeeper.client.DefaultBookieAddressResolver.resolve(DefaultBookieAddressResolver.java:66) ~[io.streamnative-bookkeeper-server-4.16.5.2.jar:4.16.5.2] at org.apache.bookkeeper.client.TopologyAwareEnsemblePlacementPolicy.resolveNetworkLocation(TopologyAwareEnsemblePlacementPolicy.java:821) ~[io.streamnative-bookkeeper-server-4.16.5.2.jar:4.16.5.2] at org.apache.bookkeeper.client.TopologyAwareEnsemblePlacementPolicy.createBookieNode(TopologyAwareEnsemblePlacementPolicy.java:811) ~[io.streamnative-bookkeeper-server-4.16.5.2.jar:4.16.5.2] at org.apache.bookkeeper.client.TopologyAwareEnsemblePlacementPolicy.convertBookieToNode(TopologyAwareEnsemblePlacementPolicy.java:845) ~[io.streamnative-bookkeeper-server-4.16.5.2.jar:4.16.5.2] at org.apache.bookkeeper.client.TopologyAwareEnsemblePlacementPolicy.convertBookiesToNodes(TopologyAwareEnsemblePlacementPolicy.java:837) ~[io.streamnative-bookkeeper-server-4.16.5.2.jar:4.16.5.2] at org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicyImpl.replaceBookie(RackawareEnsemblePlacementPolicyImpl.java:474) ~[io.streamnative-bookkeeper-server-4.16.5.2.jar:4.16.5.2] at org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy.replaceBookie(RackawareEnsemblePlacementPolicy.java:119) ~[io.streamnative-bookkeeper-server-4.16.5.2.jar:4.16.5.2] at org.apache.bookkeeper.client.BookKeeperAdmin.getReplacementBookiesByIndexes(BookKeeperAdmin.java:993) ~[io.streamnative-bookkeeper-server-4.16.5.2.jar:4.16.5.2] at org.apache.bookkeeper.client.BookKeeperAdmin.replicateLedgerFragment(BookKeeperAdmin.java:1025) ~[io.streamnative-bookkeeper-server-4.16.5.2.jar:4.16.5.2] at org.apache.bookkeeper.replication.ReplicationWorker.rereplicate(ReplicationWorker.java:473) ~[io.streamnative-bookkeeper-server-4.16.5.2.jar:4.16.5.2] at org.apache.bookkeeper.replication.ReplicationWorker.rereplicate(ReplicationWorker.java:301) ~[io.streamnative-bookkeeper-server-4.16.5.2.jar:4.16.5.2] at org.apache.bookkeeper.replication.ReplicationWorker.run(ReplicationWorker.java:249) ~[io.streamnative-bookkeeper-server-4.16.5.2.jar:4.16.5.2] at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[io.netty-netty-common-4.1.108.Final.jar:4.1.108.Final] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.bookkeeper.client.BKException$BKBookieHandleNotAvailableException: Bookie handle is not available at org.apache.bookkeeper.discover.ZKRegistrationClient.getBookieServiceInfo(ZKRegistrationClient.java:226) ~[io.streamnative-bookkeeper-server-4.16.5.2.jar:4.16.5.2] at org.apache.bookkeeper.client.DefaultBookieAddressResolver.resolve(DefaultBookieAddressResolver.java:45) ~[io.streamnative-bookkeeper-server-4.16.5.2.jar:4.16.5.2] ... 13 more ``` (cherry picked from commit fb7138329ff2faaa3e10fb4f2028e2aaac28843e) (cherry picked from commit 8cc815e84f719e085d13a09f1d31ed56a55350e4) --- .../RackawareEnsemblePlacementPolicyImpl.java | 1 + .../TopologyAwareEnsemblePlacementPolicy.java | 12 ++-- .../TestRackawareEnsemblePlacementPolicy.java | 64 +++++++++++++++++++ 3 files changed, 73 insertions(+), 4 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java index 4f9ed29f15e..64e7951df33 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java @@ -241,6 +241,7 @@ public RackawareEnsemblePlacementPolicyImpl withDefaultRack(String rack) { return this; } + @Override public String getDefaultRack() { return defaultRack; } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java index 463d9599de2..9eae79438fd 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java @@ -824,13 +824,17 @@ protected String resolveNetworkLocation(BookieId addr) { if (null != historyBookie) { return historyBookie.getNetworkLocation(); } - - LOG.error("Cannot resolve bookieId {} to a network address, resolving as {}", addr, - NetworkTopology.DEFAULT_REGION_AND_RACK, err); - return NetworkTopology.DEFAULT_REGION_AND_RACK; + String defaultRack = getDefaultRack(); + LOG.error("Cannot resolve bookieId {} to a network address, resolving as {}. {}", addr, + defaultRack, err.getMessage()); + return defaultRack; } } + protected String getDefaultRack() { + return NetworkTopology.DEFAULT_REGION_AND_RACK; + } + protected Set convertBookiesToNodes(Collection bookies) { Set nodes = new HashSet(); for (BookieId addr : bookies) { diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java index 4d0606f9021..857458296e8 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java @@ -29,6 +29,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import io.netty.util.HashedWheelTimer; import java.net.InetAddress; +import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -40,6 +41,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import junit.framework.TestCase; import org.apache.bookkeeper.client.BKException.BKNotEnoughBookiesException; @@ -2270,6 +2272,68 @@ public void testNodeWithFailures() throws Exception { assertEquals(ensemble.get(reoderSet.get(3)), addr2.toBookieId()); assertEquals(ensemble.get(reoderSet.get(0)), addr3.toBookieId()); assertEquals(ensemble.get(reoderSet.get(1)), addr4.toBookieId()); + StaticDNSResolver.reset(); + } + + @Test + public void testReplaceNotAvailableBookieWithDefaultRack() throws Exception { + repp.uninitalize(); + repp.withDefaultRack(NetworkTopology.DEFAULT_RACK); + AtomicInteger counter = new AtomicInteger(); + BookieAddressResolver mockResolver = new BookieAddressResolver() { + @Override + public BookieSocketAddress resolve(BookieId bookieId) throws BookieIdNotResolvedException { + if (bookieId.equals(addr1.toBookieId()) && counter.getAndIncrement() >= 1) { + throw new BookieIdNotResolvedException(bookieId, + new RuntimeException(addr1.toBookieId() + " shutdown")); + } + try { + return new BookieSocketAddress(bookieId.toString()); + } catch (UnknownHostException err) { + throw new BookieIdNotResolvedException(bookieId, err); + } + } + }; + + repp.initialize(conf, Optional.empty(), timer, DISABLE_ALL, NullStatsLogger.INSTANCE, + mockResolver); + BookieSocketAddress addr1 = new BookieSocketAddress("127.0.0.2", 3181); + BookieSocketAddress addr2 = new BookieSocketAddress("127.0.0.3", 3181); + BookieSocketAddress addr3 = new BookieSocketAddress("127.0.0.4", 3181); + BookieSocketAddress addr4 = new BookieSocketAddress("127.0.0.5", 3181); + // update dns mapping + StaticDNSResolver.addNodeToRack(addr1.getHostName(), NetworkTopology.DEFAULT_RACK); + StaticDNSResolver.addNodeToRack(addr2.getHostName(), "/r1"); + StaticDNSResolver.addNodeToRack(addr3.getHostName(), "/r1"); + StaticDNSResolver.addNodeToRack(addr4.getHostName(), NetworkTopology.DEFAULT_RACK); + + // Update cluster + Set addrs = new HashSet(); + addrs.add(addr1.toBookieId()); + addrs.add(addr2.toBookieId()); + addrs.add(addr3.toBookieId()); + addrs.add(addr4.toBookieId()); + repp.onClusterChanged(addrs, new HashSet()); + + // replace node under r1 + EnsemblePlacementPolicy.PlacementResult replaceBookieResponse = + repp.replaceBookie(1, 1, 1, null, new ArrayList<>(), addr1.toBookieId(), new HashSet<>()); + BookieId replacedBookie = replaceBookieResponse.getResult(); + assertEquals(addr4.toBookieId(), replacedBookie); + + //clear history bookies and make addr1 shutdown. + repp = new RackawareEnsemblePlacementPolicy(); + repp.initialize(conf, Optional.empty(), timer, DISABLE_ALL, NullStatsLogger.INSTANCE, + mockResolver); + + addrs.remove(addr1.toBookieId()); + repp.onClusterChanged(addrs, new HashSet()); + + // replace node under r1 again + replaceBookieResponse = + repp.replaceBookie(1, 1, 1, null, new ArrayList<>(), addr1.toBookieId(), new HashSet<>()); + replacedBookie = replaceBookieResponse.getResult(); + assertEquals(addr4.toBookieId(), replacedBookie); } @Test From 8a58b741aa7a1a7c64c1bbd2a1d638b84849570a Mon Sep 17 00:00:00 2001 From: Yan Zhao Date: Tue, 9 Jul 2024 09:44:45 +0800 Subject: [PATCH 04/55] Fix the completionObjects leak problem. (#4285) (cherry picked from commit 772b162e4a610918403e45fbb3ad358dd6a01ad3) (cherry picked from commit f51f5842ce9b5219d2aaca77fd6576ec34cc90ff) --- .../java/org/apache/bookkeeper/proto/PerChannelBookieClient.java | 1 + 1 file changed, 1 insertion(+) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java index 149f97fc28b..a3475533f1a 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java @@ -1162,6 +1162,7 @@ private void writeAndFlush(final Channel channel, } } else { nettyOpLogger.registerFailedEvent(MathUtils.elapsedNanos(startTime), TimeUnit.NANOSECONDS); + errorOut(key); } } finally { if (cleanupActionAfterWrite != null) { From d0e7ff0ccfd853a52337776018a6077a65facece Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Mon, 29 Jul 2024 14:33:02 +0800 Subject: [PATCH 05/55] fix: install netcat-openbsd instead of netcat in test image build (#4476) Signed-off-by: ZhangJian He (cherry picked from commit 56625245d53ce37525fc3f8eb4f3274beecf52eb) (cherry picked from commit 66e84adefa932d60dec7a41763e132fa0e2c41cd) --- tests/docker-images/all-released-versions-image/Dockerfile | 4 ++-- tests/docker-images/current-version-image/Dockerfile | 2 +- tests/docker-images/statestore-image/Dockerfile | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/docker-images/all-released-versions-image/Dockerfile b/tests/docker-images/all-released-versions-image/Dockerfile index a8c761c7e88..b7fd4e2fd10 100644 --- a/tests/docker-images/all-released-versions-image/Dockerfile +++ b/tests/docker-images/all-released-versions-image/Dockerfile @@ -17,7 +17,7 @@ # under the License. # -FROM eclipse-temurin:8-jdk +FROM eclipse-temurin:8-jdk-jammy MAINTAINER Apache BookKeeper ENV BK_JOURNALDIR=/opt/bookkeeper/data/journal @@ -31,7 +31,7 @@ RUN sed -i -e "s|http://archive\.ubuntu\.com/ubuntu/|${UBUNTU_MIRROR:-http://arc -e "s|http://security\.ubuntu\.com/ubuntu/|${UBUNTU_SECURITY_MIRROR:-http://security.ubuntu.com/ubuntu/}|g" /etc/apt/sources.list \ && echo 'Acquire::http::Timeout "30";\nAcquire::http::ConnectionAttemptDelayMsec "2000";\nAcquire::https::Timeout "30";\nAcquire::https::ConnectionAttemptDelayMsec "2000";\nAcquire::ftp::Timeout "30";\nAcquire::ftp::ConnectionAttemptDelayMsec "2000";\nAcquire::Retries "15";' > /etc/apt/apt.conf.d/99timeout_and_retries \ && apt-get update && apt-get install -qy wget curl supervisor bash ca-certificates apt-transport-https \ - && apt-get -y install netcat dnsutils less procps iputils-ping \ + && apt-get -y install netcat-openbsd dnsutils less procps iputils-ping \ && apt-get install -y --no-install-recommends gpg gpg-agent sudo \ && echo "dash dash/sh boolean false" | debconf-set-selections \ && DEBIAN_FRONTEND=noninteractive dpkg-reconfigure dash \ diff --git a/tests/docker-images/current-version-image/Dockerfile b/tests/docker-images/current-version-image/Dockerfile index 0e489eb5ea6..ae171625f4f 100644 --- a/tests/docker-images/current-version-image/Dockerfile +++ b/tests/docker-images/current-version-image/Dockerfile @@ -45,7 +45,7 @@ RUN set -x \ && apt-get install -y --no-install-recommends python3 pip \ && ln -s /usr/bin/python3 /usr/bin/python \ && apt-get install -y --no-install-recommends gpg gpg-agent wget sudo \ - && apt-get -y install netcat dnsutils less procps iputils-ping \ + && apt-get -y install netcat-openbsd dnsutils less procps iputils-ping \ && apt-get -y --purge autoremove \ && apt-get autoclean \ && apt-get clean \ diff --git a/tests/docker-images/statestore-image/Dockerfile b/tests/docker-images/statestore-image/Dockerfile index 5a7a1688855..974d11b50c8 100644 --- a/tests/docker-images/statestore-image/Dockerfile +++ b/tests/docker-images/statestore-image/Dockerfile @@ -40,7 +40,7 @@ RUN set -x \ && apt-get install -y --no-install-recommends openjdk-17-jdk \ && apt-get install -y --no-install-recommends python3 pip \ && ln -s /usr/bin/python3 /usr/bin/python \ - && apt-get -y install netcat dnsutils less procps iputils-ping \ + && apt-get -y install netcat-openbsd dnsutils less procps iputils-ping \ && apt-get install -y --no-install-recommends gpg gpg-agent wget sudo \ && apt-get -y --purge autoremove \ && apt-get autoclean \ From c94b5e3a3102a83f978fdac3afc2bb8cde5a503f Mon Sep 17 00:00:00 2001 From: Yong Zhang Date: Wed, 19 Mar 2025 10:26:02 +0800 Subject: [PATCH 06/55] Change the new ensemble log to info level (#4566) --- ### Motivation Change the new ensemble log to the info level. Sometimes, the ensemble may not satisfied with the placement policy. The log should be info level but not a warn level. Because it will fix by the auto recovery later, so this just a information from the ensemble choose. (cherry picked from commit a12943df277e698a1621d33c245913365d031257) (cherry picked from commit 0ce47aa9aeb127b2d68a11f7f48dadc15a211f44) --- .../java/org/apache/bookkeeper/client/BookieWatcherImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookieWatcherImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookieWatcherImpl.java index 978842b114a..f7af1549286 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookieWatcherImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookieWatcherImpl.java @@ -273,7 +273,7 @@ public List newEnsemble(int ensembleSize, int writeQuorumSize, if (isEnsembleAdheringToPlacementPolicy == PlacementPolicyAdherence.FAIL) { ensembleNotAdheringToPlacementPolicy.inc(); if (ensembleSize > 1) { - log.warn("New ensemble: {} is not adhering to Placement Policy. quarantinedBookies: {}", + log.info("New ensemble: {} is not adhering to Placement Policy. quarantinedBookies: {}", socketAddresses, quarantinedBookiesSet); } } @@ -289,7 +289,7 @@ public List newEnsemble(int ensembleSize, int writeQuorumSize, isEnsembleAdheringToPlacementPolicy = newEnsembleResponse.getAdheringToPolicy(); if (isEnsembleAdheringToPlacementPolicy == PlacementPolicyAdherence.FAIL) { ensembleNotAdheringToPlacementPolicy.inc(); - log.warn("New ensemble: {} is not adhering to Placement Policy", socketAddresses); + log.info("New ensemble: {} is not adhering to Placement Policy", socketAddresses); } newEnsembleTimer.registerFailedEvent(MathUtils.nowInNano() - startTime, TimeUnit.NANOSECONDS); } From 978d4968d66db29f55776bf4fe116d3eeec71899 Mon Sep 17 00:00:00 2001 From: danpi Date: Tue, 1 Apr 2025 11:53:36 +0800 Subject: [PATCH 07/55] Fix SST files not being cleaned up in the locations folder (#4555) * fix entry location compaction * replace entryLocationCompactionEnable with entryLocationCompactionInterval * Add randomCompactionDelay to avoid all the bookies triggering compaction simultaneously * Fix the style issue * Fix the style issue * Fix test --------- Co-authored-by: houbonan Co-authored-by: zymap (cherry picked from commit ede1ba972cf0458ca1dc00338b5feb5b0f617aa2) (cherry picked from commit fbd33b56210bc17965c2d4bf0d835c44c5d4836f) --- .../bookie/BookKeeperServerStats.java | 1 + .../bookie/GarbageCollectionStatus.java | 2 + .../bookie/GarbageCollectorThread.java | 38 ++++++++++++++++++- .../bookie/stats/GarbageCollectorStats.java | 7 ++++ .../ldb/SingleDirectoryDbLedgerStorage.java | 2 + .../bookkeeper/conf/ServerConfiguration.java | 30 +++++++++++++++ .../conf/TestServerConfiguration.java | 20 ++++++++++ conf/bk_server.conf | 5 +++ 8 files changed, 104 insertions(+), 1 deletion(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookKeeperServerStats.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookKeeperServerStats.java index d4657d20362..361861cda14 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookKeeperServerStats.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookKeeperServerStats.java @@ -150,6 +150,7 @@ public interface BookKeeperServerStats { String THREAD_RUNTIME = "THREAD_RUNTIME"; String MAJOR_COMPACTION_COUNT = "MAJOR_COMPACTION_TOTAL"; String MINOR_COMPACTION_COUNT = "MINOR_COMPACTION_TOTAL"; + String ENTRY_LOCATION_COMPACTION_COUNT = "ENTRY_LOCATION_COMPACTION_TOTAL"; String ACTIVE_LEDGER_COUNT = "ACTIVE_LEDGER_TOTAL"; String DELETED_LEDGER_COUNT = "DELETED_LEDGER_TOTAL"; diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectionStatus.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectionStatus.java index 3f872092f01..4ad450a64f1 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectionStatus.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectionStatus.java @@ -42,6 +42,8 @@ public class GarbageCollectionStatus { private long lastMajorCompactionTime; private long lastMinorCompactionTime; + private long lastEntryLocationCompactionTime; private long majorCompactionCounter; private long minorCompactionCounter; + private long entryLocationCompactionCounter; } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java index 4a1c0289879..5abf79b2e64 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java @@ -33,6 +33,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -85,6 +86,10 @@ public class GarbageCollectorThread implements Runnable { long majorCompactionMaxTimeMillis; long lastMajorCompactionTime; + final long entryLocationCompactionInterval; + long randomCompactionDelay; + long lastEntryLocationCompactionTime; + @Getter final boolean isForceGCAllowWhenNoSpace; @@ -196,6 +201,10 @@ public GarbageCollectorThread(ServerConfiguration conf, isForceGCAllowWhenNoSpace = conf.getIsForceGCAllowWhenNoSpace(); majorCompactionMaxTimeMillis = conf.getMajorCompactionMaxTimeMillis(); minorCompactionMaxTimeMillis = conf.getMinorCompactionMaxTimeMillis(); + entryLocationCompactionInterval = conf.getEntryLocationCompactionInterval() * SECOND; + if (entryLocationCompactionInterval > 0) { + randomCompactionDelay = ThreadLocalRandom.current().nextLong(entryLocationCompactionInterval); + } boolean isForceAllowCompaction = conf.isForceAllowCompaction(); @@ -262,12 +271,22 @@ public void removeEntryLog(long logToRemove) { } } + if (entryLocationCompactionInterval > 0) { + if (entryLocationCompactionInterval < gcWaitTime) { + throw new IOException( + "Too short entry location compaction interval : " + entryLocationCompactionInterval); + } + } + LOG.info("Minor Compaction : enabled=" + enableMinorCompaction + ", threshold=" + minorCompactionThreshold + ", interval=" + minorCompactionInterval); LOG.info("Major Compaction : enabled=" + enableMajorCompaction + ", threshold=" + majorCompactionThreshold + ", interval=" + majorCompactionInterval); + LOG.info("Entry Location Compaction : interval=" + entryLocationCompactionInterval + ", randomCompactionDelay=" + + randomCompactionDelay); - lastMinorCompactionTime = lastMajorCompactionTime = System.currentTimeMillis(); + lastMinorCompactionTime = lastMajorCompactionTime = + lastEntryLocationCompactionTime = System.currentTimeMillis(); } private EntryLogMetadataMap createEntryLogMetadataMap() throws IOException { @@ -445,6 +464,7 @@ public void runWithFlags(boolean force, boolean suspendMajor, boolean suspendMin gcStats.getMajorCompactionCounter().inc(); majorCompacting.set(false); } + } else if (((isForceMinorCompactionAllow && force) || (enableMinorCompaction && (force || curTime - lastMinorCompactionTime > minorCompactionInterval))) && (!suspendMinor)) { @@ -459,6 +479,20 @@ public void runWithFlags(boolean force, boolean suspendMajor, boolean suspendMin minorCompacting.set(false); } } + if (entryLocationCompactionInterval > 0 && (curTime - lastEntryLocationCompactionTime > ( + entryLocationCompactionInterval + randomCompactionDelay))) { + // enter entry location compaction + LOG.info( + "Enter entry location compaction, entryLocationCompactionInterval {}, randomCompactionDelay " + + "{}, lastEntryLocationCompactionTime {}", + entryLocationCompactionInterval, randomCompactionDelay, lastEntryLocationCompactionTime); + ledgerStorage.entryLocationCompact(); + lastEntryLocationCompactionTime = System.currentTimeMillis(); + randomCompactionDelay = ThreadLocalRandom.current().nextLong(entryLocationCompactionInterval); + LOG.info("Next entry location compaction interval {}", + entryLocationCompactionInterval + randomCompactionDelay); + gcStats.getEntryLocationCompactionCounter().inc(); + } gcStats.getGcThreadRuntime().registerSuccessfulEvent( MathUtils.nowInNano() - threadStart, TimeUnit.NANOSECONDS); } catch (EntryLogMetadataMapException e) { @@ -796,8 +830,10 @@ public GarbageCollectionStatus getGarbageCollectionStatus() { .minorCompacting(minorCompacting.get()) .lastMajorCompactionTime(lastMajorCompactionTime) .lastMinorCompactionTime(lastMinorCompactionTime) + .lastEntryLocationCompactionTime(lastEntryLocationCompactionTime) .majorCompactionCounter(gcStats.getMajorCompactionCounter().get()) .minorCompactionCounter(gcStats.getMinorCompactionCounter().get()) + .entryLocationCompactionCounter(gcStats.getEntryLocationCompactionCounter().get()) .build(); } } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/stats/GarbageCollectorStats.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/stats/GarbageCollectorStats.java index 1c9475608f4..3029e7d55b7 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/stats/GarbageCollectorStats.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/stats/GarbageCollectorStats.java @@ -25,6 +25,7 @@ import static org.apache.bookkeeper.bookie.BookKeeperServerStats.BOOKIE_SCOPE; import static org.apache.bookkeeper.bookie.BookKeeperServerStats.CATEGORY_SERVER; import static org.apache.bookkeeper.bookie.BookKeeperServerStats.DELETED_LEDGER_COUNT; +import static org.apache.bookkeeper.bookie.BookKeeperServerStats.ENTRY_LOCATION_COMPACTION_COUNT; import static org.apache.bookkeeper.bookie.BookKeeperServerStats.MAJOR_COMPACTION_COUNT; import static org.apache.bookkeeper.bookie.BookKeeperServerStats.MINOR_COMPACTION_COUNT; import static org.apache.bookkeeper.bookie.BookKeeperServerStats.RECLAIMED_COMPACTION_SPACE_BYTES; @@ -61,6 +62,11 @@ public class GarbageCollectorStats { help = "Number of major compactions" ) private final Counter majorCompactionCounter; + @StatsDoc( + name = ENTRY_LOCATION_COMPACTION_COUNT, + help = "Number of entry location compactions" + ) + private final Counter entryLocationCompactionCounter; @StatsDoc( name = RECLAIMED_DELETION_SPACE_BYTES, help = "Number of disk space bytes reclaimed via deleting entry log files" @@ -105,6 +111,7 @@ public GarbageCollectorStats(StatsLogger statsLogger, this.minorCompactionCounter = statsLogger.getCounter(MINOR_COMPACTION_COUNT); this.majorCompactionCounter = statsLogger.getCounter(MAJOR_COMPACTION_COUNT); + this.entryLocationCompactionCounter = statsLogger.getCounter(ENTRY_LOCATION_COMPACTION_COUNT); this.reclaimedSpaceViaCompaction = statsLogger.getCounter(RECLAIMED_COMPACTION_SPACE_BYTES); this.reclaimedSpaceViaDeletes = statsLogger.getCounter(RECLAIMED_DELETION_SPACE_BYTES); this.gcThreadRuntime = statsLogger.getOpStatsLogger(THREAD_RUNTIME); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java index 61aebd8e1a1..867ba905ffe 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java @@ -313,6 +313,8 @@ public boolean isMinorGcSuspended() { public void entryLocationCompact() { if (entryLocationIndex.isCompacting()) { // RocksDB already running compact. + log.info("Compacting directory {}, skipping this entryLocationCompaction this time.", + entryLocationIndex.getEntryLocationDBPath()); return; } cleanupExecutor.execute(() -> { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java index 019144e9942..1f04e53f575 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java @@ -105,6 +105,7 @@ public class ServerConfiguration extends AbstractConfigurationIf it is set to less than zero, the entry location compaction is disabled. + * + * @return high water mark. + */ + public long getEntryLocationCompactionInterval() { + return getLong(ENTRY_LOCATION_COMPACTION_INTERVAL, -1); + } + + /** + * Set interval to run entry location compaction. + * + * @see #getMajorCompactionInterval() + * + * @param interval + * Interval to run entry location compaction + * @return server configuration + */ + public ServerConfiguration setEntryLocationCompactionInterval(long interval) { + setProperty(ENTRY_LOCATION_COMPACTION_INTERVAL, interval); + return this; + } + /** * Should we remove pages from page cache after force write. * @@ -3187,6 +3213,10 @@ public void validate() throws ConfigurationException { if (getMajorCompactionInterval() > 0 && getMajorCompactionInterval() * SECOND < getGcWaitTime()) { throw new ConfigurationException("majorCompactionInterval should be >= gcWaitTime."); } + if (getEntryLocationCompactionInterval() > 0 + && getEntryLocationCompactionInterval() * SECOND < getGcWaitTime()) { + throw new ConfigurationException("entryLocationCompactionInterval should be >= gcWaitTime."); + } } /** diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/conf/TestServerConfiguration.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/conf/TestServerConfiguration.java index 04ac87818f7..d8aa62d0d23 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/conf/TestServerConfiguration.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/conf/TestServerConfiguration.java @@ -158,6 +158,7 @@ public void testEntryLogSizeLimit() throws ConfigurationException { public void testCompactionSettings() throws ConfigurationException { ServerConfiguration conf = new ServerConfiguration(); long major, minor; + long entryLocationCompactionInterval; // Default Values major = conf.getMajorCompactionMaxTimeMillis(); @@ -239,5 +240,24 @@ public void testCompactionSettings() throws ConfigurationException { minorThreshold = conf.getMinorCompactionThreshold(); Assert.assertEquals(0.6, majorThreshold, 0.00001); Assert.assertEquals(0.3, minorThreshold, 0.00001); + + // Default Values + entryLocationCompactionInterval = conf.getEntryLocationCompactionInterval(); + Assert.assertEquals(-1, entryLocationCompactionInterval); + + // Set entry location compaction + conf.setEntryLocationCompactionInterval(3600); + entryLocationCompactionInterval = conf.getEntryLocationCompactionInterval(); + Assert.assertEquals(3600, entryLocationCompactionInterval); + + conf.setEntryLocationCompactionInterval(550); + try { + conf.validate(); + fail(); + } catch (ConfigurationException ignore) { + } + + conf.setEntryLocationCompactionInterval(650); + conf.validate(); } } diff --git a/conf/bk_server.conf b/conf/bk_server.conf index c0a021418d9..be3d845fadc 100755 --- a/conf/bk_server.conf +++ b/conf/bk_server.conf @@ -581,6 +581,11 @@ ledgerDirectories=/tmp/bk-data # Set the rate at which compaction will readd entries. The unit is bytes added per second. # compactionRateByBytes=1000000 +# Interval to run entry location compaction, in seconds +# If it is set to less than zero, the entry location compaction is disabled. +# Note: should be greater than gcWaitTime. +# entryLocationCompactionInterval=-1 + # Flag to enable/disable transactional compaction. If it is set to true, it will use transactional compaction, # which it will use new entry log files to store compacted entries during compaction; if it is set to false, # it will use normal compaction, which it shares same entry log file with normal add operations. From 143fc78b97f9aba495f2d98fc3c52a30ff05e695 Mon Sep 17 00:00:00 2001 From: Dmitry Kryukov Date: Sat, 6 Jul 2024 04:12:13 +0300 Subject: [PATCH 08/55] fix: fix resource leaks in 3 classes (#4449) (cherry picked from commit a164bcae6462f0f7078372cae3549d91989f7383) (cherry picked from commit 47c3fc37a075b0bb991f8ecabb91415a04dcb1ec) --- .../http/service/DeleteLedgerService.java | 21 ++++++++++--------- .../BKDistributedLogNamespace.java | 7 ++++--- .../impl/BKNamespaceDriver.java | 3 +-- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/DeleteLedgerService.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/DeleteLedgerService.java index b4d465650c3..57cc127908d 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/DeleteLedgerService.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/DeleteLedgerService.java @@ -56,19 +56,20 @@ public HttpServiceResponse handle(HttpServiceRequest request) throws Exception { if (params != null && params.containsKey("ledger_id")) { ClientConfiguration clientConf = new ClientConfiguration(); clientConf.addConfiguration(conf); - BookKeeper bk = new BookKeeper(clientConf); - Long ledgerId = Long.parseLong(params.get("ledger_id")); + try (BookKeeper bk = new BookKeeper(clientConf)) { + Long ledgerId = Long.parseLong(params.get("ledger_id")); - bk.deleteLedger(ledgerId); + bk.deleteLedger(ledgerId); - String output = "Deleted ledger: " + ledgerId; - String jsonResponse = JsonUtil.toJson(output); - if (LOG.isDebugEnabled()) { - LOG.debug("output body:" + jsonResponse); + String output = "Deleted ledger: " + ledgerId; + String jsonResponse = JsonUtil.toJson(output); + if (LOG.isDebugEnabled()) { + LOG.debug("output body:" + jsonResponse); + } + response.setBody(jsonResponse); + response.setCode(HttpServer.StatusCode.OK); + return response; } - response.setBody(jsonResponse); - response.setCode(HttpServer.StatusCode.OK); - return response; } else { response.setCode(HttpServer.StatusCode.NOT_FOUND); response.setBody("Not ledger found. Should provide ledger_id="); diff --git a/stream/distributedlog/core/src/main/java/org/apache/distributedlog/BKDistributedLogNamespace.java b/stream/distributedlog/core/src/main/java/org/apache/distributedlog/BKDistributedLogNamespace.java index 0a3677d9560..8ed2cfeb362 100644 --- a/stream/distributedlog/core/src/main/java/org/apache/distributedlog/BKDistributedLogNamespace.java +++ b/stream/distributedlog/core/src/main/java/org/apache/distributedlog/BKDistributedLogNamespace.java @@ -158,12 +158,13 @@ public void deleteLog(String logName) if (!uri.isPresent()) { throw new LogNotFoundException("Log " + logName + " isn't found."); } - DistributedLogManager dlm = openLogInternal( + try (DistributedLogManager dlm = openLogInternal( uri.get(), logName, Optional.empty(), - Optional.empty()); - dlm.delete(); + Optional.empty())) { + dlm.delete(); + } } @Override diff --git a/stream/distributedlog/core/src/main/java/org/apache/distributedlog/impl/BKNamespaceDriver.java b/stream/distributedlog/core/src/main/java/org/apache/distributedlog/impl/BKNamespaceDriver.java index ea17ed76931..957995b3fd6 100644 --- a/stream/distributedlog/core/src/main/java/org/apache/distributedlog/impl/BKNamespaceDriver.java +++ b/stream/distributedlog/core/src/main/java/org/apache/distributedlog/impl/BKNamespaceDriver.java @@ -533,8 +533,7 @@ public Map enumerateLogsWithMetadataInNamespace() String namespaceRootPath = namespace.getPath(); HashMap result = new HashMap(); ZooKeeperClient zkc = writerZKC; - try { - ZooKeeper zk = Utils.sync(zkc, namespaceRootPath); + try (ZooKeeper zk = Utils.sync(zkc, namespaceRootPath)) { Stat currentStat = zk.exists(namespaceRootPath, false); if (currentStat == null) { return result; From 068db0c7f2313a7ed1fb6835b8c3fa04a705a2ca Mon Sep 17 00:00:00 2001 From: Masahiro Sakamoto Date: Wed, 10 Jul 2024 15:07:29 +0900 Subject: [PATCH 09/55] Enable ZooKeeper client to establish connection in read-only mode (#4244) ### Motivation If the system property `readonlymode.enabled` is set to true on a ZooKeeper server, read-only mode is enabled. Data can be read from the server in read-only mode even if that server is split from the quorum. https://zookeeper.apache.org/doc/current/zookeeperAdmin.html#Experimental+Options%2FFeatures To connect to the server in read-only mode, the client must also allow read-only mode. The `ZooKeeperClient` class in the bookkeeper repository also has an option called `allowReadOnlyMode`. https://github.com/apache/bookkeeper/blob/15171e1904f7196d8e9f4116ab2aecdf582e0032/bookkeeper-server/src/main/java/org/apache/bookkeeper/zookeeper/ZooKeeperClient.java#L219-L222 However, even if this option is set to true, the connection to the server in read-only mode will actually fail. The cause is in the `ZooKeeperWatcherBase` class. When the `ZooKeeperWatcherBase` class receives the `SyncConnected` event, it releases `clientConnectLatch` and assumes that the connection is complete. https://github.com/apache/bookkeeper/blob/15171e1904f7196d8e9f4116ab2aecdf582e0032/bookkeeper-server/src/main/java/org/apache/bookkeeper/zookeeper/ZooKeeperWatcherBase.java#L128-L144 However, if the server is in read-only mode, it will receive `ConnectedReadOnly` instead of `SyncConnected`. This causes the connection to time out without being completed. ### Changes Modified the switch statement in the `ZooKeeperWatcherBase` class to release `clientConnectLatch` when `ConnectedReadOnly` is received if the `allowReadOnlyMode` option is true. By the way, `allowReadOnlyMode` is never set to true in BookKeeper. So this change would be useless for BookKeeper. However, it is useful for Pulsar. Because Pulsar also uses `ZooKeeperWatcherBase` and needs to be able to connect to ZooKeeper in read-only mode. https://github.com/apache/pulsar/blob/cba1600d0f6a82f1ea194f3214a80f283fe8dc27/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/PulsarZooKeeperClient.java#L242-L244 (cherry picked from commit 4d50a445553a3a5d4af3aed973ac64bcd4132789) (cherry picked from commit 6fbc3c06445784d3186d0ecd52a28a2ff2e78904) --- .../bookkeeper/zookeeper/ZooKeeperClient.java | 6 ++-- .../zookeeper/ZooKeeperWatcherBase.java | 19 +++++++++--- .../BookKeeperClientZKSessionExpiry.java | 2 +- .../bookkeeper/client/BookKeeperTest.java | 2 +- .../replication/TestReplicationWorker.java | 2 +- .../bookkeeper/test/ZooKeeperCluster.java | 2 +- .../bookkeeper/test/ZooKeeperClusterUtil.java | 8 +++++ .../zookeeper/TestZooKeeperClient.java | 30 ++++++++++++++++++- 8 files changed, 59 insertions(+), 12 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/zookeeper/ZooKeeperClient.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/zookeeper/ZooKeeperClient.java index c742f829e0f..3bddcb2f2ba 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/zookeeper/ZooKeeperClient.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/zookeeper/ZooKeeperClient.java @@ -241,9 +241,9 @@ public ZooKeeperClient build() throws IOException, KeeperException, InterruptedE // Create a watcher manager StatsLogger watcherStatsLogger = statsLogger.scope("watcher"); - ZooKeeperWatcherBase watcherManager = - null == watchers ? new ZooKeeperWatcherBase(sessionTimeoutMs, watcherStatsLogger) : - new ZooKeeperWatcherBase(sessionTimeoutMs, watchers, watcherStatsLogger); + ZooKeeperWatcherBase watcherManager = (null == watchers) + ? new ZooKeeperWatcherBase(sessionTimeoutMs, allowReadOnlyMode, watcherStatsLogger) + : new ZooKeeperWatcherBase(sessionTimeoutMs, allowReadOnlyMode, watchers, watcherStatsLogger); ZooKeeperClient client = new ZooKeeperClient( connectString, sessionTimeoutMs, diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/zookeeper/ZooKeeperWatcherBase.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/zookeeper/ZooKeeperWatcherBase.java index 758f079d0da..e44a5f364cd 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/zookeeper/ZooKeeperWatcherBase.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/zookeeper/ZooKeeperWatcherBase.java @@ -44,6 +44,7 @@ public class ZooKeeperWatcherBase implements Watcher { .getLogger(ZooKeeperWatcherBase.class); private final int zkSessionTimeOut; + private final boolean allowReadOnlyMode; private volatile CountDownLatch clientConnectLatch = new CountDownLatch(1); private final CopyOnWriteArraySet childWatchers = new CopyOnWriteArraySet(); @@ -53,18 +54,20 @@ public class ZooKeeperWatcherBase implements Watcher { private final ConcurrentHashMap eventCounters = new ConcurrentHashMap(); - public ZooKeeperWatcherBase(int zkSessionTimeOut) { - this(zkSessionTimeOut, NullStatsLogger.INSTANCE); + public ZooKeeperWatcherBase(int zkSessionTimeOut, boolean allowReadOnlyMode) { + this(zkSessionTimeOut, allowReadOnlyMode, NullStatsLogger.INSTANCE); } - public ZooKeeperWatcherBase(int zkSessionTimeOut, StatsLogger statsLogger) { - this(zkSessionTimeOut, new HashSet(), statsLogger); + public ZooKeeperWatcherBase(int zkSessionTimeOut, boolean allowReadOnlyMode, StatsLogger statsLogger) { + this(zkSessionTimeOut, allowReadOnlyMode, new HashSet(), statsLogger); } public ZooKeeperWatcherBase(int zkSessionTimeOut, + boolean allowReadOnlyMode, Set childWatchers, StatsLogger statsLogger) { this.zkSessionTimeOut = zkSessionTimeOut; + this.allowReadOnlyMode = allowReadOnlyMode; this.childWatchers.addAll(childWatchers); this.statsLogger = statsLogger; } @@ -130,6 +133,14 @@ public void process(WatchedEvent event) { LOG.info("ZooKeeper client is connected now."); clientConnectLatch.countDown(); break; + case ConnectedReadOnly: + if (allowReadOnlyMode) { + LOG.info("ZooKeeper client is connected in read-only mode now."); + clientConnectLatch.countDown(); + } else { + LOG.warn("ZooKeeper client is connected in read-only mode, which is not allowed."); + } + break; case Disconnected: LOG.info("ZooKeeper client is disconnected from zookeeper now," + " but it is OK unless we received EXPIRED event."); diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookKeeperClientZKSessionExpiry.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookKeeperClientZKSessionExpiry.java index b1a8bb66dd7..c72834397e0 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookKeeperClientZKSessionExpiry.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookKeeperClientZKSessionExpiry.java @@ -51,7 +51,7 @@ public void run() { byte[] sessionPasswd = bkc.getZkHandle().getSessionPasswd(); try { - ZooKeeperWatcherBase watcher = new ZooKeeperWatcherBase(10000); + ZooKeeperWatcherBase watcher = new ZooKeeperWatcherBase(10000, false); ZooKeeper zk = new ZooKeeper(zkUtil.getZooKeeperConnectString(), 10000, watcher, sessionId, sessionPasswd); zk.close(); diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookKeeperTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookKeeperTest.java index 8f64c256a43..386a83709e3 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookKeeperTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookKeeperTest.java @@ -901,7 +901,7 @@ protected ZooKeeper createZooKeeper() throws IOException { public void testZKConnectionLossForLedgerCreation() throws Exception { int zkSessionTimeOut = 10000; AtomicLong ledgerIdToInjectFailure = new AtomicLong(INVALID_LEDGERID); - ZooKeeperWatcherBase zooKeeperWatcherBase = new ZooKeeperWatcherBase(zkSessionTimeOut, + ZooKeeperWatcherBase zooKeeperWatcherBase = new ZooKeeperWatcherBase(zkSessionTimeOut, false, NullStatsLogger.INSTANCE); MockZooKeeperClient zkFaultInjectionWrapper = new MockZooKeeperClient(zkUtil.getZooKeeperConnectString(), zkSessionTimeOut, zooKeeperWatcherBase, ledgerIdToInjectFailure); diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java index a9914233899..28f9915cb2c 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java @@ -1040,7 +1040,7 @@ public void testRWShutDownInTheCaseOfZKOperationFailures() throws Exception { * create MockZooKeeperClient instance and wait for it to be connected. */ int zkSessionTimeOut = 10000; - ZooKeeperWatcherBase zooKeeperWatcherBase = new ZooKeeperWatcherBase(zkSessionTimeOut, + ZooKeeperWatcherBase zooKeeperWatcherBase = new ZooKeeperWatcherBase(zkSessionTimeOut, false, NullStatsLogger.INSTANCE); MockZooKeeperClient zkFaultInjectionWrapper = new MockZooKeeperClient(zkUtil.getZooKeeperConnectString(), zkSessionTimeOut, zooKeeperWatcherBase); diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/ZooKeeperCluster.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/ZooKeeperCluster.java index 08ecbd7cc12..b0e828bd5ca 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/ZooKeeperCluster.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/ZooKeeperCluster.java @@ -64,7 +64,7 @@ void sleepCluster(int time, TimeUnit timeUnit, CountDownLatch l) default void expireSession(ZooKeeper zk) throws Exception { long id = zk.getSessionId(); byte[] password = zk.getSessionPasswd(); - ZooKeeperWatcherBase w = new ZooKeeperWatcherBase(10000); + ZooKeeperWatcherBase w = new ZooKeeperWatcherBase(10000, false); ZooKeeper zk2 = new ZooKeeper(getZooKeeperConnectString(), zk.getSessionTimeout(), w, id, password); w.waitForConnection(); zk2.close(); diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/ZooKeeperClusterUtil.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/ZooKeeperClusterUtil.java index 3eace4a62c5..6dbf182110f 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/ZooKeeperClusterUtil.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/ZooKeeperClusterUtil.java @@ -139,4 +139,12 @@ public void killCluster() throws Exception { public void sleepCluster(int time, TimeUnit timeUnit, CountDownLatch l) throws InterruptedException, IOException { throw new UnsupportedOperationException("sleepServer operation is not supported for ZooKeeperClusterUtil"); } + + public void stopPeer(int id) throws Exception { + quorumUtil.shutdown(id); + } + + public void enableLocalSession(boolean localSessionEnabled) { + quorumUtil.enableLocalSession(localSessionEnabled); + } } diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/zookeeper/TestZooKeeperClient.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/zookeeper/TestZooKeeperClient.java index 3c1bd65a64b..9e2ccf16e87 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/zookeeper/TestZooKeeperClient.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/zookeeper/TestZooKeeperClient.java @@ -171,7 +171,7 @@ public void process(WatchedEvent event) { }; final int timeout = 2000; ZooKeeperWatcherBase watcherManager = - new ZooKeeperWatcherBase(timeout).addChildWatcher(testWatcher); + new ZooKeeperWatcherBase(timeout, false).addChildWatcher(testWatcher); List watchers = new ArrayList(1); watchers.add(testWatcher); ZooKeeperClient client = new ShutdownZkServerClient( @@ -895,4 +895,32 @@ public void processResult(int rc, String path, Object ctx) { logger.info("Delete children from znode " + path); } + @Test + public void testAllowReadOnlyMode() throws Exception { + if (zkUtil instanceof ZooKeeperClusterUtil) { + System.setProperty("readonlymode.enabled", "true"); + ((ZooKeeperClusterUtil) zkUtil).enableLocalSession(true); + zkUtil.restartCluster(); + Thread.sleep(2000); + ((ZooKeeperClusterUtil) zkUtil).stopPeer(2); + ((ZooKeeperClusterUtil) zkUtil).stopPeer(3); + } + + try (ZooKeeperClient client = ZooKeeperClient.newBuilder() + .connectString(zkUtil.getZooKeeperConnectString()) + .sessionTimeoutMs(30000) + .watchers(new HashSet()) + .operationRetryPolicy(retryPolicy) + .allowReadOnlyMode(true) + .build()) { + Assert.assertTrue("Client failed to connect a ZooKeeper in read-only mode.", + client.getState().isConnected()); + } finally { + if (zkUtil instanceof ZooKeeperClusterUtil) { + System.setProperty("readonlymode.enabled", "false"); + ((ZooKeeperClusterUtil) zkUtil).enableLocalSession(false); + } + } + } + } From f3cca704b66d3835c5df10041e628cfb310c9c06 Mon Sep 17 00:00:00 2001 From: Yan Zhao Date: Mon, 15 Jul 2024 14:25:46 +0800 Subject: [PATCH 10/55] Allocator support exitOnOutOfMemory config. (#3984) * Allocator support exitOnOutOfMemory config. (cherry picked from commit 15b106c7610f11dfc091c8f73c28d1666113c25f) (cherry picked from commit a05bc53c59863361897a9706b29b981b3c55699e) --- bookkeeper-common-allocator/pom.xml | 10 +++ .../allocator/ByteBufAllocatorBuilder.java | 2 + .../impl/ByteBufAllocatorBuilderImpl.java | 9 +- .../allocator/impl/ByteBufAllocatorImpl.java | 34 ++++++-- .../bookkeeper/common/util/ShutdownUtil.java | 86 +++++++++++++++++++ .../bookkeeper/common/util/package-info.java | 21 +++++ .../impl/ByteBufAllocatorBuilderTest.java | 28 ++++++ .../bookkeeper/bookie/BookieResources.java | 1 + .../apache/bookkeeper/client/BookKeeper.java | 1 + .../conf/AbstractConfiguration.java | 10 +++ .../conf/AbstractConfigurationTest.java | 10 +++ .../tools/perf/journal/JournalWriter.java | 1 + 12 files changed, 206 insertions(+), 7 deletions(-) create mode 100644 bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/util/ShutdownUtil.java create mode 100644 bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/util/package-info.java diff --git a/bookkeeper-common-allocator/pom.xml b/bookkeeper-common-allocator/pom.xml index 1298bafb7be..755b28fd764 100644 --- a/bookkeeper-common-allocator/pom.xml +++ b/bookkeeper-common-allocator/pom.xml @@ -29,6 +29,16 @@ io.netty netty-buffer + + org.mockito + mockito-core + test + + + org.mockito + mockito-inline + test + diff --git a/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/allocator/ByteBufAllocatorBuilder.java b/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/allocator/ByteBufAllocatorBuilder.java index 3e36a23d170..cb244140b35 100644 --- a/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/allocator/ByteBufAllocatorBuilder.java +++ b/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/allocator/ByteBufAllocatorBuilder.java @@ -92,4 +92,6 @@ static ByteBufAllocatorBuilder create() { *

Default is {@link LeakDetectionPolicy#Disabled} */ ByteBufAllocatorBuilder leakDetectionPolicy(LeakDetectionPolicy leakDetectionPolicy); + + ByteBufAllocatorBuilder exitOnOutOfMemory(boolean exitOnOutOfMemory); } diff --git a/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/allocator/impl/ByteBufAllocatorBuilderImpl.java b/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/allocator/impl/ByteBufAllocatorBuilderImpl.java index 69c57232aff..4b5469a3f7e 100644 --- a/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/allocator/impl/ByteBufAllocatorBuilderImpl.java +++ b/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/allocator/impl/ByteBufAllocatorBuilderImpl.java @@ -37,11 +37,12 @@ public class ByteBufAllocatorBuilderImpl implements ByteBufAllocatorBuilder { OutOfMemoryPolicy outOfMemoryPolicy = OutOfMemoryPolicy.FallbackToHeap; Consumer outOfMemoryListener = null; LeakDetectionPolicy leakDetectionPolicy = LeakDetectionPolicy.Disabled; + boolean exitOnOutOfMemory = false; @Override public ByteBufAllocatorWithOomHandler build() { return new ByteBufAllocatorImpl(pooledAllocator, unpooledAllocator, poolingPolicy, poolingConcurrency, - outOfMemoryPolicy, outOfMemoryListener, leakDetectionPolicy); + outOfMemoryPolicy, outOfMemoryListener, leakDetectionPolicy, exitOnOutOfMemory); } @Override @@ -86,4 +87,10 @@ public ByteBufAllocatorBuilder leakDetectionPolicy(LeakDetectionPolicy leakDetec return this; } + @Override + public ByteBufAllocatorBuilder exitOnOutOfMemory(boolean exitOnOutOfMemory) { + this.exitOnOutOfMemory = exitOnOutOfMemory; + return this; + } + } diff --git a/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/allocator/impl/ByteBufAllocatorImpl.java b/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/allocator/impl/ByteBufAllocatorImpl.java index 87582cca92c..3bc06f8e7ea 100644 --- a/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/allocator/impl/ByteBufAllocatorImpl.java +++ b/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/allocator/impl/ByteBufAllocatorImpl.java @@ -29,6 +29,7 @@ import org.apache.bookkeeper.common.allocator.LeakDetectionPolicy; import org.apache.bookkeeper.common.allocator.OutOfMemoryPolicy; import org.apache.bookkeeper.common.allocator.PoolingPolicy; +import org.apache.bookkeeper.common.util.ShutdownUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,15 +49,24 @@ public class ByteBufAllocatorImpl extends AbstractByteBufAllocator implements By private final PoolingPolicy poolingPolicy; private final OutOfMemoryPolicy outOfMemoryPolicy; private Consumer outOfMemoryListener; + private final boolean exitOnOutOfMemory; ByteBufAllocatorImpl(ByteBufAllocator pooledAllocator, ByteBufAllocator unpooledAllocator, PoolingPolicy poolingPolicy, int poolingConcurrency, OutOfMemoryPolicy outOfMemoryPolicy, Consumer outOfMemoryListener, LeakDetectionPolicy leakDetectionPolicy) { - super(poolingPolicy == PoolingPolicy.PooledDirect /* preferDirect */); + this(pooledAllocator, unpooledAllocator, poolingPolicy, poolingConcurrency, outOfMemoryPolicy, + outOfMemoryListener, leakDetectionPolicy, false); + } + ByteBufAllocatorImpl(ByteBufAllocator pooledAllocator, ByteBufAllocator unpooledAllocator, + PoolingPolicy poolingPolicy, int poolingConcurrency, OutOfMemoryPolicy outOfMemoryPolicy, + Consumer outOfMemoryListener, + LeakDetectionPolicy leakDetectionPolicy, boolean exitOnOutOfMemory) { + super(poolingPolicy == PoolingPolicy.PooledDirect /* preferDirect */); this.poolingPolicy = poolingPolicy; this.outOfMemoryPolicy = outOfMemoryPolicy; + this.exitOnOutOfMemory = exitOnOutOfMemory; if (outOfMemoryListener == null) { this.outOfMemoryListener = (v) -> { log.error("Unable to allocate memory", v); @@ -146,7 +156,7 @@ protected ByteBuf newHeapBuffer(int initialCapacity, int maxCapacity) { : unpooledAllocator; return alloc.heapBuffer(initialCapacity, maxCapacity); } catch (OutOfMemoryError e) { - outOfMemoryListener.accept(e); + consumeOOMError(e); throw e; } } @@ -166,12 +176,12 @@ private ByteBuf newDirectBuffer(int initialCapacity, int maxCapacity, boolean ca try { return unpooledAllocator.heapBuffer(initialCapacity, maxCapacity); } catch (OutOfMemoryError e2) { - outOfMemoryListener.accept(e2); + consumeOOMError(e2); throw e2; } } else { // ThrowException - outOfMemoryListener.accept(e); + consumeOOMError(e); throw e; } } @@ -181,12 +191,24 @@ private ByteBuf newDirectBuffer(int initialCapacity, int maxCapacity, boolean ca try { return unpooledAllocator.directBuffer(initialCapacity, maxCapacity); } catch (OutOfMemoryError e) { - outOfMemoryListener.accept(e); - throw e; + consumeOOMError(e); + throw e; } } } + private void consumeOOMError(OutOfMemoryError outOfMemoryError) { + try { + outOfMemoryListener.accept(outOfMemoryError); + } catch (Throwable e) { + log.warn("Consume outOfMemory error failed.", e); + } + if (exitOnOutOfMemory) { + log.info("Exiting JVM process for OOM error: {}", outOfMemoryError.getMessage(), outOfMemoryError); + ShutdownUtil.triggerImmediateForcefulShutdown(); + } + } + @Override public boolean isDirectBufferPooled() { return pooledAllocator != null && pooledAllocator.isDirectBufferPooled(); diff --git a/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/util/ShutdownUtil.java b/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/util/ShutdownUtil.java new file mode 100644 index 00000000000..a398b57fe74 --- /dev/null +++ b/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/util/ShutdownUtil.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.bookkeeper.common.util; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import lombok.extern.slf4j.Slf4j; + +/** + * Forked from Pulsar. + */ +@Slf4j +public class ShutdownUtil { + private static final Method log4j2ShutdownMethod; + + static { + // use reflection to find org.apache.logging.log4j.LogManager.shutdown method + Method shutdownMethod = null; + try { + shutdownMethod = Class.forName("org.apache.logging.log4j.LogManager") + .getMethod("shutdown"); + } catch (ClassNotFoundException | NoSuchMethodException e) { + // ignore when Log4j2 isn't found, log at debug level + log.debug("Cannot find org.apache.logging.log4j.LogManager.shutdown method", e); + } + log4j2ShutdownMethod = shutdownMethod; + } + + /** + * Triggers an immediate forceful shutdown of the current process. + * + * @param status Termination status. By convention, a nonzero status code indicates abnormal termination. + * @see Runtime#halt(int) + */ + public static void triggerImmediateForcefulShutdown(int status) { + triggerImmediateForcefulShutdown(status, true); + } + public static void triggerImmediateForcefulShutdown(int status, boolean logging) { + try { + if (status != 0 && logging) { + log.warn("Triggering immediate shutdown of current process with status {}", status, + new Exception("Stacktrace for immediate shutdown")); + } + shutdownLogging(); + } finally { + Runtime.getRuntime().halt(status); + } + } + + private static void shutdownLogging() { + // flush log buffers and shutdown log4j2 logging to prevent log truncation + if (log4j2ShutdownMethod != null) { + try { + // use reflection to call org.apache.logging.log4j.LogManager.shutdown() + log4j2ShutdownMethod.invoke(null); + } catch (IllegalAccessException | InvocationTargetException e) { + log.error("Unable to call org.apache.logging.log4j.LogManager.shutdown using reflection.", e); + } + } + } + + /** + * Triggers an immediate forceful shutdown of the current process using 1 as the status code. + * + * @see Runtime#halt(int) + */ + public static void triggerImmediateForcefulShutdown() { + triggerImmediateForcefulShutdown(1); + } +} \ No newline at end of file diff --git a/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/util/package-info.java b/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/util/package-info.java new file mode 100644 index 00000000000..55031dd8f8d --- /dev/null +++ b/bookkeeper-common-allocator/src/main/java/org/apache/bookkeeper/common/util/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * defines the utilities for allocator used across the project. + */ +package org.apache.bookkeeper.common.util; \ No newline at end of file diff --git a/bookkeeper-common-allocator/src/test/java/org/apache/bookkeeper/common/allocator/impl/ByteBufAllocatorBuilderTest.java b/bookkeeper-common-allocator/src/test/java/org/apache/bookkeeper/common/allocator/impl/ByteBufAllocatorBuilderTest.java index 6f2538d6c81..40c41fa65bc 100644 --- a/bookkeeper-common-allocator/src/test/java/org/apache/bookkeeper/common/allocator/impl/ByteBufAllocatorBuilderTest.java +++ b/bookkeeper-common-allocator/src/test/java/org/apache/bookkeeper/common/allocator/impl/ByteBufAllocatorBuilderTest.java @@ -23,6 +23,7 @@ import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.when; import io.netty.buffer.ByteBuf; @@ -35,7 +36,10 @@ import org.apache.bookkeeper.common.allocator.ByteBufAllocatorBuilder; import org.apache.bookkeeper.common.allocator.OutOfMemoryPolicy; import org.apache.bookkeeper.common.allocator.PoolingPolicy; +import org.apache.bookkeeper.common.util.ShutdownUtil; import org.junit.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; /** * Tests for {@link ByteBufAllocatorBuilderImpl}. @@ -87,6 +91,30 @@ public void testOomWithException() { assertEquals(outOfDirectMemException, receivedException.get()); } + @Test() + public void testOomExit() { + ByteBufAllocator baseAlloc = mock(ByteBufAllocator.class); + when(baseAlloc.directBuffer(anyInt(), anyInt())).thenThrow(outOfDirectMemException); + + ByteBufAllocator alloc = ByteBufAllocatorBuilder.create() + .pooledAllocator(baseAlloc) + .outOfMemoryPolicy(OutOfMemoryPolicy.ThrowException) + .exitOnOutOfMemory(true) + .build(); + + MockedStatic mockedStatic = mockStatic(ShutdownUtil.class); + + try { + alloc.buffer(); + fail("Should have thrown exception"); + } catch (OutOfMemoryError e) { + // Expected + assertEquals(outOfDirectMemException, e); + } + + mockedStatic.verify(() -> ShutdownUtil.triggerImmediateForcefulShutdown(), Mockito.times(1)); + } + @Test public void testOomWithFallback() { ByteBufAllocator baseAlloc = mock(ByteBufAllocator.class); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieResources.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieResources.java index c9b71b9968d..755efd5be02 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieResources.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieResources.java @@ -70,6 +70,7 @@ public static ByteBufAllocatorWithOomHandler createAllocator(ServerConfiguration .poolingConcurrency(conf.getAllocatorPoolingConcurrency()) .outOfMemoryPolicy(conf.getAllocatorOutOfMemoryPolicy()) .leakDetectionPolicy(conf.getAllocatorLeakDetectionPolicy()) + .exitOnOutOfMemory(conf.exitOnOutOfMemory()) .build(); } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeper.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeper.java index d7043dc8c9a..0362aadcbaa 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeper.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeper.java @@ -478,6 +478,7 @@ public BookKeeper(ClientConfiguration conf, ZooKeeper zk, EventLoopGroup eventLo .poolingConcurrency(conf.getAllocatorPoolingConcurrency()) .outOfMemoryPolicy(conf.getAllocatorOutOfMemoryPolicy()) .leakDetectionPolicy(conf.getAllocatorLeakDetectionPolicy()) + .exitOnOutOfMemory(conf.exitOnOutOfMemory()) .build(); } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/AbstractConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/AbstractConfiguration.java index 438dc40983e..6d2a82bb551 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/AbstractConfiguration.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/AbstractConfiguration.java @@ -183,6 +183,7 @@ public abstract class AbstractConfiguration protected static final String ALLOCATOR_POOLING_CONCURRENCY = "allocatorPoolingConcurrency"; protected static final String ALLOCATOR_OOM_POLICY = "allocatorOutOfMemoryPolicy"; protected static final String ALLOCATOR_LEAK_DETECTION_POLICY = "allocatorLeakDetectionPolicy"; + protected static final String ALLOCATOR_EXIT_ON_OUT_OF_MEMORY = "allocatorExitOnOutOfMemory"; // option to limit stats logging public static final String LIMIT_STATS_LOGGING = "limitStatsLogging"; @@ -1156,6 +1157,15 @@ public T setAllocatorLeakDetectionPolicy(LeakDetectionPolicy leakDetectionPolicy return getThis(); } + public T setExitOnOutOfMemory(boolean exitOnOutOfMemory) { + this.setProperty(ALLOCATOR_EXIT_ON_OUT_OF_MEMORY, exitOnOutOfMemory); + return getThis(); + } + + public boolean exitOnOutOfMemory() { + return getBoolean(ALLOCATOR_EXIT_ON_OUT_OF_MEMORY, false); + } + /** * Return whether the busy-wait is enabled for BookKeeper and Netty IO threads. * diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/conf/AbstractConfigurationTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/conf/AbstractConfigurationTest.java index a6333a47d32..194ab2c68d0 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/conf/AbstractConfigurationTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/conf/AbstractConfigurationTest.java @@ -19,6 +19,8 @@ package org.apache.bookkeeper.conf; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.CALLS_REAL_METHODS; import static org.mockito.Mockito.mock; @@ -179,4 +181,12 @@ public void testAllocatorLeakDetectionPolicy() { System.getProperties().put(nettyLevelKey, nettyLevelStr); } } + + @Test + public void testExitOnOutOfMemory() { + assertFalse(conf.exitOnOutOfMemory()); + conf.setExitOnOutOfMemory(true); + assertTrue(conf.exitOnOutOfMemory()); + } + } diff --git a/tools/perf/src/main/java/org/apache/bookkeeper/tools/perf/journal/JournalWriter.java b/tools/perf/src/main/java/org/apache/bookkeeper/tools/perf/journal/JournalWriter.java index 383ccfb9825..e287fbd94af 100644 --- a/tools/perf/src/main/java/org/apache/bookkeeper/tools/perf/journal/JournalWriter.java +++ b/tools/perf/src/main/java/org/apache/bookkeeper/tools/perf/journal/JournalWriter.java @@ -495,6 +495,7 @@ private static ByteBufAllocator getAllocator(ServerConfiguration conf) { log.error("Unable to allocate memory, exiting bookie", ex); }) .leakDetectionPolicy(conf.getAllocatorLeakDetectionPolicy()) + .exitOnOutOfMemory(conf.exitOnOutOfMemory()) .build(); } From 042423fa44f85f56c0f11e78876c8f8a8a852a5e Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Mon, 29 Jul 2024 17:10:19 +0800 Subject: [PATCH 11/55] [fix][ci] Fix OWASP Dependency Check download by using NVD API key (#4473) Signed-off-by: ZhangJian He (cherry picked from commit 7ab29e6a7a12921d87604744bc26ae85c8fde351) (cherry picked from commit 8996ccdb67e5b8c6b32d653a872acc64fad6d89d) --- .github/workflows/bk-ci.yml | 1 + pom.xml | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/bk-ci.yml b/.github/workflows/bk-ci.yml index d5b258bd007..9a8af608bb5 100644 --- a/.github/workflows/bk-ci.yml +++ b/.github/workflows/bk-ci.yml @@ -34,6 +34,7 @@ on: env: MAVEN_OPTS: -Xss1500k -Xmx1500m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 + NIST_NVD_API_KEY: ${{ secrets.NIST_NVD_API_KEY }} concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/pom.xml b/pom.xml index a831f281c72..aebe9f0776d 100644 --- a/pom.xml +++ b/pom.xml @@ -192,7 +192,7 @@ 3.3.1 3.12.1 3.2.5 - 9.2.0 + 10.0.2 3.10.1 1.4.1.Final 0.6.1 @@ -1209,6 +1209,10 @@ ${dependency-check-maven.version} false + + NIST_NVD_API_KEY + + src/owasp-dependency-check-suppressions.xml From af0642a111ae3d202393ad894b7d408eee7f9866 Mon Sep 17 00:00:00 2001 From: Aurora Twinkle Date: Sun, 21 Jul 2024 05:48:11 +0800 Subject: [PATCH 12/55] fix[rocksdb]: fix error rocksdb default config for CFOptions (#4466) * fix[rocksdb]: fix error rocksdb default config for CFOptions --------- Co-authored-by: duanlinlin (cherry picked from commit 4ca020a5f0805a748a1e8d983015e5221a534fd7) (cherry picked from commit 3b06ecf938718e7089e9097631bcd02e62abaadb) --- .../test/resources/conf/default_rocksdb.conf | 11 +++++++--- .../conf/entry_location_rocksdb.conf | 4 ++-- .../conf/ledger_metadata_rocksdb.conf | 11 +++++++--- .../test_entry_location_rocksdb.conf | 22 ++++++++++++++++++- conf/default_rocksdb.conf.default | 9 ++++++-- conf/entry_location_rocksdb.conf.default | 8 +++++-- conf/ledger_metadata_rocksdb.conf.default | 9 ++++++-- 7 files changed, 59 insertions(+), 15 deletions(-) diff --git a/bookkeeper-server/src/test/resources/conf/default_rocksdb.conf b/bookkeeper-server/src/test/resources/conf/default_rocksdb.conf index 0f3a08779ed..df433a33e96 100644 --- a/bookkeeper-server/src/test/resources/conf/default_rocksdb.conf +++ b/bookkeeper-server/src/test/resources/conf/default_rocksdb.conf @@ -23,7 +23,12 @@ info_log_level=INFO_LEVEL # set by jni: options.setKeepLogFileNum keep_log_file_num=30 - -[CFOptions "default"] # set by jni: options.setLogFileTimeToRoll - log_file_time_to_roll=86400 \ No newline at end of file + log_file_time_to_roll=86400 + + [CFOptions "default"] + #no default setting in CFOptions + +[TableOptions/BlockBasedTable "default"] + # set by jni: tableOptions.setChecksumType + checksum=kxxHash \ No newline at end of file diff --git a/bookkeeper-server/src/test/resources/conf/entry_location_rocksdb.conf b/bookkeeper-server/src/test/resources/conf/entry_location_rocksdb.conf index 6f6c1b4d052..260111fa140 100644 --- a/bookkeeper-server/src/test/resources/conf/entry_location_rocksdb.conf +++ b/bookkeeper-server/src/test/resources/conf/entry_location_rocksdb.conf @@ -26,7 +26,7 @@ # set by jni: options.setLogFileTimeToRoll log_file_time_to_roll=86400 # set by jni: options.setMaxBackgroundJobs or options.setIncreaseParallelism - max_background_jobs=2 + max_background_jobs=32 # set by jni: options.setMaxSubcompactions max_subcompactions=1 # set by jni: options.setMaxTotalWalSize @@ -57,7 +57,7 @@ [TableOptions/BlockBasedTable "default"] # set by jni: tableOptions.setBlockSize block_size=65536 - # set by jni: tableOptions.setBlockCache +# set by jni: tableOptions.setBlockCache, default value is: maxDirectMemory() / ledgerDirsSize / 10; block_cache=206150041 # set by jni: tableOptions.setFormatVersion format_version=2 diff --git a/bookkeeper-server/src/test/resources/conf/ledger_metadata_rocksdb.conf b/bookkeeper-server/src/test/resources/conf/ledger_metadata_rocksdb.conf index 0f3a08779ed..f84c0167ed7 100644 --- a/bookkeeper-server/src/test/resources/conf/ledger_metadata_rocksdb.conf +++ b/bookkeeper-server/src/test/resources/conf/ledger_metadata_rocksdb.conf @@ -23,7 +23,12 @@ info_log_level=INFO_LEVEL # set by jni: options.setKeepLogFileNum keep_log_file_num=30 - -[CFOptions "default"] # set by jni: options.setLogFileTimeToRoll - log_file_time_to_roll=86400 \ No newline at end of file + log_file_time_to_roll=86400 + + [CFOptions "default"] + #no default setting in CFOptions + +[TableOptions/BlockBasedTable "default"] + # set by jni: tableOptions.setChecksumType + checksum=kxxHash \ No newline at end of file diff --git a/bookkeeper-server/src/test/resources/test_entry_location_rocksdb.conf b/bookkeeper-server/src/test/resources/test_entry_location_rocksdb.conf index 9d1c3e08c47..f0971b8d310 100644 --- a/bookkeeper-server/src/test/resources/test_entry_location_rocksdb.conf +++ b/bookkeeper-server/src/test/resources/test_entry_location_rocksdb.conf @@ -19,10 +19,22 @@ [DBOptions] # set by jni: options.setCreateIfMissing create_if_missing=true + # set by jni: options.setInfoLogLevel + info_log_level=INFO_LEVEL # set by jni: options.setKeepLogFileNum keep_log_file_num=1 + # set by jni: options.setLogFileTimeToRoll + log_file_time_to_roll=86400 + # set by jni: options.setMaxBackgroundJobs or options.setIncreaseParallelism + max_background_jobs=32 + # set by jni: options.setMaxSubcompactions + max_subcompactions=1 # set by jni: options.setMaxTotalWalSize max_total_wal_size=1000 + # set by jni: options.setMaxOpenFiles + max_open_files=-1 + # set by jni: options.setDeleteObsoleteFilesPeriodMicros + delete_obsolete_files_period_micros=3600000000 [CFOptions "default"] # set by jni: options.setCompressionType @@ -31,13 +43,21 @@ write_buffer_size=1024 # set by jni: options.setMaxWriteBufferNumber max_write_buffer_number=1 + # set by jni: options.setNumLevels + num_levels=7 + # set by jni: options.setLevelZeroFileNumCompactionTrigger + level0_file_num_compaction_trigger=4 + # set by jni: options.setMaxBytesForLevelBase + max_bytes_for_level_base=268435456 + # set by jni: options.setTargetFileSizeBase + target_file_size_base=67108864 # set by jni: options.setLevelCompactionDynamicLevelBytes level_compaction_dynamic_level_bytes=true [TableOptions/BlockBasedTable "default"] # set by jni: tableOptions.setBlockSize block_size=65536 - # set by jni: tableOptions.setBlockCache +# set by jni: tableOptions.setBlockCache, default value is: maxDirectMemory() / ledgerDirsSize / 10; block_cache=206150041 # set by jni: tableOptions.setFormatVersion format_version=2 diff --git a/conf/default_rocksdb.conf.default b/conf/default_rocksdb.conf.default index e9b8e7c3ecd..d8a77dd9d3f 100644 --- a/conf/default_rocksdb.conf.default +++ b/conf/default_rocksdb.conf.default @@ -16,6 +16,10 @@ # * limitations under the License. # */ +# When modifying this file, please also modify the configuration files(at +# bookkeeper-server/src/test/resources/conf) in the +# test case to ensure unit test coverage. + [DBOptions] # set by jni: options.setCreateIfMissing create_if_missing=true @@ -23,11 +27,12 @@ info_log_level=INFO_LEVEL # set by jni: options.setKeepLogFileNum keep_log_file_num=30 - -[CFOptions "default"] # set by jni: options.setLogFileTimeToRoll log_file_time_to_roll=86400 + [CFOptions "default"] + #no default setting in CFOptions + [TableOptions/BlockBasedTable "default"] # set by jni: tableOptions.setChecksumType checksum=kxxHash \ No newline at end of file diff --git a/conf/entry_location_rocksdb.conf.default b/conf/entry_location_rocksdb.conf.default index 6f6c1b4d052..4d5ce58a859 100644 --- a/conf/entry_location_rocksdb.conf.default +++ b/conf/entry_location_rocksdb.conf.default @@ -16,6 +16,10 @@ # * limitations under the License. # */ +# When modifying this file, please also modify the configuration files(at +# bookkeeper-server/src/test/resources/conf) in the +# test case to ensure unit test coverage. + [DBOptions] # set by jni: options.setCreateIfMissing create_if_missing=true @@ -26,7 +30,7 @@ # set by jni: options.setLogFileTimeToRoll log_file_time_to_roll=86400 # set by jni: options.setMaxBackgroundJobs or options.setIncreaseParallelism - max_background_jobs=2 + max_background_jobs=32 # set by jni: options.setMaxSubcompactions max_subcompactions=1 # set by jni: options.setMaxTotalWalSize @@ -57,7 +61,7 @@ [TableOptions/BlockBasedTable "default"] # set by jni: tableOptions.setBlockSize block_size=65536 - # set by jni: tableOptions.setBlockCache + # set by jni: tableOptions.setBlockCache, default value is: maxDirectMemory() / ledgerDirsSize / 10; block_cache=206150041 # set by jni: tableOptions.setFormatVersion format_version=2 diff --git a/conf/ledger_metadata_rocksdb.conf.default b/conf/ledger_metadata_rocksdb.conf.default index e9b8e7c3ecd..0bf79349220 100644 --- a/conf/ledger_metadata_rocksdb.conf.default +++ b/conf/ledger_metadata_rocksdb.conf.default @@ -16,6 +16,10 @@ # * limitations under the License. # */ +# When modifying this file, please also modify the configuration files(at +# bookkeeper-server/src/test/resources/conf) in the +# test case to ensure unit test coverage. + [DBOptions] # set by jni: options.setCreateIfMissing create_if_missing=true @@ -23,10 +27,11 @@ info_log_level=INFO_LEVEL # set by jni: options.setKeepLogFileNum keep_log_file_num=30 - -[CFOptions "default"] # set by jni: options.setLogFileTimeToRoll log_file_time_to_roll=86400 + + [CFOptions "default"] + #no default setting in CFOptions [TableOptions/BlockBasedTable "default"] # set by jni: tableOptions.setChecksumType From b9c26719a05f1860435588837b978a5fe077638c Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 16 Aug 2024 13:43:31 +0300 Subject: [PATCH 13/55] Set default format_version to 5 for RocksDB databases (#4480) (cherry picked from commit 8c7dea6503a635707e36f4392ddfc44d2ccb9b54) (cherry picked from commit 697ff5702340dddfcd63a7b76f84a02a2eea0946) --- .../bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java | 2 +- .../src/test/resources/conf/default_rocksdb.conf | 2 ++ .../src/test/resources/conf/entry_location_rocksdb.conf | 2 +- .../src/test/resources/conf/ledger_metadata_rocksdb.conf | 4 +++- .../src/test/resources/test_entry_location_rocksdb.conf | 2 +- conf/bk_server.conf | 2 +- conf/default_rocksdb.conf.default | 2 ++ conf/entry_location_rocksdb.conf.default | 2 +- conf/ledger_metadata_rocksdb.conf.default | 4 +++- 9 files changed, 15 insertions(+), 7 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java index a77a0a18f7c..0c13c1c8bff 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java @@ -198,7 +198,7 @@ private RocksDB initializeRocksDBWithBookieConf(String basePath, String subPath, int blockSize = conf.getInt(ROCKSDB_BLOCK_SIZE, 64 * 1024); int bloomFilterBitsPerKey = conf.getInt(ROCKSDB_BLOOM_FILTERS_BITS_PER_KEY, 10); boolean lz4CompressionEnabled = conf.getBoolean(ROCKSDB_LZ4_COMPRESSION_ENABLED, true); - int formatVersion = conf.getInt(ROCKSDB_FORMAT_VERSION, 2); + int formatVersion = conf.getInt(ROCKSDB_FORMAT_VERSION, 5); if (lz4CompressionEnabled) { options.setCompressionType(CompressionType.LZ4_COMPRESSION); diff --git a/bookkeeper-server/src/test/resources/conf/default_rocksdb.conf b/bookkeeper-server/src/test/resources/conf/default_rocksdb.conf index df433a33e96..b7b0ae677dd 100644 --- a/bookkeeper-server/src/test/resources/conf/default_rocksdb.conf +++ b/bookkeeper-server/src/test/resources/conf/default_rocksdb.conf @@ -30,5 +30,7 @@ #no default setting in CFOptions [TableOptions/BlockBasedTable "default"] + # set by jni: tableOptions.setFormatVersion + format_version=5 # set by jni: tableOptions.setChecksumType checksum=kxxHash \ No newline at end of file diff --git a/bookkeeper-server/src/test/resources/conf/entry_location_rocksdb.conf b/bookkeeper-server/src/test/resources/conf/entry_location_rocksdb.conf index 260111fa140..976e53b47b7 100644 --- a/bookkeeper-server/src/test/resources/conf/entry_location_rocksdb.conf +++ b/bookkeeper-server/src/test/resources/conf/entry_location_rocksdb.conf @@ -60,7 +60,7 @@ # set by jni: tableOptions.setBlockCache, default value is: maxDirectMemory() / ledgerDirsSize / 10; block_cache=206150041 # set by jni: tableOptions.setFormatVersion - format_version=2 + format_version=5 # set by jni: tableOptions.setChecksumType checksum=kxxHash # set by jni: tableOptions.setFilterPolicy, bloomfilter:[bits_per_key]:[use_block_based_builder] diff --git a/bookkeeper-server/src/test/resources/conf/ledger_metadata_rocksdb.conf b/bookkeeper-server/src/test/resources/conf/ledger_metadata_rocksdb.conf index f84c0167ed7..b7b0ae677dd 100644 --- a/bookkeeper-server/src/test/resources/conf/ledger_metadata_rocksdb.conf +++ b/bookkeeper-server/src/test/resources/conf/ledger_metadata_rocksdb.conf @@ -25,10 +25,12 @@ keep_log_file_num=30 # set by jni: options.setLogFileTimeToRoll log_file_time_to_roll=86400 - + [CFOptions "default"] #no default setting in CFOptions [TableOptions/BlockBasedTable "default"] + # set by jni: tableOptions.setFormatVersion + format_version=5 # set by jni: tableOptions.setChecksumType checksum=kxxHash \ No newline at end of file diff --git a/bookkeeper-server/src/test/resources/test_entry_location_rocksdb.conf b/bookkeeper-server/src/test/resources/test_entry_location_rocksdb.conf index f0971b8d310..7640fd2c9e8 100644 --- a/bookkeeper-server/src/test/resources/test_entry_location_rocksdb.conf +++ b/bookkeeper-server/src/test/resources/test_entry_location_rocksdb.conf @@ -60,7 +60,7 @@ # set by jni: tableOptions.setBlockCache, default value is: maxDirectMemory() / ledgerDirsSize / 10; block_cache=206150041 # set by jni: tableOptions.setFormatVersion - format_version=2 + format_version=5 # set by jni: tableOptions.setChecksumType checksum=kxxHash # set by jni: tableOptions.setFilterPolicy, bloomfilter:[bits_per_key]:[use_block_based_builder] diff --git a/conf/bk_server.conf b/conf/bk_server.conf index be3d845fadc..db7cd9cfe08 100755 --- a/conf/bk_server.conf +++ b/conf/bk_server.conf @@ -779,7 +779,7 @@ gcEntryLogMetadataCacheEnabled=false # dbStorage_rocksDB_numFilesInLevel0=4 # dbStorage_rocksDB_maxSizeInLevel1MB=256 # dbStorage_rocksDB_logPath= -# dbStorage_rocksDB_format_version=2 +# dbStorage_rocksDB_format_version=5 ############################################################################# ## DirectIO entry logger configuration diff --git a/conf/default_rocksdb.conf.default b/conf/default_rocksdb.conf.default index d8a77dd9d3f..ccedbb79ebb 100644 --- a/conf/default_rocksdb.conf.default +++ b/conf/default_rocksdb.conf.default @@ -34,5 +34,7 @@ #no default setting in CFOptions [TableOptions/BlockBasedTable "default"] + # set by jni: tableOptions.setFormatVersion + format_version=5 # set by jni: tableOptions.setChecksumType checksum=kxxHash \ No newline at end of file diff --git a/conf/entry_location_rocksdb.conf.default b/conf/entry_location_rocksdb.conf.default index 4d5ce58a859..e4dc394243f 100644 --- a/conf/entry_location_rocksdb.conf.default +++ b/conf/entry_location_rocksdb.conf.default @@ -64,7 +64,7 @@ # set by jni: tableOptions.setBlockCache, default value is: maxDirectMemory() / ledgerDirsSize / 10; block_cache=206150041 # set by jni: tableOptions.setFormatVersion - format_version=2 + format_version=5 # set by jni: tableOptions.setChecksumType checksum=kxxHash # set by jni: tableOptions.setFilterPolicy, bloomfilter:[bits_per_key]:[use_block_based_builder] diff --git a/conf/ledger_metadata_rocksdb.conf.default b/conf/ledger_metadata_rocksdb.conf.default index 0bf79349220..ccedbb79ebb 100644 --- a/conf/ledger_metadata_rocksdb.conf.default +++ b/conf/ledger_metadata_rocksdb.conf.default @@ -29,10 +29,12 @@ keep_log_file_num=30 # set by jni: options.setLogFileTimeToRoll log_file_time_to_roll=86400 - + [CFOptions "default"] #no default setting in CFOptions [TableOptions/BlockBasedTable "default"] + # set by jni: tableOptions.setFormatVersion + format_version=5 # set by jni: tableOptions.setChecksumType checksum=kxxHash \ No newline at end of file From 3642f0588339a3f42a9805fceef52e51052ad228 Mon Sep 17 00:00:00 2001 From: Dezhiliu <33149602+liudezhi2098@users.noreply.github.com> Date: Wed, 21 Aug 2024 06:57:15 +0800 Subject: [PATCH 14/55] when failed to choose a bookie, print out the list of ensemble. (#4482) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Motivation In file RackawareEnsemblePlacementPolicyImpl.java In the log.warn below, we should print out the list of ensemble, instead of the object. The “ensemble” in line-619 should be changed into “ensemble.toList()”. https://github.com/apache/bookkeeper/blob/999cd0f2ab14404be4d6c24e388456dbe56bb1a8/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java#L600C3-L619C47 ### Changes The “ensemble” in line-619 changed into “ensemble.toList()”. (cherry picked from commit 7c41204506122ed6904289f4814d4130274874aa) (cherry picked from commit 9bc90b421dbc8b32965b93e40abdb1d00ecf109b) --- .../client/RackawareEnsemblePlacementPolicyImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java index 64e7951df33..3ddc54cd901 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java @@ -536,7 +536,7 @@ public BookieNode selectFromNetworkLocation( LOG.warn("Failed to choose a bookie from network location {}, " + "the bookies in the network location are {}, excluded bookies {}, " + "current ensemble {}, fallback to choose bookie randomly from the cluster.", - networkLoc, topology.getLeaves(networkLoc), excludeBookies, ensemble); + networkLoc, topology.getLeaves(networkLoc), excludeBookies, ensemble.toList()); // randomly choose one from whole cluster, ignore the provided predicate. return selectRandom(1, excludeBookies, predicate, ensemble).get(0); } @@ -562,7 +562,7 @@ public BookieNode selectFromNetworkLocation(String networkLoc, LOG.warn("Failed to choose a bookie node from network location {}, " + "the bookies in the network location are {}, excluded bookies {}, " + "current ensemble {}, fallback to choose bookie randomly from the cluster.", - networkLoc, topology.getLeaves(networkLoc), excludeBookies, ensemble); + networkLoc, topology.getLeaves(networkLoc), excludeBookies, ensemble.toList()); return selectFromNetworkLocation(excludeRacks, excludeBookies, predicate, ensemble, fallbackToRandom); } } From 1fe52a9c323f8ead0f58edfa72b726c8531b1d8d Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 13 Sep 2024 01:39:52 +0300 Subject: [PATCH 15/55] Upgrade to Netty 4.1.113.Final and netty-tcnative 2.0.66.Final (#4502) Netty release 4.1.113.Final contains some bug fixes that are relevant for Bookkeeper. It's also worth keeping Netty updated since besides bug fixes, we will also catch possible regressions closer to releases when a change in Netty causes a regression. * [Netty 4.1.113 release notes](https://netty.io/news/2024/09/04/4-1-113-Final.html) * [Netty 4.1.112 release notes](https://netty.io/news/2024/07/19/4-1-112-Final.html) - upgrade Netty to 4.1.113.Final - upgrades netty-tcnative to 2.0.66.Final (cherry picked from commit 57f9e025d91fda3116df7d0e909cec0292d0bbf9) (cherry picked from commit 818a37a8c67b4678d0ee892f1e1b13e1e0617579) --- .../src/main/resources/LICENSE-all.bin.txt | 104 +++++++++--------- .../src/main/resources/LICENSE-bkctl.bin.txt | 92 ++++++++-------- .../src/main/resources/LICENSE-server.bin.txt | 104 +++++++++--------- .../src/main/resources/NOTICE-all.bin.txt | 46 ++++---- .../src/main/resources/NOTICE-bkctl.bin.txt | 34 +++--- .../src/main/resources/NOTICE-server.bin.txt | 46 ++++---- pom.xml | 2 +- 7 files changed, 214 insertions(+), 214 deletions(-) diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index 52ea9954fe1..1cd38248221 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -217,32 +217,32 @@ Apache Software License, Version 2. - lib/commons-io-commons-io-2.7.jar [8] - lib/commons-lang-commons-lang-2.6.jar [9] - lib/commons-logging-commons-logging-1.1.1.jar [10] -- lib/io.netty-netty-buffer-4.1.111.Final.jar [11] -- lib/io.netty-netty-codec-4.1.111.Final.jar [11] -- lib/io.netty-netty-codec-dns-4.1.111.Final.jar [11] -- lib/io.netty-netty-codec-http-4.1.111.Final.jar [11] -- lib/io.netty-netty-codec-http2-4.1.111.Final.jar [11] -- lib/io.netty-netty-codec-socks-4.1.111.Final.jar [11] -- lib/io.netty-netty-common-4.1.111.Final.jar [11] -- lib/io.netty-netty-handler-4.1.111.Final.jar [11] -- lib/io.netty-netty-handler-proxy-4.1.111.Final.jar [11] -- lib/io.netty-netty-resolver-4.1.111.Final.jar [11] -- lib/io.netty-netty-resolver-dns-4.1.111.Final.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-linux-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-osx-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-osx-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-windows-x86_64.jar [11] -- lib/io.netty-netty-tcnative-classes-2.0.65.Final.jar [11] -- lib/io.netty-netty-transport-4.1.111.Final.jar [11] -- lib/io.netty-netty-transport-classes-epoll-4.1.111.Final.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.111.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.111.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-buffer-4.1.113.Final.jar [11] +- lib/io.netty-netty-codec-4.1.113.Final.jar [11] +- lib/io.netty-netty-codec-dns-4.1.113.Final.jar [11] +- lib/io.netty-netty-codec-http-4.1.113.Final.jar [11] +- lib/io.netty-netty-codec-http2-4.1.113.Final.jar [11] +- lib/io.netty-netty-codec-socks-4.1.113.Final.jar [11] +- lib/io.netty-netty-common-4.1.113.Final.jar [11] +- lib/io.netty-netty-handler-4.1.113.Final.jar [11] +- lib/io.netty-netty-handler-proxy-4.1.113.Final.jar [11] +- lib/io.netty-netty-resolver-4.1.113.Final.jar [11] +- lib/io.netty-netty-resolver-dns-4.1.113.Final.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-windows-x86_64.jar [11] +- lib/io.netty-netty-tcnative-classes-2.0.66.Final.jar [11] +- lib/io.netty-netty-transport-4.1.113.Final.jar [11] +- lib/io.netty-netty-transport-classes-epoll-4.1.113.Final.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.113.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.113.Final-linux-x86_64.jar [11] - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-x86_64.jar [11] - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-aarch_64.jar [11] - lib/io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.25.Final.jar [11] -- lib/io.netty-netty-transport-native-unix-common-4.1.111.Final.jar [11] +- lib/io.netty-netty-transport-native-unix-common-4.1.113.Final.jar [11] - lib/io.prometheus-simpleclient-0.15.0.jar [12] - lib/io.prometheus-simpleclient_common-0.15.0.jar [12] - lib/io.prometheus-simpleclient_hotspot-0.15.0.jar [12] @@ -332,7 +332,7 @@ Apache Software License, Version 2. [8] Source available at https://github.com/apache/commons-io/tree/rel/commons-io-2.7 [9] Source available at https://github.com/apache/commons-lang/tree/LANG_2_6 [10] Source available at https://github.com/apache/commons-logging/tree/commons-logging-1.1.1 -[11] Source available at https://github.com/netty/netty/tree/netty-4.1.111.Final +[11] Source available at https://github.com/netty/netty/tree/netty-4.1.113.Final [12] Source available at https://github.com/prometheus/client_java/tree/parent-0.15.0 [13] Source available at https://github.com/vert-x3/vertx-auth/tree/4.3.2 [14] Source available at https://github.com/vert-x3/vertx-bridge-common/tree/4.3.2 @@ -372,9 +372,9 @@ Apache Software License, Version 2. [53] Source available at https://github.com/carrotsearch/hppc/tree/0.9.1 ------------------------------------------------------------------------------------ -lib/io.netty-netty-codec-4.1.111.Final.jar bundles some 3rd party dependencies +lib/io.netty-netty-codec-4.1.113.Final.jar bundles some 3rd party dependencies -lib/io.netty-netty-codec-4.1.111.Final.jar contains the extensions to Java Collections Framework which has +lib/io.netty-netty-codec-4.1.113.Final.jar contains the extensions to Java Collections Framework which has been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * LICENSE: @@ -383,7 +383,7 @@ been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified version of Robert Harder's Public Domain +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified version of Robert Harder's Public Domain Base64 Encoder and Decoder, which can be obtained at: * LICENSE: @@ -391,7 +391,7 @@ Base64 Encoder and Decoder, which can be obtained at: * HOMEPAGE: * http://iharder.sourceforge.net/current/java/base64/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'Webbit', an event based +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'Webbit', an event based WebSocket and HTTP server, which can be obtained at: * LICENSE: @@ -399,7 +399,7 @@ WebSocket and HTTP server, which can be obtained at: * HOMEPAGE: * https://github.com/joewalnes/webbit -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'SLF4J', a simple logging +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'SLF4J', a simple logging facade for Java, which can be obtained at: * LICENSE: @@ -407,7 +407,7 @@ facade for Java, which can be obtained at: * HOMEPAGE: * http://www.slf4j.org/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'Apache Harmony', an open source +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'Apache Harmony', an open source Java SE, which can be obtained at: * NOTICE: @@ -417,7 +417,7 @@ Java SE, which can be obtained at: * HOMEPAGE: * http://archive.apache.org/dist/harmony/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression and decompression library written by Matthew J. Francis. It can be obtained at: * LICENSE: @@ -425,7 +425,7 @@ and decompression library written by Matthew J. Francis. It can be obtained at: * HOMEPAGE: * https://code.google.com/p/jbzip2/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct the suffix array and the Burrows-Wheeler transformed string for any input string of a constant-size alphabet written by Yuta Mori. It can be obtained at: @@ -434,7 +434,7 @@ a constant-size alphabet written by Yuta Mori. It can be obtained at: * HOMEPAGE: * https://github.com/y-256/libdivsufsort -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, which can be obtained at: * LICENSE: @@ -442,7 +442,7 @@ Java Concurrency Tools for the JVM, which can be obtained at: * HOMEPAGE: * https://github.com/JCTools/JCTools -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in +lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in pure Java, which can be obtained at: * LICENSE: @@ -450,7 +450,7 @@ pure Java, which can be obtained at: * HOMEPAGE: * http://www.jcraft.com/jzlib/ -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and +lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * LICENSE: @@ -458,7 +458,7 @@ decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * HOMEPAGE: * https://github.com/ning/compress -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'lz4', a LZ4 Java compression +lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'lz4', a LZ4 Java compression and decompression library written by Adrien Grand. It can be obtained at: * LICENSE: @@ -466,7 +466,7 @@ and decompression library written by Adrien Grand. It can be obtained at: * HOMEPAGE: * https://github.com/jpountz/lz4-java -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'lzma-java', a LZMA Java compression +lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'lzma-java', a LZMA Java compression and decompression library, which can be obtained at: * LICENSE: @@ -474,7 +474,7 @@ and decompression library, which can be obtained at: * HOMEPAGE: * https://github.com/jponge/lzma-java -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression and decompression library written by William Kinney. It can be obtained at: * LICENSE: @@ -482,7 +482,7 @@ and decompression library written by William Kinney. It can be obtained at: * HOMEPAGE: * https://code.google.com/p/jfastlz/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data interchange format, which can be obtained at: * LICENSE: @@ -490,7 +490,7 @@ Google's data interchange format, which can be obtained at: * HOMEPAGE: * https://github.com/google/protobuf -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate +lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate a temporary self-signed X.509 certificate when the JVM does not provide the equivalent functionality. It can be obtained at: @@ -499,7 +499,7 @@ equivalent functionality. It can be obtained at: * HOMEPAGE: * http://www.bouncycastle.org/ -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Snappy', a compression library produced +lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Snappy', a compression library produced by Google Inc, which can be obtained at: * LICENSE: @@ -507,7 +507,7 @@ by Google Inc, which can be obtained at: * HOMEPAGE: * https://github.com/google/snappy -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java +lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java serialization API, which can be obtained at: * LICENSE: @@ -515,7 +515,7 @@ serialization API, which can be obtained at: * HOMEPAGE: * https://github.com/jboss-remoting/jboss-marshalling -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Caliper', Google's micro- +lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Caliper', Google's micro- benchmarking framework, which can be obtained at: * LICENSE: @@ -523,7 +523,7 @@ benchmarking framework, which can be obtained at: * HOMEPAGE: * https://github.com/google/caliper -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Apache Commons Logging', a logging +lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Apache Commons Logging', a logging framework, which can be obtained at: * LICENSE: @@ -531,7 +531,7 @@ framework, which can be obtained at: * HOMEPAGE: * http://commons.apache.org/logging/ -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Apache Log4J', a logging framework, which +lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Apache Log4J', a logging framework, which can be obtained at: * LICENSE: @@ -539,7 +539,7 @@ can be obtained at: * HOMEPAGE: * http://logging.apache.org/log4j/ -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Aalto XML', an ultra-high performance +lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Aalto XML', an ultra-high performance non-blocking XML processor, which can be obtained at: * LICENSE: @@ -547,7 +547,7 @@ non-blocking XML processor, which can be obtained at: * HOMEPAGE: * http://wiki.fasterxml.com/AaltoHome -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * LICENSE: @@ -555,7 +555,7 @@ the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * HOMEPAGE: * https://github.com/twitter/hpack -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * LICENSE: @@ -563,7 +563,7 @@ the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * HOMEPAGE: * https://github.com/python-hyper/hpack/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at: * LICENSE: @@ -571,7 +571,7 @@ the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at * HOMEPAGE: * https://github.com/nghttp2/nghttp2/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library provides utilities for the java.lang API, which can be obtained at: * LICENSE: @@ -580,7 +580,7 @@ provides utilities for the java.lang API, which can be obtained at: * https://commons.apache.org/proper/commons-lang/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', +lib/io.netty-netty-codec-4.1.113.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. * LICENSE: @@ -588,7 +588,7 @@ that provides an easy way to ensure a user has everything necessary to run the M * HOMEPAGE: * https://github.com/takari/maven-wrapper -lib/io.netty-netty-codec-4.1.111.Final.jar contains the dnsinfo.h header file, +lib/io.netty-netty-codec-4.1.113.Final.jar contains the dnsinfo.h header file, that provides a way to retrieve the system DNS configuration on MacOS. This private header is also used by Apple's open source mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/). diff --git a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt index dfd2fea4f0c..a22f92976fa 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt @@ -217,26 +217,26 @@ Apache Software License, Version 2. - lib/commons-io-commons-io-2.7.jar [8] - lib/commons-lang-commons-lang-2.6.jar [9] - lib/commons-logging-commons-logging-1.1.1.jar [10] -- lib/io.netty-netty-buffer-4.1.111.Final.jar [11] -- lib/io.netty-netty-codec-4.1.111.Final.jar [11] -- lib/io.netty-netty-common-4.1.111.Final.jar [11] -- lib/io.netty-netty-handler-4.1.111.Final.jar [11] -- lib/io.netty-netty-resolver-4.1.111.Final.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-linux-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-osx-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-osx-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-windows-x86_64.jar [11] -- lib/io.netty-netty-tcnative-classes-2.0.65.Final.jar [11] -- lib/io.netty-netty-transport-4.1.111.Final.jar [11] -- lib/io.netty-netty-transport-classes-epoll-4.1.111.Final.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.111.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.111.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-buffer-4.1.113.Final.jar [11] +- lib/io.netty-netty-codec-4.1.113.Final.jar [11] +- lib/io.netty-netty-common-4.1.113.Final.jar [11] +- lib/io.netty-netty-handler-4.1.113.Final.jar [11] +- lib/io.netty-netty-resolver-4.1.113.Final.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-windows-x86_64.jar [11] +- lib/io.netty-netty-tcnative-classes-2.0.66.Final.jar [11] +- lib/io.netty-netty-transport-4.1.113.Final.jar [11] +- lib/io.netty-netty-transport-classes-epoll-4.1.113.Final.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.113.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.113.Final-linux-x86_64.jar [11] - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-x86_64.jar [11] - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-aarch_64.jar [11] - lib/io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.25.Final.jar [11] -- lib/io.netty-netty-transport-native-unix-common-4.1.111.Final.jar [11] +- lib/io.netty-netty-transport-native-unix-common-4.1.113.Final.jar [11] - lib/org.apache.logging.log4j-log4j-api-2.18.0.jar [16] - lib/org.apache.logging.log4j-log4j-core-2.18.0.jar [16] - lib/org.apache.logging.log4j-log4j-slf4j-impl-2.18.0.jar [16] @@ -299,7 +299,7 @@ Apache Software License, Version 2. [8] Source available at https://github.com/apache/commons-io/tree/rel/commons-io-2.7 [9] Source available at https://github.com/apache/commons-lang/tree/LANG_2_6 [10] Source available at https://github.com/apache/commons-logging/tree/commons-logging-1.1.1 -[11] Source available at https://github.com/netty/netty/tree/netty-4.1.111.Final +[11] Source available at https://github.com/netty/netty/tree/netty-4.1.113.Final [16] Source available at https://github.com/apache/logging-log4j2/tree/rel/2.18.0 [18] Source available at https://github.com/apache/commons-collections/tree/collections-4.1 [19] Source available at https://github.com/apache/commons-lang/tree/LANG_3_6 @@ -330,9 +330,9 @@ Apache Software License, Version 2. [52] Source available at https://github.com/carrotsearch/hppc/tree/0.9.1 ------------------------------------------------------------------------------------ -lib/io.netty-netty-codec-4.1.111.Final.jar bundles some 3rd party dependencies +lib/io.netty-netty-codec-4.1.113.Final.jar bundles some 3rd party dependencies -lib/io.netty-netty-codec-4.1.111.Final.jar contains the extensions to Java Collections Framework which has +lib/io.netty-netty-codec-4.1.113.Final.jar contains the extensions to Java Collections Framework which has been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * LICENSE: @@ -341,7 +341,7 @@ been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified version of Robert Harder's Public Domain +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified version of Robert Harder's Public Domain Base64 Encoder and Decoder, which can be obtained at: * LICENSE: @@ -349,7 +349,7 @@ Base64 Encoder and Decoder, which can be obtained at: * HOMEPAGE: * http://iharder.sourceforge.net/current/java/base64/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'Webbit', an event based +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'Webbit', an event based WebSocket and HTTP server, which can be obtained at: * LICENSE: @@ -357,7 +357,7 @@ WebSocket and HTTP server, which can be obtained at: * HOMEPAGE: * https://github.com/joewalnes/webbit -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'SLF4J', a simple logging +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'SLF4J', a simple logging facade for Java, which can be obtained at: * LICENSE: @@ -365,7 +365,7 @@ facade for Java, which can be obtained at: * HOMEPAGE: * http://www.slf4j.org/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'Apache Harmony', an open source +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'Apache Harmony', an open source Java SE, which can be obtained at: * NOTICE: @@ -375,7 +375,7 @@ Java SE, which can be obtained at: * HOMEPAGE: * http://archive.apache.org/dist/harmony/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression and decompression library written by Matthew J. Francis. It can be obtained at: * LICENSE: @@ -383,7 +383,7 @@ and decompression library written by Matthew J. Francis. It can be obtained at: * HOMEPAGE: * https://code.google.com/p/jbzip2/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct the suffix array and the Burrows-Wheeler transformed string for any input string of a constant-size alphabet written by Yuta Mori. It can be obtained at: @@ -392,7 +392,7 @@ a constant-size alphabet written by Yuta Mori. It can be obtained at: * HOMEPAGE: * https://github.com/y-256/libdivsufsort -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, which can be obtained at: * LICENSE: @@ -400,7 +400,7 @@ Java Concurrency Tools for the JVM, which can be obtained at: * HOMEPAGE: * https://github.com/JCTools/JCTools -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in +lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in pure Java, which can be obtained at: * LICENSE: @@ -408,7 +408,7 @@ pure Java, which can be obtained at: * HOMEPAGE: * http://www.jcraft.com/jzlib/ -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and +lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * LICENSE: @@ -416,7 +416,7 @@ decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * HOMEPAGE: * https://github.com/ning/compress -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'lz4', a LZ4 Java compression +lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'lz4', a LZ4 Java compression and decompression library written by Adrien Grand. It can be obtained at: * LICENSE: @@ -424,7 +424,7 @@ and decompression library written by Adrien Grand. It can be obtained at: * HOMEPAGE: * https://github.com/jpountz/lz4-java -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'lzma-java', a LZMA Java compression +lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'lzma-java', a LZMA Java compression and decompression library, which can be obtained at: * LICENSE: @@ -432,7 +432,7 @@ and decompression library, which can be obtained at: * HOMEPAGE: * https://github.com/jponge/lzma-java -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression and decompression library written by William Kinney. It can be obtained at: * LICENSE: @@ -440,7 +440,7 @@ and decompression library written by William Kinney. It can be obtained at: * HOMEPAGE: * https://code.google.com/p/jfastlz/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data interchange format, which can be obtained at: * LICENSE: @@ -448,7 +448,7 @@ Google's data interchange format, which can be obtained at: * HOMEPAGE: * https://github.com/google/protobuf -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate +lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate a temporary self-signed X.509 certificate when the JVM does not provide the equivalent functionality. It can be obtained at: @@ -457,7 +457,7 @@ equivalent functionality. It can be obtained at: * HOMEPAGE: * http://www.bouncycastle.org/ -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Snappy', a compression library produced +lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Snappy', a compression library produced by Google Inc, which can be obtained at: * LICENSE: @@ -465,7 +465,7 @@ by Google Inc, which can be obtained at: * HOMEPAGE: * https://github.com/google/snappy -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java +lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java serialization API, which can be obtained at: * LICENSE: @@ -473,7 +473,7 @@ serialization API, which can be obtained at: * HOMEPAGE: * https://github.com/jboss-remoting/jboss-marshalling -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Caliper', Google's micro- +lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Caliper', Google's micro- benchmarking framework, which can be obtained at: * LICENSE: @@ -481,7 +481,7 @@ benchmarking framework, which can be obtained at: * HOMEPAGE: * https://github.com/google/caliper -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Apache Commons Logging', a logging +lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Apache Commons Logging', a logging framework, which can be obtained at: * LICENSE: @@ -489,7 +489,7 @@ framework, which can be obtained at: * HOMEPAGE: * http://commons.apache.org/logging/ -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Apache Log4J', a logging framework, which +lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Apache Log4J', a logging framework, which can be obtained at: * LICENSE: @@ -497,7 +497,7 @@ can be obtained at: * HOMEPAGE: * http://logging.apache.org/log4j/ -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Aalto XML', an ultra-high performance +lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Aalto XML', an ultra-high performance non-blocking XML processor, which can be obtained at: * LICENSE: @@ -505,7 +505,7 @@ non-blocking XML processor, which can be obtained at: * HOMEPAGE: * http://wiki.fasterxml.com/AaltoHome -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * LICENSE: @@ -513,7 +513,7 @@ the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * HOMEPAGE: * https://github.com/twitter/hpack -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * LICENSE: @@ -521,7 +521,7 @@ the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * HOMEPAGE: * https://github.com/python-hyper/hpack/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at: * LICENSE: @@ -529,7 +529,7 @@ the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at * HOMEPAGE: * https://github.com/nghttp2/nghttp2/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library provides utilities for the java.lang API, which can be obtained at: * LICENSE: @@ -538,7 +538,7 @@ provides utilities for the java.lang API, which can be obtained at: * https://commons.apache.org/proper/commons-lang/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', +lib/io.netty-netty-codec-4.1.113.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. * LICENSE: @@ -546,7 +546,7 @@ that provides an easy way to ensure a user has everything necessary to run the M * HOMEPAGE: * https://github.com/takari/maven-wrapper -lib/io.netty-netty-codec-4.1.111.Final.jar contains the dnsinfo.h header file, +lib/io.netty-netty-codec-4.1.113.Final.jar contains the dnsinfo.h header file, that provides a way to retrieve the system DNS configuration on MacOS. This private header is also used by Apple's open source mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/). diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt index fea213f758c..6136624d948 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt @@ -217,32 +217,32 @@ Apache Software License, Version 2. - lib/commons-io-commons-io-2.7.jar [8] - lib/commons-lang-commons-lang-2.6.jar [9] - lib/commons-logging-commons-logging-1.1.1.jar [10] -- lib/io.netty-netty-buffer-4.1.111.Final.jar [11] -- lib/io.netty-netty-codec-4.1.111.Final.jar [11] -- lib/io.netty-netty-codec-dns-4.1.111.Final.jar [11] -- lib/io.netty-netty-codec-http-4.1.111.Final.jar [11] -- lib/io.netty-netty-codec-http2-4.1.111.Final.jar [11] -- lib/io.netty-netty-codec-socks-4.1.111.Final.jar [11] -- lib/io.netty-netty-common-4.1.111.Final.jar [11] -- lib/io.netty-netty-handler-4.1.111.Final.jar [11] -- lib/io.netty-netty-handler-proxy-4.1.111.Final.jar [11] -- lib/io.netty-netty-resolver-4.1.111.Final.jar [11] -- lib/io.netty-netty-resolver-dns-4.1.111.Final.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-linux-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-osx-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-osx-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-windows-x86_64.jar [11] -- lib/io.netty-netty-tcnative-classes-2.0.65.Final.jar [11] -- lib/io.netty-netty-transport-4.1.111.Final.jar [11] -- lib/io.netty-netty-transport-classes-epoll-4.1.111.Final.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.111.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.111.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-buffer-4.1.113.Final.jar [11] +- lib/io.netty-netty-codec-4.1.113.Final.jar [11] +- lib/io.netty-netty-codec-dns-4.1.113.Final.jar [11] +- lib/io.netty-netty-codec-http-4.1.113.Final.jar [11] +- lib/io.netty-netty-codec-http2-4.1.113.Final.jar [11] +- lib/io.netty-netty-codec-socks-4.1.113.Final.jar [11] +- lib/io.netty-netty-common-4.1.113.Final.jar [11] +- lib/io.netty-netty-handler-4.1.113.Final.jar [11] +- lib/io.netty-netty-handler-proxy-4.1.113.Final.jar [11] +- lib/io.netty-netty-resolver-4.1.113.Final.jar [11] +- lib/io.netty-netty-resolver-dns-4.1.113.Final.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-windows-x86_64.jar [11] +- lib/io.netty-netty-tcnative-classes-2.0.66.Final.jar [11] +- lib/io.netty-netty-transport-4.1.113.Final.jar [11] +- lib/io.netty-netty-transport-classes-epoll-4.1.113.Final.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.113.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.113.Final-linux-x86_64.jar [11] - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-x86_64.jar [11] - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-aarch_64.jar [11] - lib/io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.25.Final.jar [11] -- lib/io.netty-netty-transport-native-unix-common-4.1.111.Final.jar [11] +- lib/io.netty-netty-transport-native-unix-common-4.1.113.Final.jar [11] - lib/io.prometheus-simpleclient-0.15.0.jar [12] - lib/io.prometheus-simpleclient_common-0.15.0.jar [12] - lib/io.prometheus-simpleclient_hotspot-0.15.0.jar [12] @@ -328,7 +328,7 @@ Apache Software License, Version 2. [8] Source available at https://github.com/apache/commons-io/tree/rel/commons-io-2.7 [9] Source available at https://github.com/apache/commons-lang/tree/LANG_2_6 [10] Source available at https://github.com/apache/commons-logging/tree/commons-logging-1.1.1 -[11] Source available at https://github.com/netty/netty/tree/netty-4.1.111.Final +[11] Source available at https://github.com/netty/netty/tree/netty-4.1.113.Final [12] Source available at https://github.com/prometheus/client_java/tree/parent-0.15.0 [13] Source available at https://github.com/vert-x3/vertx-auth/tree/4.3.2 [14] Source available at https://github.com/vert-x3/vertx-bridge-common/tree/4.3.2 @@ -367,9 +367,9 @@ Apache Software License, Version 2. [52] Source available at https://github.com/carrotsearch/hppc/tree/0.9.1 ------------------------------------------------------------------------------------ -lib/io.netty-netty-codec-4.1.111.Final.jar bundles some 3rd party dependencies +lib/io.netty-netty-codec-4.1.113.Final.jar bundles some 3rd party dependencies -lib/io.netty-netty-codec-4.1.111.Final.jar contains the extensions to Java Collections Framework which has +lib/io.netty-netty-codec-4.1.113.Final.jar contains the extensions to Java Collections Framework which has been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * LICENSE: @@ -378,7 +378,7 @@ been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified version of Robert Harder's Public Domain +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified version of Robert Harder's Public Domain Base64 Encoder and Decoder, which can be obtained at: * LICENSE: @@ -386,7 +386,7 @@ Base64 Encoder and Decoder, which can be obtained at: * HOMEPAGE: * http://iharder.sourceforge.net/current/java/base64/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'Webbit', an event based +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'Webbit', an event based WebSocket and HTTP server, which can be obtained at: * LICENSE: @@ -394,7 +394,7 @@ WebSocket and HTTP server, which can be obtained at: * HOMEPAGE: * https://github.com/joewalnes/webbit -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'SLF4J', a simple logging +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'SLF4J', a simple logging facade for Java, which can be obtained at: * LICENSE: @@ -402,7 +402,7 @@ facade for Java, which can be obtained at: * HOMEPAGE: * http://www.slf4j.org/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'Apache Harmony', an open source +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'Apache Harmony', an open source Java SE, which can be obtained at: * NOTICE: @@ -412,7 +412,7 @@ Java SE, which can be obtained at: * HOMEPAGE: * http://archive.apache.org/dist/harmony/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression and decompression library written by Matthew J. Francis. It can be obtained at: * LICENSE: @@ -420,7 +420,7 @@ and decompression library written by Matthew J. Francis. It can be obtained at: * HOMEPAGE: * https://code.google.com/p/jbzip2/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct the suffix array and the Burrows-Wheeler transformed string for any input string of a constant-size alphabet written by Yuta Mori. It can be obtained at: @@ -429,7 +429,7 @@ a constant-size alphabet written by Yuta Mori. It can be obtained at: * HOMEPAGE: * https://github.com/y-256/libdivsufsort -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, which can be obtained at: * LICENSE: @@ -437,7 +437,7 @@ Java Concurrency Tools for the JVM, which can be obtained at: * HOMEPAGE: * https://github.com/JCTools/JCTools -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in +lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in pure Java, which can be obtained at: * LICENSE: @@ -445,7 +445,7 @@ pure Java, which can be obtained at: * HOMEPAGE: * http://www.jcraft.com/jzlib/ -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and +lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * LICENSE: @@ -453,7 +453,7 @@ decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * HOMEPAGE: * https://github.com/ning/compress -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'lz4', a LZ4 Java compression +lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'lz4', a LZ4 Java compression and decompression library written by Adrien Grand. It can be obtained at: * LICENSE: @@ -461,7 +461,7 @@ and decompression library written by Adrien Grand. It can be obtained at: * HOMEPAGE: * https://github.com/jpountz/lz4-java -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'lzma-java', a LZMA Java compression +lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'lzma-java', a LZMA Java compression and decompression library, which can be obtained at: * LICENSE: @@ -469,7 +469,7 @@ and decompression library, which can be obtained at: * HOMEPAGE: * https://github.com/jponge/lzma-java -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression and decompression library written by William Kinney. It can be obtained at: * LICENSE: @@ -477,7 +477,7 @@ and decompression library written by William Kinney. It can be obtained at: * HOMEPAGE: * https://code.google.com/p/jfastlz/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data interchange format, which can be obtained at: * LICENSE: @@ -485,7 +485,7 @@ Google's data interchange format, which can be obtained at: * HOMEPAGE: * https://github.com/google/protobuf -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate +lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate a temporary self-signed X.509 certificate when the JVM does not provide the equivalent functionality. It can be obtained at: @@ -494,7 +494,7 @@ equivalent functionality. It can be obtained at: * HOMEPAGE: * http://www.bouncycastle.org/ -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Snappy', a compression library produced +lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Snappy', a compression library produced by Google Inc, which can be obtained at: * LICENSE: @@ -502,7 +502,7 @@ by Google Inc, which can be obtained at: * HOMEPAGE: * https://github.com/google/snappy -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java +lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java serialization API, which can be obtained at: * LICENSE: @@ -510,7 +510,7 @@ serialization API, which can be obtained at: * HOMEPAGE: * https://github.com/jboss-remoting/jboss-marshalling -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Caliper', Google's micro- +lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Caliper', Google's micro- benchmarking framework, which can be obtained at: * LICENSE: @@ -518,7 +518,7 @@ benchmarking framework, which can be obtained at: * HOMEPAGE: * https://github.com/google/caliper -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Apache Commons Logging', a logging +lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Apache Commons Logging', a logging framework, which can be obtained at: * LICENSE: @@ -526,7 +526,7 @@ framework, which can be obtained at: * HOMEPAGE: * http://commons.apache.org/logging/ -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Apache Log4J', a logging framework, which +lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Apache Log4J', a logging framework, which can be obtained at: * LICENSE: @@ -534,7 +534,7 @@ can be obtained at: * HOMEPAGE: * http://logging.apache.org/log4j/ -lib/io.netty-netty-codec-4.1.111.Final.jar optionally depends on 'Aalto XML', an ultra-high performance +lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Aalto XML', an ultra-high performance non-blocking XML processor, which can be obtained at: * LICENSE: @@ -542,7 +542,7 @@ non-blocking XML processor, which can be obtained at: * HOMEPAGE: * http://wiki.fasterxml.com/AaltoHome -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * LICENSE: @@ -550,7 +550,7 @@ the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * HOMEPAGE: * https://github.com/twitter/hpack -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * LICENSE: @@ -558,7 +558,7 @@ the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * HOMEPAGE: * https://github.com/python-hyper/hpack/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at: * LICENSE: @@ -566,7 +566,7 @@ the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at * HOMEPAGE: * https://github.com/nghttp2/nghttp2/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library +lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library provides utilities for the java.lang API, which can be obtained at: * LICENSE: @@ -575,7 +575,7 @@ provides utilities for the java.lang API, which can be obtained at: * https://commons.apache.org/proper/commons-lang/ -lib/io.netty-netty-codec-4.1.111.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', +lib/io.netty-netty-codec-4.1.113.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. * LICENSE: @@ -583,7 +583,7 @@ that provides an easy way to ensure a user has everything necessary to run the M * HOMEPAGE: * https://github.com/takari/maven-wrapper -lib/io.netty-netty-codec-4.1.111.Final.jar contains the dnsinfo.h header file, +lib/io.netty-netty-codec-4.1.113.Final.jar contains the dnsinfo.h header file, that provides a way to retrieve the system DNS configuration on MacOS. This private header is also used by Apple's open source mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/). diff --git a/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt index af368d1d948..15a546ab8f9 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt @@ -23,31 +23,31 @@ LongAdder), which was released with the following comments: http://creativecommons.org/publicdomain/zero/1.0/ ------------------------------------------------------------------------------------ -- lib/io.netty-netty-buffer-4.1.111.Final.jar -- lib/io.netty-netty-codec-4.1.111.Final.jar -- lib/io.netty-netty-codec-dns-4.1.111.Final.jar -- lib/io.netty-netty-codec-http-4.1.111.Final.jar -- lib/io.netty-netty-codec-http2-4.1.111.Final.jar -- lib/io.netty-netty-codec-socks-4.1.111.Final.jar -- lib/io.netty-netty-common-4.1.111.Final.jar -- lib/io.netty-netty-handler-4.1.111.Final.jar -- lib/io.netty-netty-handler-proxy-4.1.111.Final.jar -- lib/io.netty-netty-resolver-4.1.111.Final.jar -- lib/io.netty-netty-resolver-dns-4.1.111.Final.jar -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final.jar -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-linux-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-osx-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-osx-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-windows-x86_64.jar [11] -- lib/io.netty-netty-tcnative-classes-2.0.65.Final.jar -- lib/io.netty-netty-transport-4.1.111.Final.jar -- lib/io.netty-netty-transport-classes-epoll-4.1.111.Final.jar -- lib/io.netty-netty-transport-native-epoll-4.1.111.Final-linux-aarch_64.jar -- lib/io.netty-netty-transport-native-epoll-4.1.111.Final-linux-x86_64.jar +- lib/io.netty-netty-buffer-4.1.113.Final.jar +- lib/io.netty-netty-codec-4.1.113.Final.jar +- lib/io.netty-netty-codec-dns-4.1.113.Final.jar +- lib/io.netty-netty-codec-http-4.1.113.Final.jar +- lib/io.netty-netty-codec-http2-4.1.113.Final.jar +- lib/io.netty-netty-codec-socks-4.1.113.Final.jar +- lib/io.netty-netty-common-4.1.113.Final.jar +- lib/io.netty-netty-handler-4.1.113.Final.jar +- lib/io.netty-netty-handler-proxy-4.1.113.Final.jar +- lib/io.netty-netty-resolver-4.1.113.Final.jar +- lib/io.netty-netty-resolver-dns-4.1.113.Final.jar +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final.jar +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-windows-x86_64.jar [11] +- lib/io.netty-netty-tcnative-classes-2.0.66.Final.jar +- lib/io.netty-netty-transport-4.1.113.Final.jar +- lib/io.netty-netty-transport-classes-epoll-4.1.113.Final.jar +- lib/io.netty-netty-transport-native-epoll-4.1.113.Final-linux-aarch_64.jar +- lib/io.netty-netty-transport-native-epoll-4.1.113.Final-linux-x86_64.jar - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-x86_64.jar - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-aarch_64.jar -- lib/io.netty-netty-transport-native-unix-common-4.1.111.Final.jar +- lib/io.netty-netty-transport-native-unix-common-4.1.113.Final.jar The Netty Project diff --git a/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt index 88eaaf8a5d0..c5e9b51ab44 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt +++ b/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt @@ -5,25 +5,25 @@ This product includes software developed at The Apache Software Foundation (http://www.apache.org/). ------------------------------------------------------------------------------------ -- lib/io.netty-netty-buffer-4.1.111.Final.jar -- lib/io.netty-netty-codec-4.1.111.Final.jar -- lib/io.netty-netty-common-4.1.111.Final.jar -- lib/io.netty-netty-handler-4.1.111.Final.jar -- lib/io.netty-netty-resolver-4.1.111.Final.jar -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final.jar -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-linux-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-osx-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-osx-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-windows-x86_64.jar [11] -- lib/io.netty-netty-tcnative-classes-2.0.65.Final.jar -- lib/io.netty-netty-transport-4.1.111.Final.jar -- lib/io.netty-netty-transport-classes-epoll-4.1.111.Final.jar -- lib/io.netty-netty-transport-native-epoll-4.1.111.Final-linux-aarch_64.jar -- lib/io.netty-netty-transport-native-epoll-4.1.111.Final-linux-x86_64.jar +- lib/io.netty-netty-buffer-4.1.113.Final.jar +- lib/io.netty-netty-codec-4.1.113.Final.jar +- lib/io.netty-netty-common-4.1.113.Final.jar +- lib/io.netty-netty-handler-4.1.113.Final.jar +- lib/io.netty-netty-resolver-4.1.113.Final.jar +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final.jar +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-windows-x86_64.jar [11] +- lib/io.netty-netty-tcnative-classes-2.0.66.Final.jar +- lib/io.netty-netty-transport-4.1.113.Final.jar +- lib/io.netty-netty-transport-classes-epoll-4.1.113.Final.jar +- lib/io.netty-netty-transport-native-epoll-4.1.113.Final-linux-aarch_64.jar +- lib/io.netty-netty-transport-native-epoll-4.1.113.Final-linux-x86_64.jar - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-x86_64.jar - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-aarch_64.jar -- lib/io.netty-netty-transport-native-unix-common-4.1.111.Final.jar +- lib/io.netty-netty-transport-native-unix-common-4.1.113.Final.jar The Netty Project diff --git a/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt index 2873f19dedb..208afe4c377 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt @@ -5,31 +5,31 @@ This product includes software developed at The Apache Software Foundation (http://www.apache.org/). ------------------------------------------------------------------------------------ -- lib/io.netty-netty-buffer-4.1.111.Final.jar -- lib/io.netty-netty-codec-4.1.111.Final.jar -- lib/io.netty-netty-codec-dns-4.1.111.Final.jar -- lib/io.netty-netty-codec-http-4.1.111.Final.jar -- lib/io.netty-netty-codec-http2-4.1.111.Final.jar -- lib/io.netty-netty-codec-socks-4.1.111.Final.jar -- lib/io.netty-netty-common-4.1.111.Final.jar -- lib/io.netty-netty-handler-4.1.111.Final.jar -- lib/io.netty-netty-handler-proxy-4.1.111.Final.jar -- lib/io.netty-netty-resolver-4.1.111.Final.jar -- lib/io.netty-netty-resolver-dns-4.1.111.Final.jar -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final.jar -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-linux-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-osx-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-osx-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.65.Final-windows-x86_64.jar [11] -- lib/io.netty-netty-tcnative-classes-2.0.65.Final.jar -- lib/io.netty-netty-transport-4.1.111.Final.jar -- lib/io.netty-netty-transport-classes-epoll-4.1.111.Final.jar -- lib/io.netty-netty-transport-native-epoll-4.1.111.Final-linux-aarch_64.jar -- lib/io.netty-netty-transport-native-epoll-4.1.111.Final-linux-x86_64.jar +- lib/io.netty-netty-buffer-4.1.113.Final.jar +- lib/io.netty-netty-codec-4.1.113.Final.jar +- lib/io.netty-netty-codec-dns-4.1.113.Final.jar +- lib/io.netty-netty-codec-http-4.1.113.Final.jar +- lib/io.netty-netty-codec-http2-4.1.113.Final.jar +- lib/io.netty-netty-codec-socks-4.1.113.Final.jar +- lib/io.netty-netty-common-4.1.113.Final.jar +- lib/io.netty-netty-handler-4.1.113.Final.jar +- lib/io.netty-netty-handler-proxy-4.1.113.Final.jar +- lib/io.netty-netty-resolver-4.1.113.Final.jar +- lib/io.netty-netty-resolver-dns-4.1.113.Final.jar +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final.jar +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-windows-x86_64.jar [11] +- lib/io.netty-netty-tcnative-classes-2.0.66.Final.jar +- lib/io.netty-netty-transport-4.1.113.Final.jar +- lib/io.netty-netty-transport-classes-epoll-4.1.113.Final.jar +- lib/io.netty-netty-transport-native-epoll-4.1.113.Final-linux-aarch_64.jar +- lib/io.netty-netty-transport-native-epoll-4.1.113.Final-linux-x86_64.jar - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-x86_64.jar - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-aarch_64.jar -- lib/io.netty-netty-transport-native-unix-common-4.1.111.Final.jar +- lib/io.netty-netty-transport-native-unix-common-4.1.113.Final.jar The Netty Project diff --git a/pom.xml b/pom.xml index aebe9f0776d..2e8d3c048a9 100644 --- a/pom.xml +++ b/pom.xml @@ -154,7 +154,7 @@ 2.18.0 1.3.0 3.12.4 - 4.1.111.Final + 4.1.113.Final 0.0.25.Final 9.1.3 2.0.9 From b19ddb33a5fa8217b649e960c1e7eff82d8a3dee Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Fri, 13 Sep 2024 09:01:55 +0800 Subject: [PATCH 16/55] [fix] Fix data lost after when writing ledger and deleting legder execute concurrency (#4462) ### Motivation | step | `BK client 1` | `BK client 2` | | --- | --- | --- | | 1 | create ledger `1` | | 2 | | open ledger `1` | | 3 | | delete ledger `1` | | 4 | write data to ledger `1` | At the step `4`, the write should fail, but it succeeds. It leads users to assume the data has been written, but it can not be read. You can reproduce the issue by `testWriteAfterDeleted` There is a scenario that will lead to Pulsar loss messages - `broker-2` created a ledger - `broker-2`'s ZK session is expired, which will lead the topic it owned to be assigned to other brokers - `broker-0` owned the topic again - it will delete the last empty ledger - consumers connected to `broker-0` - producers connected to `broker-2` - send messages to the topic - on `broker-2`, the ledger can not be closed due to the ledger metadata has been deleted ### Changes Once the ledger is fenced, it can not be wrote anymore. (cherry picked from commit 47ef48e074df42705bc0bf14bac5b3cc05c5f0c3) (cherry picked from commit 93118aa55bc07d9c11298dda25330f8554d9b3ea) --- .../apache/bookkeeper/bookie/BookieImpl.java | 14 ++-- .../bookkeeper/bookie/HandleFactory.java | 2 +- .../bookkeeper/bookie/HandleFactoryImpl.java | 31 ++++++++- .../bookkeeper/bookie/BookieJournalTest.java | 4 +- .../apache/bookkeeper/client/TestFencing.java | 65 +++++++++++++++++++ 5 files changed, 105 insertions(+), 11 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java index d0874ffa165..968977ecdc3 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java @@ -527,7 +527,7 @@ public void process(int journalVersion, long offset, ByteBuffer recBuff) throws masterKeyCache.put(ledgerId, masterKey); // Force to re-insert the master key in ledger storage - handles.getHandle(ledgerId, masterKey); + handles.getHandle(ledgerId, masterKey, true); } else { throw new IOException("Invalid journal. Contains journalKey " + " but layout version (" + journalVersion @@ -539,7 +539,7 @@ public void process(int journalVersion, long offset, ByteBuffer recBuff) throws if (key == null) { key = ledgerStorage.readMasterKey(ledgerId); } - LedgerDescriptor handle = handles.getHandle(ledgerId, key); + LedgerDescriptor handle = handles.getHandle(ledgerId, key, true); handle.setFenced(); } else { throw new IOException("Invalid journal. Contains fenceKey " @@ -557,7 +557,7 @@ public void process(int journalVersion, long offset, ByteBuffer recBuff) throws if (key == null) { key = ledgerStorage.readMasterKey(ledgerId); } - LedgerDescriptor handle = handles.getHandle(ledgerId, key); + LedgerDescriptor handle = handles.getHandle(ledgerId, key, true); handle.setExplicitLac(explicitLacBuf); } else { throw new IOException("Invalid journal. Contains explicitLAC " + " but layout version (" @@ -580,7 +580,7 @@ public void process(int journalVersion, long offset, ByteBuffer recBuff) throws if (key == null) { key = ledgerStorage.readMasterKey(ledgerId); } - LedgerDescriptor handle = handles.getHandle(ledgerId, key); + LedgerDescriptor handle = handles.getHandle(ledgerId, key, true); recBuff.rewind(); handle.addEntry(Unpooled.wrappedBuffer(recBuff)); @@ -902,7 +902,7 @@ LedgerDescriptor getLedgerForEntry(ByteBuf entry, final byte[] masterKey) throws IOException, BookieException { final long ledgerId = entry.getLong(entry.readerIndex()); - return handles.getHandle(ledgerId, masterKey); + return handles.getHandle(ledgerId, masterKey, false); } private Journal getJournal(long ledgerId) { @@ -1011,7 +1011,7 @@ public void setExplicitLac(ByteBuf entry, WriteCallback writeCallback, Object ct ByteBuf explicitLACEntry = null; try { long ledgerId = entry.getLong(entry.readerIndex()); - LedgerDescriptor handle = handles.getHandle(ledgerId, masterKey); + LedgerDescriptor handle = handles.getHandle(ledgerId, masterKey, false); synchronized (handle) { entry.markReaderIndex(); handle.setExplicitLac(entry); @@ -1100,7 +1100,7 @@ public void addEntry(ByteBuf entry, boolean ackBeforeSync, WriteCallback cb, Obj */ public CompletableFuture fenceLedger(long ledgerId, byte[] masterKey) throws IOException, BookieException { - LedgerDescriptor handle = handles.getHandle(ledgerId, masterKey); + LedgerDescriptor handle = handles.getHandle(ledgerId, masterKey, false); return handle.fenceAndLogInJournal(getJournal(ledgerId)); } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/HandleFactory.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/HandleFactory.java index 22500b74cb6..c81294d4dbe 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/HandleFactory.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/HandleFactory.java @@ -24,7 +24,7 @@ import java.io.IOException; interface HandleFactory { - LedgerDescriptor getHandle(long ledgerId, byte[] masterKey) + LedgerDescriptor getHandle(long ledgerId, byte[] masterKey, boolean journalReplay) throws IOException, BookieException; LedgerDescriptor getReadOnlyHandle(long ledgerId) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/HandleFactoryImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/HandleFactoryImpl.java index 3f643019c9a..b331f125061 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/HandleFactoryImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/HandleFactoryImpl.java @@ -21,7 +21,10 @@ package org.apache.bookkeeper.bookie; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import java.io.IOException; +import java.time.Duration; import org.apache.bookkeeper.bookie.LedgerStorage.LedgerDeletionListener; import org.apache.bookkeeper.util.collections.ConcurrentLongHashMap; @@ -29,6 +32,14 @@ class HandleFactoryImpl implements HandleFactory, LedgerDeletionListener { private final ConcurrentLongHashMap ledgers; private final ConcurrentLongHashMap readOnlyLedgers; + /** + * Once the ledger was marked "fenced" before, the ledger was accessed by multi clients. One client is calling + * "delete" now, and other clients may call "write" continuously later. We mark these ledgers can not be written + * anymore. And maintains the state for 7 days is safety. + */ + private final Cache recentlyFencedAndDeletedLedgers = CacheBuilder.newBuilder() + .expireAfterAccess(Duration.ofDays(7)).build(); + final LedgerStorage ledgerStorage; HandleFactoryImpl(LedgerStorage ledgerStorage) { @@ -40,10 +51,14 @@ class HandleFactoryImpl implements HandleFactory, LedgerDeletionListener { } @Override - public LedgerDescriptor getHandle(final long ledgerId, final byte[] masterKey) throws IOException, BookieException { + public LedgerDescriptor getHandle(final long ledgerId, final byte[] masterKey, boolean journalReplay) + throws IOException, BookieException { LedgerDescriptor handle = ledgers.get(ledgerId); if (handle == null) { + if (!journalReplay && recentlyFencedAndDeletedLedgers.getIfPresent(ledgerId) != null) { + throw BookieException.create(BookieException.Code.LedgerFencedException); + } handle = LedgerDescriptor.create(masterKey, ledgerId, ledgerStorage); ledgers.putIfAbsent(ledgerId, handle); } @@ -64,8 +79,22 @@ public LedgerDescriptor getReadOnlyHandle(final long ledgerId) throws IOExceptio return handle; } + private void markIfConflictWritingOccurs(long ledgerId) { + LedgerDescriptor ledgerDescriptor = ledgers.get(ledgerId); + try { + if (ledgerDescriptor != null && ledgerDescriptor.isFenced()) { + recentlyFencedAndDeletedLedgers.put(ledgerId, true); + } + } catch (IOException | BookieException ex) { + // The ledger is in limbo state. + recentlyFencedAndDeletedLedgers.put(ledgerId, true); + } + } + @Override public void ledgerDeleted(long ledgerId) { + markIfConflictWritingOccurs(ledgerId); + // Do delete. ledgers.remove(ledgerId); readOnlyLedgers.remove(ledgerId); } diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieJournalTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieJournalTest.java index a048bffc8a0..f489f3ffb93 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieJournalTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieJournalTest.java @@ -453,7 +453,7 @@ public void testV4Journal() throws Exception { } catch (Bookie.NoEntryException e) { // correct behaviour } - assertTrue(b.handles.getHandle(1, "testPasswd".getBytes()).isFenced()); + assertTrue(b.handles.getHandle(1, "testPasswd".getBytes(), false).isFenced()); b.shutdown(); } @@ -485,7 +485,7 @@ public void testV5Journal() throws Exception { } catch (Bookie.NoEntryException e) { // correct behavior } - assertTrue(b.handles.getHandle(1, "testV5Journal".getBytes()).isFenced()); + assertTrue(b.handles.getHandle(1, "testV5Journal".getBytes(), false).isFenced()); b.shutdown(); } diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestFencing.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestFencing.java index 34751525aa6..77382b4ebde 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestFencing.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestFencing.java @@ -26,10 +26,18 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.bookie.Bookie; +import org.apache.bookkeeper.bookie.InterleavedLedgerStorage; +import org.apache.bookkeeper.bookie.LedgerStorage; +import org.apache.bookkeeper.bookie.SortedLedgerStorage; +import org.apache.bookkeeper.bookie.storage.ldb.SingleDirectoryDbLedgerStorage; import org.apache.bookkeeper.client.BookKeeper.DigestType; import org.apache.bookkeeper.conf.ClientConfiguration; import org.apache.bookkeeper.net.BookieId; import org.apache.bookkeeper.test.BookKeeperClusterTestCase; +import org.apache.bookkeeper.test.TestStatsProvider; +import org.awaitility.reflect.WhiteboxImpl; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,6 +46,7 @@ * This unit test tests ledger fencing. * */ +@Slf4j public class TestFencing extends BookKeeperClusterTestCase { private static final Logger LOG = LoggerFactory.getLogger(TestFencing.class); @@ -77,6 +86,7 @@ public void testBasicFencing() throws Exception { fail("Should have thrown an exception when trying to write"); } catch (BKException.BKLedgerFencedException e) { // correct behaviour + log.info("expected a fenced error", e); } /* @@ -87,6 +97,61 @@ public void testBasicFencing() throws Exception { readlh.getLastAddConfirmed() == writelh.getLastAddConfirmed()); } + @Test + public void testWriteAfterDeleted() throws Exception { + LedgerHandle writeLedger; + writeLedger = bkc.createLedger(digestType, "password".getBytes()); + + String tmp = "BookKeeper is cool!"; + for (int i = 0; i < 10; i++) { + long entryId = writeLedger.addEntry(tmp.getBytes()); + LOG.info("entryId: {}", entryId); + } + + // Fence and delete. + BookKeeperTestClient bkc2 = new BookKeeperTestClient(baseClientConf, new TestStatsProvider()); + LedgerHandle readLedger = bkc2.openLedger(writeLedger.getId(), digestType, "password".getBytes()); + bkc2.deleteLedger(readLedger.ledgerId); + + // Waiting for GC. + for (ServerTester server : servers) { + triggerGC(server.getServer().getBookie()); + } + + try { + long entryId = writeLedger.addEntry(tmp.getBytes()); + LOG.info("Not expected: entryId: {}", entryId); + LOG.error("Should have thrown an exception"); + fail("Should have thrown an exception when trying to write"); + } catch (BKException.BKLedgerFencedException e) { + log.info("expected a fenced error", e); + // correct behaviour + } + + /* + * Check it has been recovered properly. + */ + assertTrue("Has not recovered correctly: " + readLedger.getLastAddConfirmed() + + " original " + writeLedger.getLastAddConfirmed(), + readLedger.getLastAddConfirmed() == writeLedger.getLastAddConfirmed()); + + // cleanup. + bkc2.close(); + } + + private void triggerGC(Bookie bookie) { + LedgerStorage ledgerStorage = bookie.getLedgerStorage(); + if (ledgerStorage instanceof InterleavedLedgerStorage + || ledgerStorage instanceof SingleDirectoryDbLedgerStorage) { + Runnable gcThread = WhiteboxImpl.getInternalState(ledgerStorage, "gcThread"); + gcThread.run(); + } else if (ledgerStorage instanceof SortedLedgerStorage) { + Object actLedgerStorage = WhiteboxImpl.getInternalState(ledgerStorage, "interleavedLedgerStorage"); + Runnable gcThread = WhiteboxImpl.getInternalState(actLedgerStorage, "gcThread"); + gcThread.run(); + } + } + private static int threadCount = 0; class LedgerOpenThread extends Thread { From 2ebc925585fad9109aaf9047c97e01a702229347 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 25 Sep 2024 04:32:45 +0300 Subject: [PATCH 17/55] Upgrade protobuf to 3.25.5 to address CVE-2024-7254 (#4508) CVE-2024-7254 Upgrade protobuf to 3.25.5 (cherry picked from commit 0229b5d7cfd93850f05a16f20172f0d39492672f) (cherry picked from commit b9c567cbe4d005e19d23ad1db308c8902a8db69b) --- bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt | 8 ++++---- bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt | 8 ++++---- bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt | 8 ++++---- pom.xml | 4 ++-- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index 1cd38248221..b52d8cde66f 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -640,13 +640,13 @@ This product bundles Google Protocol Buffers, which is available under a "3-clau license. Bundled as - - lib/com.google.protobuf-protobuf-java-3.21.9.jar -Source available at https://github.com/google/protobuf/tree/v3.21.9 + - lib/com.google.protobuf-protobuf-java-3.25.5.jar +Source available at https://github.com/google/protobuf/tree/v3.25.5 For details, see deps/protobuf-3.14.0/LICENSE. Bundled as - - lib/com.google.protobuf-protobuf-java-util-3.21.9.jar -Source available at https://github.com/protocolbuffers/protobuf/tree/v3.21.9 + - lib/com.google.protobuf-protobuf-java-util-3.25.5.jar +Source available at https://github.com/protocolbuffers/protobuf/tree/v3.25.5 For details, see deps/protobuf-3.12.0/LICENSE. ------------------------------------------------------------------------------------ This product bundles the JCP Standard Java Servlet API, which is available under a diff --git a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt index a22f92976fa..b0db094e5b3 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt @@ -561,13 +561,13 @@ This product bundles Google Protocol Buffers, which is available under a "3-clau license. Bundled as - - lib/com.google.protobuf-protobuf-java-3.21.9.jar -Source available at https://github.com/google/protobuf/tree/v3.21.9 + - lib/com.google.protobuf-protobuf-java-3.25.5.jar +Source available at https://github.com/google/protobuf/tree/v3.25.5 For details, see deps/protobuf-3.14.0/LICENSE. Bundled as - - lib/com.google.protobuf-protobuf-java-util-3.21.9.jar -Source available at https://github.com/protocolbuffers/protobuf/tree/v3.21.9 + - lib/com.google.protobuf-protobuf-java-util-3.25.5.jar +Source available at https://github.com/protocolbuffers/protobuf/tree/v3.25.5 For details, see deps/protobuf-3.12.0/LICENSE. ------------------------------------------------------------------------------------ This product bundles Simple Logging Facade for Java, which is available under a diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt index 6136624d948..06391016441 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt @@ -629,13 +629,13 @@ This product bundles Google Protocol Buffers, which is available under a "3-clau license. Bundled as - - lib/com.google.protobuf-protobuf-java-3.21.9.jar -Source available at https://github.com/google/protobuf/tree/v3.21.9 + - lib/com.google.protobuf-protobuf-java-3.25.5.jar +Source available at https://github.com/google/protobuf/tree/v3.25.5 For details, see deps/protobuf-3.14.0/LICENSE. Bundled as - - lib/com.google.protobuf-protobuf-java-util-3.21.9.jar -Source available at https://github.com/protocolbuffers/protobuf/tree/v3.21.9 + - lib/com.google.protobuf-protobuf-java-util-3.25.5.jar +Source available at https://github.com/protocolbuffers/protobuf/tree/v3.25.5 For details, see deps/protobuf-3.12.0/LICENSE. ------------------------------------------------------------------------------------ This product bundles the JCP Standard Java Servlet API, which is available under a diff --git a/pom.xml b/pom.xml index 2e8d3c048a9..7170c615ecc 100644 --- a/pom.xml +++ b/pom.xml @@ -162,8 +162,8 @@ 0.8.3 4.5.13 4.4.15 - 3.21.9 - 3.21.9 + 3.25.5 + ${protobuf.version} ${grpc.version} 0.9.11 7.9.2 From b551eb2a2283d8c897f1b1e39e878d65b9a247bd Mon Sep 17 00:00:00 2001 From: Shogo Takayama <37703914+ange-k@users.noreply.github.com> Date: Wed, 13 Nov 2024 17:12:29 +0900 Subject: [PATCH 18/55] Optimize reorderReadSequence to check WriteSet instead of entire Ensemble (#4478) (cherry picked from commit 0376bdce03257def64b95b787e466dac6a9de588) (cherry picked from commit 33a1985c95d79802103f5ef1d84e3c9b81f56dfd) --- .../RackawareEnsemblePlacementPolicyImpl.java | 5 ++- .../TestRackawareEnsemblePlacementPolicy.java | 39 +++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java index 3ddc54cd901..86a9d507fe5 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java @@ -866,8 +866,9 @@ DistributionSchedule.WriteSet reorderReadSequenceWithRegion( if (useRegionAware || reorderReadsRandom) { isAnyBookieUnavailable = true; } else { - for (int i = 0; i < ensemble.size(); i++) { - BookieId bookieAddr = ensemble.get(i); + for (int i = 0; i < writeSet.size(); i++) { + int idx = writeSet.get(i); + BookieId bookieAddr = ensemble.get(idx); if ((!knownBookies.containsKey(bookieAddr) && !readOnlyBookies.contains(bookieAddr)) || slowBookies.getIfPresent(bookieAddr) != null) { // Found at least one bookie not available in the ensemble, or in slowBookies diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java index 857458296e8..0bda0f33306 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicy.java @@ -64,6 +64,7 @@ import org.apache.bookkeeper.stats.Gauge; import org.apache.bookkeeper.stats.NullStatsLogger; import org.apache.bookkeeper.test.TestStatsProvider; +import org.apache.bookkeeper.test.TestStatsProvider.TestOpStatsLogger; import org.apache.bookkeeper.test.TestStatsProvider.TestStatsLogger; import org.apache.bookkeeper.util.StaticDNSResolver; import org.apache.commons.collections4.CollectionUtils; @@ -2275,6 +2276,44 @@ public void testNodeWithFailures() throws Exception { StaticDNSResolver.reset(); } + @Test + public void testSlowBookieInEnsembleOnly() throws Exception { + repp.uninitalize(); + updateMyRack("/r1/rack1"); + + TestStatsProvider statsProvider = new TestStatsProvider(); + TestStatsLogger statsLogger = statsProvider.getStatsLogger(""); + + repp = new RackawareEnsemblePlacementPolicy(); + repp.initialize(conf, Optional.empty(), timer, + DISABLE_ALL, statsLogger, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); + repp.withDefaultRack(NetworkTopology.DEFAULT_REGION_AND_RACK); + + TestOpStatsLogger readRequestsReorderedCounter = (TestOpStatsLogger) statsLogger + .getOpStatsLogger(BookKeeperClientStats.READ_REQUESTS_REORDERED); + + // Update cluster + Set addrs = new HashSet(); + addrs.add(addr1.toBookieId()); + addrs.add(addr2.toBookieId()); + addrs.add(addr3.toBookieId()); + addrs.add(addr4.toBookieId()); + repp.onClusterChanged(addrs, new HashSet()); + repp.registerSlowBookie(addr1.toBookieId(), 0L); + Map bookiePendingMap = new HashMap<>(); + bookiePendingMap.put(addr1.toBookieId(), 1L); + repp.onClusterChanged(addrs, new HashSet<>()); + + DistributionSchedule.WriteSet writeSet = writeSetFromValues(1, 2, 3); + + DistributionSchedule.WriteSet reorderSet = repp.reorderReadSequence( + ensemble, getBookiesHealthInfo(new HashMap<>(), bookiePendingMap), writeSet); + + // If the slow bookie is only present in the ensemble, no reordering occurs. + assertEquals(writeSet, reorderSet); + assertEquals(0, readRequestsReorderedCounter.getSuccessCount()); + } + @Test public void testReplaceNotAvailableBookieWithDefaultRack() throws Exception { repp.uninitalize(); From 2cefaff6912d23d69c2c41789a4ba2740da90703 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 13 Nov 2024 15:29:25 +0000 Subject: [PATCH 19/55] Upgrade Zookeeper to 3.9.3 to address CVE-2024-51504 (#4523) * Upgrade Zookeeper to 3.9.3 to address CVE-2024-51504 * Upgrade curator to 5.7.1 (cherry picked from commit af8baa18ad9f5d079400f65d8f686a78bf93c393) (cherry picked from commit 7cba2a665abc6b7568b4628aafb2d3f928620dfc) --- .../src/main/resources/LICENSE-all.bin.txt | 14 +++++++------- .../src/main/resources/LICENSE-bkctl.bin.txt | 14 +++++++------- .../src/main/resources/LICENSE-server.bin.txt | 14 +++++++------- pom.xml | 4 ++-- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index b52d8cde66f..5f0a35d2bbf 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -261,9 +261,9 @@ Apache Software License, Version 2. - lib/org.apache.logging.log4j-log4j-slf4j-impl-2.18.0.jar [17] - lib/org.apache.commons-commons-collections4-4.1.jar [19] - lib/org.apache.commons-commons-lang3-3.6.jar [20] -- lib/org.apache.zookeeper-zookeeper-3.8.3.jar [21] -- lib/org.apache.zookeeper-zookeeper-jute-3.8.3.jar [21] -- lib/org.apache.zookeeper-zookeeper-3.8.3-tests.jar [21] +- lib/org.apache.zookeeper-zookeeper-3.9.3.jar [21] +- lib/org.apache.zookeeper-zookeeper-jute-3.9.3.jar [21] +- lib/org.apache.zookeeper-zookeeper-3.9.3-tests.jar [21] - lib/org.eclipse.jetty-jetty-http-9.4.53.v20231009.jar [22] - lib/org.eclipse.jetty-jetty-io-9.4.53.v20231009.jar [22] - lib/org.eclipse.jetty-jetty-security-9.4.53.v20231009.jar [22] @@ -296,9 +296,9 @@ Apache Software License, Version 2. - lib/io.grpc-grpc-testing-1.54.1.jar [33] - lib/io.grpc-grpc-xds-1.54.1.jar [33] - lib/io.grpc-grpc-rls-1.54.1.jar[33] -- lib/org.apache.curator-curator-client-5.1.0.jar [34] -- lib/org.apache.curator-curator-framework-5.1.0.jar [34] -- lib/org.apache.curator-curator-recipes-5.1.0.jar [34] +- lib/org.apache.curator-curator-client-5.7.1.jar [34] +- lib/org.apache.curator-curator-framework-5.7.1.jar [34] +- lib/org.apache.curator-curator-recipes-5.7.1.jar [34] - lib/com.google.errorprone-error_prone_annotations-2.9.0.jar [36] - lib/org.apache.yetus-audience-annotations-0.12.0.jar [37] - lib/org.jctools-jctools-core-2.1.2.jar [38] @@ -351,7 +351,7 @@ Apache Software License, Version 2. [29] Source available at https://github.com/google/gson/tree/gson-parent-2.10.1 [30] Source available at https://github.com/census-instrumentation/opencensus-java/tree/v0.28.0 [33] Source available at https://github.com/grpc/grpc-java/tree/v1.56.0 -[34] Source available at https://github.com/apache/curator/releases/tag/apache.curator-5.1.0 +[34] Source available at https://github.com/apache/curator/releases/tag/apache.curator-5.7.1 [36] Source available at https://github.com/google/error-prone/tree/v2.9.0 [37] Source available at https://github.com/apache/yetus/tree/rel/0.12.0 [38] Source available at https://github.com/JCTools/JCTools/tree/v2.1.2 diff --git a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt index b0db094e5b3..cab6b3fc9a2 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt @@ -242,9 +242,9 @@ Apache Software License, Version 2. - lib/org.apache.logging.log4j-log4j-slf4j-impl-2.18.0.jar [16] - lib/org.apache.commons-commons-collections4-4.1.jar [18] - lib/org.apache.commons-commons-lang3-3.6.jar [19] -- lib/org.apache.zookeeper-zookeeper-3.8.3.jar [20] -- lib/org.apache.zookeeper-zookeeper-jute-3.8.3.jar [20] -- lib/org.apache.zookeeper-zookeeper-3.8.3-tests.jar [20] +- lib/org.apache.zookeeper-zookeeper-3.9.3.jar [20] +- lib/org.apache.zookeeper-zookeeper-jute-3.9.3.jar [20] +- lib/org.apache.zookeeper-zookeeper-3.9.3-tests.jar [20] - lib/com.beust-jcommander-1.82.jar [23] - lib/net.jpountz.lz4-lz4-1.3.0.jar [25] - lib/com.google.api.grpc-proto-google-common-protos-2.9.0.jar [27] @@ -267,9 +267,9 @@ Apache Software License, Version 2. - lib/io.grpc-grpc-testing-1.54.1.jar [32] - lib/io.grpc-grpc-xds-1.54.1.jar [32] - lib/io.grpc-grpc-rls-1.54.1.jar[32] -- lib/org.apache.curator-curator-client-5.1.0.jar [33] -- lib/org.apache.curator-curator-framework-5.1.0.jar [33] -- lib/org.apache.curator-curator-recipes-5.1.0.jar [33] +- lib/org.apache.curator-curator-client-5.7.1.jar [33] +- lib/org.apache.curator-curator-framework-5.7.1.jar [33] +- lib/org.apache.curator-curator-recipes-5.7.1.jar [33] - lib/com.google.errorprone-error_prone_annotations-2.9.0.jar [35] - lib/org.apache.yetus-audience-annotations-0.12.0.jar [36] - lib/org.jctools-jctools-core-2.1.2.jar [37] @@ -310,7 +310,7 @@ Apache Software License, Version 2. [28] Source available at https://github.com/google/gson/tree/gson-parent-2.10.1 [29] Source available at https://github.com/census-instrumentation/opencensus-java/tree/v0.28.0 [32] Source available at https://github.com/grpc/grpc-java/tree/v1.56.0 -[33] Source available at https://github.com/apache/curator/tree/apache-curator-5.1.0 +[33] Source available at https://github.com/apache/curator/tree/apache-curator-5.7.1 [35] Source available at https://github.com/google/error-prone/tree/v2.9.0 [36] Source available at https://github.com/apache/yetus/tree/rel/0.12.0 [37] Source available at https://github.com/JCTools/JCTools/tree/v2.1.2 diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt index 06391016441..498ba5014de 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt @@ -261,9 +261,9 @@ Apache Software License, Version 2. - lib/org.apache.logging.log4j-log4j-slf4j-impl-2.18.0.jar [17] - lib/org.apache.commons-commons-collections4-4.1.jar [19] - lib/org.apache.commons-commons-lang3-3.6.jar [20] -- lib/org.apache.zookeeper-zookeeper-3.8.3.jar [21] -- lib/org.apache.zookeeper-zookeeper-jute-3.8.3.jar [21] -- lib/org.apache.zookeeper-zookeeper-3.8.3-tests.jar [21] +- lib/org.apache.zookeeper-zookeeper-3.9.3.jar [21] +- lib/org.apache.zookeeper-zookeeper-jute-3.9.3.jar [21] +- lib/org.apache.zookeeper-zookeeper-3.9.3-tests.jar [21] - lib/org.eclipse.jetty-jetty-http-9.4.53.v20231009.jar [22] - lib/org.eclipse.jetty-jetty-io-9.4.53.v20231009.jar [22] - lib/org.eclipse.jetty-jetty-security-9.4.53.v20231009.jar [22] @@ -296,9 +296,9 @@ Apache Software License, Version 2. - lib/io.grpc-grpc-testing-1.54.1.jar [33] - lib/io.grpc-grpc-xds-1.54.1.jar [33] - lib/io.grpc-grpc-rls-1.54.1.jar[33] -- lib/org.apache.curator-curator-client-5.1.0.jar [34] -- lib/org.apache.curator-curator-framework-5.1.0.jar [34] -- lib/org.apache.curator-curator-recipes-5.1.0.jar [34] +- lib/org.apache.curator-curator-client-5.7.1.jar [34] +- lib/org.apache.curator-curator-framework-5.7.1.jar [34] +- lib/org.apache.curator-curator-recipes-5.7.1.jar [34] - lib/com.google.errorprone-error_prone_annotations-2.9.0.jar [36] - lib/org.apache.yetus-audience-annotations-0.12.0.jar [37] - lib/org.jctools-jctools-core-2.1.2.jar [38] @@ -347,7 +347,7 @@ Apache Software License, Version 2. [29] Source available at https://github.com/google/gson/tree/gson-parent-2.10.1 [30] Source available at https://github.com/census-instrumentation/opencensus-java/tree/v0.28.0 [33] Source available at https://github.com/grpc/grpc-java/tree/v1.56.0 -[34] Source available at https://github.com/apache/curator/releases/tag/apache.curator-5.1.0 +[34] Source available at https://github.com/apache/curator/releases/tag/apache.curator-5.7.1 [36] Source available at https://github.com/google/error-prone/tree/v2.9.0 [37] Source available at https://github.com/apache/yetus/tree/rel/0.12.0 [38] Source available at https://github.com/JCTools/JCTools/tree/v2.1.2 diff --git a/pom.xml b/pom.xml index 7170c615ecc..af34d3cfa00 100644 --- a/pom.xml +++ b/pom.xml @@ -126,7 +126,7 @@ 3.6 2.7 1.0.2.4 - 5.1.0 + 5.7.1 4.1.12.1 0.7.7 3.2.2 @@ -174,7 +174,7 @@ 1.3.2 1.19.4 4.5.7 - 3.8.3 + 3.9.3 1.1.10.5 2.1.2 0.9.1 From 12a1f14ae2f84649e43bbf8d8bc3f11d65bdb4b4 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 19 Nov 2024 02:12:08 +0200 Subject: [PATCH 20/55] Upgrade to Netty 4.1.115.Final to address CVE-2024-47535 (#4524) (cherry picked from commit d873ca4e0c775b0e61c3c63fd98bdfcb09d33d6d) (cherry picked from commit 48bcef70d17a9688048fd08d344931465af96e25) --- .../src/main/resources/LICENSE-all.bin.txt | 104 +++++++++--------- .../src/main/resources/LICENSE-bkctl.bin.txt | 92 ++++++++-------- .../src/main/resources/LICENSE-server.bin.txt | 104 +++++++++--------- .../src/main/resources/NOTICE-all.bin.txt | 46 ++++---- .../src/main/resources/NOTICE-bkctl.bin.txt | 34 +++--- .../src/main/resources/NOTICE-server.bin.txt | 46 ++++---- pom.xml | 2 +- 7 files changed, 214 insertions(+), 214 deletions(-) diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index 5f0a35d2bbf..9faf244e122 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -217,32 +217,32 @@ Apache Software License, Version 2. - lib/commons-io-commons-io-2.7.jar [8] - lib/commons-lang-commons-lang-2.6.jar [9] - lib/commons-logging-commons-logging-1.1.1.jar [10] -- lib/io.netty-netty-buffer-4.1.113.Final.jar [11] -- lib/io.netty-netty-codec-4.1.113.Final.jar [11] -- lib/io.netty-netty-codec-dns-4.1.113.Final.jar [11] -- lib/io.netty-netty-codec-http-4.1.113.Final.jar [11] -- lib/io.netty-netty-codec-http2-4.1.113.Final.jar [11] -- lib/io.netty-netty-codec-socks-4.1.113.Final.jar [11] -- lib/io.netty-netty-common-4.1.113.Final.jar [11] -- lib/io.netty-netty-handler-4.1.113.Final.jar [11] -- lib/io.netty-netty-handler-proxy-4.1.113.Final.jar [11] -- lib/io.netty-netty-resolver-4.1.113.Final.jar [11] -- lib/io.netty-netty-resolver-dns-4.1.113.Final.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-windows-x86_64.jar [11] -- lib/io.netty-netty-tcnative-classes-2.0.66.Final.jar [11] -- lib/io.netty-netty-transport-4.1.113.Final.jar [11] -- lib/io.netty-netty-transport-classes-epoll-4.1.113.Final.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.113.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.113.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-buffer-4.1.115.Final.jar [11] +- lib/io.netty-netty-codec-4.1.115.Final.jar [11] +- lib/io.netty-netty-codec-dns-4.1.115.Final.jar [11] +- lib/io.netty-netty-codec-http-4.1.115.Final.jar [11] +- lib/io.netty-netty-codec-http2-4.1.115.Final.jar [11] +- lib/io.netty-netty-codec-socks-4.1.115.Final.jar [11] +- lib/io.netty-netty-common-4.1.115.Final.jar [11] +- lib/io.netty-netty-handler-4.1.115.Final.jar [11] +- lib/io.netty-netty-handler-proxy-4.1.115.Final.jar [11] +- lib/io.netty-netty-resolver-4.1.115.Final.jar [11] +- lib/io.netty-netty-resolver-dns-4.1.115.Final.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-osx-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-osx-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-windows-x86_64.jar [11] +- lib/io.netty-netty-tcnative-classes-2.0.69.Final.jar [11] +- lib/io.netty-netty-transport-4.1.115.Final.jar [11] +- lib/io.netty-netty-transport-classes-epoll-4.1.115.Final.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.115.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.115.Final-linux-x86_64.jar [11] - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-x86_64.jar [11] - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-aarch_64.jar [11] - lib/io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.25.Final.jar [11] -- lib/io.netty-netty-transport-native-unix-common-4.1.113.Final.jar [11] +- lib/io.netty-netty-transport-native-unix-common-4.1.115.Final.jar [11] - lib/io.prometheus-simpleclient-0.15.0.jar [12] - lib/io.prometheus-simpleclient_common-0.15.0.jar [12] - lib/io.prometheus-simpleclient_hotspot-0.15.0.jar [12] @@ -332,7 +332,7 @@ Apache Software License, Version 2. [8] Source available at https://github.com/apache/commons-io/tree/rel/commons-io-2.7 [9] Source available at https://github.com/apache/commons-lang/tree/LANG_2_6 [10] Source available at https://github.com/apache/commons-logging/tree/commons-logging-1.1.1 -[11] Source available at https://github.com/netty/netty/tree/netty-4.1.113.Final +[11] Source available at https://github.com/netty/netty/tree/netty-4.1.115.Final [12] Source available at https://github.com/prometheus/client_java/tree/parent-0.15.0 [13] Source available at https://github.com/vert-x3/vertx-auth/tree/4.3.2 [14] Source available at https://github.com/vert-x3/vertx-bridge-common/tree/4.3.2 @@ -372,9 +372,9 @@ Apache Software License, Version 2. [53] Source available at https://github.com/carrotsearch/hppc/tree/0.9.1 ------------------------------------------------------------------------------------ -lib/io.netty-netty-codec-4.1.113.Final.jar bundles some 3rd party dependencies +lib/io.netty-netty-codec-4.1.115.Final.jar bundles some 3rd party dependencies -lib/io.netty-netty-codec-4.1.113.Final.jar contains the extensions to Java Collections Framework which has +lib/io.netty-netty-codec-4.1.115.Final.jar contains the extensions to Java Collections Framework which has been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * LICENSE: @@ -383,7 +383,7 @@ been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified version of Robert Harder's Public Domain +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified version of Robert Harder's Public Domain Base64 Encoder and Decoder, which can be obtained at: * LICENSE: @@ -391,7 +391,7 @@ Base64 Encoder and Decoder, which can be obtained at: * HOMEPAGE: * http://iharder.sourceforge.net/current/java/base64/ -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'Webbit', an event based +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'Webbit', an event based WebSocket and HTTP server, which can be obtained at: * LICENSE: @@ -399,7 +399,7 @@ WebSocket and HTTP server, which can be obtained at: * HOMEPAGE: * https://github.com/joewalnes/webbit -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'SLF4J', a simple logging +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'SLF4J', a simple logging facade for Java, which can be obtained at: * LICENSE: @@ -407,7 +407,7 @@ facade for Java, which can be obtained at: * HOMEPAGE: * http://www.slf4j.org/ -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'Apache Harmony', an open source +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'Apache Harmony', an open source Java SE, which can be obtained at: * NOTICE: @@ -417,7 +417,7 @@ Java SE, which can be obtained at: * HOMEPAGE: * http://archive.apache.org/dist/harmony/ -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression and decompression library written by Matthew J. Francis. It can be obtained at: * LICENSE: @@ -425,7 +425,7 @@ and decompression library written by Matthew J. Francis. It can be obtained at: * HOMEPAGE: * https://code.google.com/p/jbzip2/ -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct the suffix array and the Burrows-Wheeler transformed string for any input string of a constant-size alphabet written by Yuta Mori. It can be obtained at: @@ -434,7 +434,7 @@ a constant-size alphabet written by Yuta Mori. It can be obtained at: * HOMEPAGE: * https://github.com/y-256/libdivsufsort -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, which can be obtained at: * LICENSE: @@ -442,7 +442,7 @@ Java Concurrency Tools for the JVM, which can be obtained at: * HOMEPAGE: * https://github.com/JCTools/JCTools -lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in +lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in pure Java, which can be obtained at: * LICENSE: @@ -450,7 +450,7 @@ pure Java, which can be obtained at: * HOMEPAGE: * http://www.jcraft.com/jzlib/ -lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and +lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * LICENSE: @@ -458,7 +458,7 @@ decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * HOMEPAGE: * https://github.com/ning/compress -lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'lz4', a LZ4 Java compression +lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'lz4', a LZ4 Java compression and decompression library written by Adrien Grand. It can be obtained at: * LICENSE: @@ -466,7 +466,7 @@ and decompression library written by Adrien Grand. It can be obtained at: * HOMEPAGE: * https://github.com/jpountz/lz4-java -lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'lzma-java', a LZMA Java compression +lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'lzma-java', a LZMA Java compression and decompression library, which can be obtained at: * LICENSE: @@ -474,7 +474,7 @@ and decompression library, which can be obtained at: * HOMEPAGE: * https://github.com/jponge/lzma-java -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression and decompression library written by William Kinney. It can be obtained at: * LICENSE: @@ -482,7 +482,7 @@ and decompression library written by William Kinney. It can be obtained at: * HOMEPAGE: * https://code.google.com/p/jfastlz/ -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data interchange format, which can be obtained at: * LICENSE: @@ -490,7 +490,7 @@ Google's data interchange format, which can be obtained at: * HOMEPAGE: * https://github.com/google/protobuf -lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate +lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate a temporary self-signed X.509 certificate when the JVM does not provide the equivalent functionality. It can be obtained at: @@ -499,7 +499,7 @@ equivalent functionality. It can be obtained at: * HOMEPAGE: * http://www.bouncycastle.org/ -lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Snappy', a compression library produced +lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Snappy', a compression library produced by Google Inc, which can be obtained at: * LICENSE: @@ -507,7 +507,7 @@ by Google Inc, which can be obtained at: * HOMEPAGE: * https://github.com/google/snappy -lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java +lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java serialization API, which can be obtained at: * LICENSE: @@ -515,7 +515,7 @@ serialization API, which can be obtained at: * HOMEPAGE: * https://github.com/jboss-remoting/jboss-marshalling -lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Caliper', Google's micro- +lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Caliper', Google's micro- benchmarking framework, which can be obtained at: * LICENSE: @@ -523,7 +523,7 @@ benchmarking framework, which can be obtained at: * HOMEPAGE: * https://github.com/google/caliper -lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Apache Commons Logging', a logging +lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Apache Commons Logging', a logging framework, which can be obtained at: * LICENSE: @@ -531,7 +531,7 @@ framework, which can be obtained at: * HOMEPAGE: * http://commons.apache.org/logging/ -lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Apache Log4J', a logging framework, which +lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Apache Log4J', a logging framework, which can be obtained at: * LICENSE: @@ -539,7 +539,7 @@ can be obtained at: * HOMEPAGE: * http://logging.apache.org/log4j/ -lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Aalto XML', an ultra-high performance +lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Aalto XML', an ultra-high performance non-blocking XML processor, which can be obtained at: * LICENSE: @@ -547,7 +547,7 @@ non-blocking XML processor, which can be obtained at: * HOMEPAGE: * http://wiki.fasterxml.com/AaltoHome -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * LICENSE: @@ -555,7 +555,7 @@ the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * HOMEPAGE: * https://github.com/twitter/hpack -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * LICENSE: @@ -563,7 +563,7 @@ the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * HOMEPAGE: * https://github.com/python-hyper/hpack/ -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at: * LICENSE: @@ -571,7 +571,7 @@ the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at * HOMEPAGE: * https://github.com/nghttp2/nghttp2/ -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library provides utilities for the java.lang API, which can be obtained at: * LICENSE: @@ -580,7 +580,7 @@ provides utilities for the java.lang API, which can be obtained at: * https://commons.apache.org/proper/commons-lang/ -lib/io.netty-netty-codec-4.1.113.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', +lib/io.netty-netty-codec-4.1.115.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. * LICENSE: @@ -588,7 +588,7 @@ that provides an easy way to ensure a user has everything necessary to run the M * HOMEPAGE: * https://github.com/takari/maven-wrapper -lib/io.netty-netty-codec-4.1.113.Final.jar contains the dnsinfo.h header file, +lib/io.netty-netty-codec-4.1.115.Final.jar contains the dnsinfo.h header file, that provides a way to retrieve the system DNS configuration on MacOS. This private header is also used by Apple's open source mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/). diff --git a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt index cab6b3fc9a2..dd20fe2fe22 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt @@ -217,26 +217,26 @@ Apache Software License, Version 2. - lib/commons-io-commons-io-2.7.jar [8] - lib/commons-lang-commons-lang-2.6.jar [9] - lib/commons-logging-commons-logging-1.1.1.jar [10] -- lib/io.netty-netty-buffer-4.1.113.Final.jar [11] -- lib/io.netty-netty-codec-4.1.113.Final.jar [11] -- lib/io.netty-netty-common-4.1.113.Final.jar [11] -- lib/io.netty-netty-handler-4.1.113.Final.jar [11] -- lib/io.netty-netty-resolver-4.1.113.Final.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-windows-x86_64.jar [11] -- lib/io.netty-netty-tcnative-classes-2.0.66.Final.jar [11] -- lib/io.netty-netty-transport-4.1.113.Final.jar [11] -- lib/io.netty-netty-transport-classes-epoll-4.1.113.Final.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.113.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.113.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-buffer-4.1.115.Final.jar [11] +- lib/io.netty-netty-codec-4.1.115.Final.jar [11] +- lib/io.netty-netty-common-4.1.115.Final.jar [11] +- lib/io.netty-netty-handler-4.1.115.Final.jar [11] +- lib/io.netty-netty-resolver-4.1.115.Final.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-osx-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-osx-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-windows-x86_64.jar [11] +- lib/io.netty-netty-tcnative-classes-2.0.69.Final.jar [11] +- lib/io.netty-netty-transport-4.1.115.Final.jar [11] +- lib/io.netty-netty-transport-classes-epoll-4.1.115.Final.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.115.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.115.Final-linux-x86_64.jar [11] - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-x86_64.jar [11] - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-aarch_64.jar [11] - lib/io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.25.Final.jar [11] -- lib/io.netty-netty-transport-native-unix-common-4.1.113.Final.jar [11] +- lib/io.netty-netty-transport-native-unix-common-4.1.115.Final.jar [11] - lib/org.apache.logging.log4j-log4j-api-2.18.0.jar [16] - lib/org.apache.logging.log4j-log4j-core-2.18.0.jar [16] - lib/org.apache.logging.log4j-log4j-slf4j-impl-2.18.0.jar [16] @@ -299,7 +299,7 @@ Apache Software License, Version 2. [8] Source available at https://github.com/apache/commons-io/tree/rel/commons-io-2.7 [9] Source available at https://github.com/apache/commons-lang/tree/LANG_2_6 [10] Source available at https://github.com/apache/commons-logging/tree/commons-logging-1.1.1 -[11] Source available at https://github.com/netty/netty/tree/netty-4.1.113.Final +[11] Source available at https://github.com/netty/netty/tree/netty-4.1.115.Final [16] Source available at https://github.com/apache/logging-log4j2/tree/rel/2.18.0 [18] Source available at https://github.com/apache/commons-collections/tree/collections-4.1 [19] Source available at https://github.com/apache/commons-lang/tree/LANG_3_6 @@ -330,9 +330,9 @@ Apache Software License, Version 2. [52] Source available at https://github.com/carrotsearch/hppc/tree/0.9.1 ------------------------------------------------------------------------------------ -lib/io.netty-netty-codec-4.1.113.Final.jar bundles some 3rd party dependencies +lib/io.netty-netty-codec-4.1.115.Final.jar bundles some 3rd party dependencies -lib/io.netty-netty-codec-4.1.113.Final.jar contains the extensions to Java Collections Framework which has +lib/io.netty-netty-codec-4.1.115.Final.jar contains the extensions to Java Collections Framework which has been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * LICENSE: @@ -341,7 +341,7 @@ been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified version of Robert Harder's Public Domain +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified version of Robert Harder's Public Domain Base64 Encoder and Decoder, which can be obtained at: * LICENSE: @@ -349,7 +349,7 @@ Base64 Encoder and Decoder, which can be obtained at: * HOMEPAGE: * http://iharder.sourceforge.net/current/java/base64/ -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'Webbit', an event based +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'Webbit', an event based WebSocket and HTTP server, which can be obtained at: * LICENSE: @@ -357,7 +357,7 @@ WebSocket and HTTP server, which can be obtained at: * HOMEPAGE: * https://github.com/joewalnes/webbit -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'SLF4J', a simple logging +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'SLF4J', a simple logging facade for Java, which can be obtained at: * LICENSE: @@ -365,7 +365,7 @@ facade for Java, which can be obtained at: * HOMEPAGE: * http://www.slf4j.org/ -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'Apache Harmony', an open source +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'Apache Harmony', an open source Java SE, which can be obtained at: * NOTICE: @@ -375,7 +375,7 @@ Java SE, which can be obtained at: * HOMEPAGE: * http://archive.apache.org/dist/harmony/ -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression and decompression library written by Matthew J. Francis. It can be obtained at: * LICENSE: @@ -383,7 +383,7 @@ and decompression library written by Matthew J. Francis. It can be obtained at: * HOMEPAGE: * https://code.google.com/p/jbzip2/ -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct the suffix array and the Burrows-Wheeler transformed string for any input string of a constant-size alphabet written by Yuta Mori. It can be obtained at: @@ -392,7 +392,7 @@ a constant-size alphabet written by Yuta Mori. It can be obtained at: * HOMEPAGE: * https://github.com/y-256/libdivsufsort -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, which can be obtained at: * LICENSE: @@ -400,7 +400,7 @@ Java Concurrency Tools for the JVM, which can be obtained at: * HOMEPAGE: * https://github.com/JCTools/JCTools -lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in +lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in pure Java, which can be obtained at: * LICENSE: @@ -408,7 +408,7 @@ pure Java, which can be obtained at: * HOMEPAGE: * http://www.jcraft.com/jzlib/ -lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and +lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * LICENSE: @@ -416,7 +416,7 @@ decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * HOMEPAGE: * https://github.com/ning/compress -lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'lz4', a LZ4 Java compression +lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'lz4', a LZ4 Java compression and decompression library written by Adrien Grand. It can be obtained at: * LICENSE: @@ -424,7 +424,7 @@ and decompression library written by Adrien Grand. It can be obtained at: * HOMEPAGE: * https://github.com/jpountz/lz4-java -lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'lzma-java', a LZMA Java compression +lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'lzma-java', a LZMA Java compression and decompression library, which can be obtained at: * LICENSE: @@ -432,7 +432,7 @@ and decompression library, which can be obtained at: * HOMEPAGE: * https://github.com/jponge/lzma-java -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression and decompression library written by William Kinney. It can be obtained at: * LICENSE: @@ -440,7 +440,7 @@ and decompression library written by William Kinney. It can be obtained at: * HOMEPAGE: * https://code.google.com/p/jfastlz/ -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data interchange format, which can be obtained at: * LICENSE: @@ -448,7 +448,7 @@ Google's data interchange format, which can be obtained at: * HOMEPAGE: * https://github.com/google/protobuf -lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate +lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate a temporary self-signed X.509 certificate when the JVM does not provide the equivalent functionality. It can be obtained at: @@ -457,7 +457,7 @@ equivalent functionality. It can be obtained at: * HOMEPAGE: * http://www.bouncycastle.org/ -lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Snappy', a compression library produced +lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Snappy', a compression library produced by Google Inc, which can be obtained at: * LICENSE: @@ -465,7 +465,7 @@ by Google Inc, which can be obtained at: * HOMEPAGE: * https://github.com/google/snappy -lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java +lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java serialization API, which can be obtained at: * LICENSE: @@ -473,7 +473,7 @@ serialization API, which can be obtained at: * HOMEPAGE: * https://github.com/jboss-remoting/jboss-marshalling -lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Caliper', Google's micro- +lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Caliper', Google's micro- benchmarking framework, which can be obtained at: * LICENSE: @@ -481,7 +481,7 @@ benchmarking framework, which can be obtained at: * HOMEPAGE: * https://github.com/google/caliper -lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Apache Commons Logging', a logging +lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Apache Commons Logging', a logging framework, which can be obtained at: * LICENSE: @@ -489,7 +489,7 @@ framework, which can be obtained at: * HOMEPAGE: * http://commons.apache.org/logging/ -lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Apache Log4J', a logging framework, which +lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Apache Log4J', a logging framework, which can be obtained at: * LICENSE: @@ -497,7 +497,7 @@ can be obtained at: * HOMEPAGE: * http://logging.apache.org/log4j/ -lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Aalto XML', an ultra-high performance +lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Aalto XML', an ultra-high performance non-blocking XML processor, which can be obtained at: * LICENSE: @@ -505,7 +505,7 @@ non-blocking XML processor, which can be obtained at: * HOMEPAGE: * http://wiki.fasterxml.com/AaltoHome -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * LICENSE: @@ -513,7 +513,7 @@ the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * HOMEPAGE: * https://github.com/twitter/hpack -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * LICENSE: @@ -521,7 +521,7 @@ the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * HOMEPAGE: * https://github.com/python-hyper/hpack/ -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at: * LICENSE: @@ -529,7 +529,7 @@ the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at * HOMEPAGE: * https://github.com/nghttp2/nghttp2/ -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library provides utilities for the java.lang API, which can be obtained at: * LICENSE: @@ -538,7 +538,7 @@ provides utilities for the java.lang API, which can be obtained at: * https://commons.apache.org/proper/commons-lang/ -lib/io.netty-netty-codec-4.1.113.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', +lib/io.netty-netty-codec-4.1.115.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. * LICENSE: @@ -546,7 +546,7 @@ that provides an easy way to ensure a user has everything necessary to run the M * HOMEPAGE: * https://github.com/takari/maven-wrapper -lib/io.netty-netty-codec-4.1.113.Final.jar contains the dnsinfo.h header file, +lib/io.netty-netty-codec-4.1.115.Final.jar contains the dnsinfo.h header file, that provides a way to retrieve the system DNS configuration on MacOS. This private header is also used by Apple's open source mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/). diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt index 498ba5014de..e6430f129a9 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt @@ -217,32 +217,32 @@ Apache Software License, Version 2. - lib/commons-io-commons-io-2.7.jar [8] - lib/commons-lang-commons-lang-2.6.jar [9] - lib/commons-logging-commons-logging-1.1.1.jar [10] -- lib/io.netty-netty-buffer-4.1.113.Final.jar [11] -- lib/io.netty-netty-codec-4.1.113.Final.jar [11] -- lib/io.netty-netty-codec-dns-4.1.113.Final.jar [11] -- lib/io.netty-netty-codec-http-4.1.113.Final.jar [11] -- lib/io.netty-netty-codec-http2-4.1.113.Final.jar [11] -- lib/io.netty-netty-codec-socks-4.1.113.Final.jar [11] -- lib/io.netty-netty-common-4.1.113.Final.jar [11] -- lib/io.netty-netty-handler-4.1.113.Final.jar [11] -- lib/io.netty-netty-handler-proxy-4.1.113.Final.jar [11] -- lib/io.netty-netty-resolver-4.1.113.Final.jar [11] -- lib/io.netty-netty-resolver-dns-4.1.113.Final.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-windows-x86_64.jar [11] -- lib/io.netty-netty-tcnative-classes-2.0.66.Final.jar [11] -- lib/io.netty-netty-transport-4.1.113.Final.jar [11] -- lib/io.netty-netty-transport-classes-epoll-4.1.113.Final.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.113.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.113.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-buffer-4.1.115.Final.jar [11] +- lib/io.netty-netty-codec-4.1.115.Final.jar [11] +- lib/io.netty-netty-codec-dns-4.1.115.Final.jar [11] +- lib/io.netty-netty-codec-http-4.1.115.Final.jar [11] +- lib/io.netty-netty-codec-http2-4.1.115.Final.jar [11] +- lib/io.netty-netty-codec-socks-4.1.115.Final.jar [11] +- lib/io.netty-netty-common-4.1.115.Final.jar [11] +- lib/io.netty-netty-handler-4.1.115.Final.jar [11] +- lib/io.netty-netty-handler-proxy-4.1.115.Final.jar [11] +- lib/io.netty-netty-resolver-4.1.115.Final.jar [11] +- lib/io.netty-netty-resolver-dns-4.1.115.Final.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-osx-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-osx-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-windows-x86_64.jar [11] +- lib/io.netty-netty-tcnative-classes-2.0.69.Final.jar [11] +- lib/io.netty-netty-transport-4.1.115.Final.jar [11] +- lib/io.netty-netty-transport-classes-epoll-4.1.115.Final.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.115.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.115.Final-linux-x86_64.jar [11] - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-x86_64.jar [11] - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-aarch_64.jar [11] - lib/io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.25.Final.jar [11] -- lib/io.netty-netty-transport-native-unix-common-4.1.113.Final.jar [11] +- lib/io.netty-netty-transport-native-unix-common-4.1.115.Final.jar [11] - lib/io.prometheus-simpleclient-0.15.0.jar [12] - lib/io.prometheus-simpleclient_common-0.15.0.jar [12] - lib/io.prometheus-simpleclient_hotspot-0.15.0.jar [12] @@ -328,7 +328,7 @@ Apache Software License, Version 2. [8] Source available at https://github.com/apache/commons-io/tree/rel/commons-io-2.7 [9] Source available at https://github.com/apache/commons-lang/tree/LANG_2_6 [10] Source available at https://github.com/apache/commons-logging/tree/commons-logging-1.1.1 -[11] Source available at https://github.com/netty/netty/tree/netty-4.1.113.Final +[11] Source available at https://github.com/netty/netty/tree/netty-4.1.115.Final [12] Source available at https://github.com/prometheus/client_java/tree/parent-0.15.0 [13] Source available at https://github.com/vert-x3/vertx-auth/tree/4.3.2 [14] Source available at https://github.com/vert-x3/vertx-bridge-common/tree/4.3.2 @@ -367,9 +367,9 @@ Apache Software License, Version 2. [52] Source available at https://github.com/carrotsearch/hppc/tree/0.9.1 ------------------------------------------------------------------------------------ -lib/io.netty-netty-codec-4.1.113.Final.jar bundles some 3rd party dependencies +lib/io.netty-netty-codec-4.1.115.Final.jar bundles some 3rd party dependencies -lib/io.netty-netty-codec-4.1.113.Final.jar contains the extensions to Java Collections Framework which has +lib/io.netty-netty-codec-4.1.115.Final.jar contains the extensions to Java Collections Framework which has been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * LICENSE: @@ -378,7 +378,7 @@ been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified version of Robert Harder's Public Domain +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified version of Robert Harder's Public Domain Base64 Encoder and Decoder, which can be obtained at: * LICENSE: @@ -386,7 +386,7 @@ Base64 Encoder and Decoder, which can be obtained at: * HOMEPAGE: * http://iharder.sourceforge.net/current/java/base64/ -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'Webbit', an event based +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'Webbit', an event based WebSocket and HTTP server, which can be obtained at: * LICENSE: @@ -394,7 +394,7 @@ WebSocket and HTTP server, which can be obtained at: * HOMEPAGE: * https://github.com/joewalnes/webbit -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'SLF4J', a simple logging +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'SLF4J', a simple logging facade for Java, which can be obtained at: * LICENSE: @@ -402,7 +402,7 @@ facade for Java, which can be obtained at: * HOMEPAGE: * http://www.slf4j.org/ -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'Apache Harmony', an open source +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'Apache Harmony', an open source Java SE, which can be obtained at: * NOTICE: @@ -412,7 +412,7 @@ Java SE, which can be obtained at: * HOMEPAGE: * http://archive.apache.org/dist/harmony/ -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression and decompression library written by Matthew J. Francis. It can be obtained at: * LICENSE: @@ -420,7 +420,7 @@ and decompression library written by Matthew J. Francis. It can be obtained at: * HOMEPAGE: * https://code.google.com/p/jbzip2/ -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct the suffix array and the Burrows-Wheeler transformed string for any input string of a constant-size alphabet written by Yuta Mori. It can be obtained at: @@ -429,7 +429,7 @@ a constant-size alphabet written by Yuta Mori. It can be obtained at: * HOMEPAGE: * https://github.com/y-256/libdivsufsort -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, which can be obtained at: * LICENSE: @@ -437,7 +437,7 @@ Java Concurrency Tools for the JVM, which can be obtained at: * HOMEPAGE: * https://github.com/JCTools/JCTools -lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in +lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in pure Java, which can be obtained at: * LICENSE: @@ -445,7 +445,7 @@ pure Java, which can be obtained at: * HOMEPAGE: * http://www.jcraft.com/jzlib/ -lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and +lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * LICENSE: @@ -453,7 +453,7 @@ decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * HOMEPAGE: * https://github.com/ning/compress -lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'lz4', a LZ4 Java compression +lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'lz4', a LZ4 Java compression and decompression library written by Adrien Grand. It can be obtained at: * LICENSE: @@ -461,7 +461,7 @@ and decompression library written by Adrien Grand. It can be obtained at: * HOMEPAGE: * https://github.com/jpountz/lz4-java -lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'lzma-java', a LZMA Java compression +lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'lzma-java', a LZMA Java compression and decompression library, which can be obtained at: * LICENSE: @@ -469,7 +469,7 @@ and decompression library, which can be obtained at: * HOMEPAGE: * https://github.com/jponge/lzma-java -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression and decompression library written by William Kinney. It can be obtained at: * LICENSE: @@ -477,7 +477,7 @@ and decompression library written by William Kinney. It can be obtained at: * HOMEPAGE: * https://code.google.com/p/jfastlz/ -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data interchange format, which can be obtained at: * LICENSE: @@ -485,7 +485,7 @@ Google's data interchange format, which can be obtained at: * HOMEPAGE: * https://github.com/google/protobuf -lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate +lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate a temporary self-signed X.509 certificate when the JVM does not provide the equivalent functionality. It can be obtained at: @@ -494,7 +494,7 @@ equivalent functionality. It can be obtained at: * HOMEPAGE: * http://www.bouncycastle.org/ -lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Snappy', a compression library produced +lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Snappy', a compression library produced by Google Inc, which can be obtained at: * LICENSE: @@ -502,7 +502,7 @@ by Google Inc, which can be obtained at: * HOMEPAGE: * https://github.com/google/snappy -lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java +lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java serialization API, which can be obtained at: * LICENSE: @@ -510,7 +510,7 @@ serialization API, which can be obtained at: * HOMEPAGE: * https://github.com/jboss-remoting/jboss-marshalling -lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Caliper', Google's micro- +lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Caliper', Google's micro- benchmarking framework, which can be obtained at: * LICENSE: @@ -518,7 +518,7 @@ benchmarking framework, which can be obtained at: * HOMEPAGE: * https://github.com/google/caliper -lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Apache Commons Logging', a logging +lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Apache Commons Logging', a logging framework, which can be obtained at: * LICENSE: @@ -526,7 +526,7 @@ framework, which can be obtained at: * HOMEPAGE: * http://commons.apache.org/logging/ -lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Apache Log4J', a logging framework, which +lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Apache Log4J', a logging framework, which can be obtained at: * LICENSE: @@ -534,7 +534,7 @@ can be obtained at: * HOMEPAGE: * http://logging.apache.org/log4j/ -lib/io.netty-netty-codec-4.1.113.Final.jar optionally depends on 'Aalto XML', an ultra-high performance +lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Aalto XML', an ultra-high performance non-blocking XML processor, which can be obtained at: * LICENSE: @@ -542,7 +542,7 @@ non-blocking XML processor, which can be obtained at: * HOMEPAGE: * http://wiki.fasterxml.com/AaltoHome -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * LICENSE: @@ -550,7 +550,7 @@ the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * HOMEPAGE: * https://github.com/twitter/hpack -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * LICENSE: @@ -558,7 +558,7 @@ the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * HOMEPAGE: * https://github.com/python-hyper/hpack/ -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at: * LICENSE: @@ -566,7 +566,7 @@ the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at * HOMEPAGE: * https://github.com/nghttp2/nghttp2/ -lib/io.netty-netty-codec-4.1.113.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library +lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library provides utilities for the java.lang API, which can be obtained at: * LICENSE: @@ -575,7 +575,7 @@ provides utilities for the java.lang API, which can be obtained at: * https://commons.apache.org/proper/commons-lang/ -lib/io.netty-netty-codec-4.1.113.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', +lib/io.netty-netty-codec-4.1.115.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. * LICENSE: @@ -583,7 +583,7 @@ that provides an easy way to ensure a user has everything necessary to run the M * HOMEPAGE: * https://github.com/takari/maven-wrapper -lib/io.netty-netty-codec-4.1.113.Final.jar contains the dnsinfo.h header file, +lib/io.netty-netty-codec-4.1.115.Final.jar contains the dnsinfo.h header file, that provides a way to retrieve the system DNS configuration on MacOS. This private header is also used by Apple's open source mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/). diff --git a/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt index 15a546ab8f9..bacdd97dff5 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt @@ -23,31 +23,31 @@ LongAdder), which was released with the following comments: http://creativecommons.org/publicdomain/zero/1.0/ ------------------------------------------------------------------------------------ -- lib/io.netty-netty-buffer-4.1.113.Final.jar -- lib/io.netty-netty-codec-4.1.113.Final.jar -- lib/io.netty-netty-codec-dns-4.1.113.Final.jar -- lib/io.netty-netty-codec-http-4.1.113.Final.jar -- lib/io.netty-netty-codec-http2-4.1.113.Final.jar -- lib/io.netty-netty-codec-socks-4.1.113.Final.jar -- lib/io.netty-netty-common-4.1.113.Final.jar -- lib/io.netty-netty-handler-4.1.113.Final.jar -- lib/io.netty-netty-handler-proxy-4.1.113.Final.jar -- lib/io.netty-netty-resolver-4.1.113.Final.jar -- lib/io.netty-netty-resolver-dns-4.1.113.Final.jar -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final.jar -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-windows-x86_64.jar [11] -- lib/io.netty-netty-tcnative-classes-2.0.66.Final.jar -- lib/io.netty-netty-transport-4.1.113.Final.jar -- lib/io.netty-netty-transport-classes-epoll-4.1.113.Final.jar -- lib/io.netty-netty-transport-native-epoll-4.1.113.Final-linux-aarch_64.jar -- lib/io.netty-netty-transport-native-epoll-4.1.113.Final-linux-x86_64.jar +- lib/io.netty-netty-buffer-4.1.115.Final.jar +- lib/io.netty-netty-codec-4.1.115.Final.jar +- lib/io.netty-netty-codec-dns-4.1.115.Final.jar +- lib/io.netty-netty-codec-http-4.1.115.Final.jar +- lib/io.netty-netty-codec-http2-4.1.115.Final.jar +- lib/io.netty-netty-codec-socks-4.1.115.Final.jar +- lib/io.netty-netty-common-4.1.115.Final.jar +- lib/io.netty-netty-handler-4.1.115.Final.jar +- lib/io.netty-netty-handler-proxy-4.1.115.Final.jar +- lib/io.netty-netty-resolver-4.1.115.Final.jar +- lib/io.netty-netty-resolver-dns-4.1.115.Final.jar +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final.jar +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-osx-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-osx-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-windows-x86_64.jar [11] +- lib/io.netty-netty-tcnative-classes-2.0.69.Final.jar +- lib/io.netty-netty-transport-4.1.115.Final.jar +- lib/io.netty-netty-transport-classes-epoll-4.1.115.Final.jar +- lib/io.netty-netty-transport-native-epoll-4.1.115.Final-linux-aarch_64.jar +- lib/io.netty-netty-transport-native-epoll-4.1.115.Final-linux-x86_64.jar - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-x86_64.jar - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-aarch_64.jar -- lib/io.netty-netty-transport-native-unix-common-4.1.113.Final.jar +- lib/io.netty-netty-transport-native-unix-common-4.1.115.Final.jar The Netty Project diff --git a/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt index c5e9b51ab44..9d65a58dd09 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt +++ b/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt @@ -5,25 +5,25 @@ This product includes software developed at The Apache Software Foundation (http://www.apache.org/). ------------------------------------------------------------------------------------ -- lib/io.netty-netty-buffer-4.1.113.Final.jar -- lib/io.netty-netty-codec-4.1.113.Final.jar -- lib/io.netty-netty-common-4.1.113.Final.jar -- lib/io.netty-netty-handler-4.1.113.Final.jar -- lib/io.netty-netty-resolver-4.1.113.Final.jar -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final.jar -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-windows-x86_64.jar [11] -- lib/io.netty-netty-tcnative-classes-2.0.66.Final.jar -- lib/io.netty-netty-transport-4.1.113.Final.jar -- lib/io.netty-netty-transport-classes-epoll-4.1.113.Final.jar -- lib/io.netty-netty-transport-native-epoll-4.1.113.Final-linux-aarch_64.jar -- lib/io.netty-netty-transport-native-epoll-4.1.113.Final-linux-x86_64.jar +- lib/io.netty-netty-buffer-4.1.115.Final.jar +- lib/io.netty-netty-codec-4.1.115.Final.jar +- lib/io.netty-netty-common-4.1.115.Final.jar +- lib/io.netty-netty-handler-4.1.115.Final.jar +- lib/io.netty-netty-resolver-4.1.115.Final.jar +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final.jar +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-osx-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-osx-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-windows-x86_64.jar [11] +- lib/io.netty-netty-tcnative-classes-2.0.69.Final.jar +- lib/io.netty-netty-transport-4.1.115.Final.jar +- lib/io.netty-netty-transport-classes-epoll-4.1.115.Final.jar +- lib/io.netty-netty-transport-native-epoll-4.1.115.Final-linux-aarch_64.jar +- lib/io.netty-netty-transport-native-epoll-4.1.115.Final-linux-x86_64.jar - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-x86_64.jar - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-aarch_64.jar -- lib/io.netty-netty-transport-native-unix-common-4.1.113.Final.jar +- lib/io.netty-netty-transport-native-unix-common-4.1.115.Final.jar The Netty Project diff --git a/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt index 208afe4c377..69aef1d677d 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt @@ -5,31 +5,31 @@ This product includes software developed at The Apache Software Foundation (http://www.apache.org/). ------------------------------------------------------------------------------------ -- lib/io.netty-netty-buffer-4.1.113.Final.jar -- lib/io.netty-netty-codec-4.1.113.Final.jar -- lib/io.netty-netty-codec-dns-4.1.113.Final.jar -- lib/io.netty-netty-codec-http-4.1.113.Final.jar -- lib/io.netty-netty-codec-http2-4.1.113.Final.jar -- lib/io.netty-netty-codec-socks-4.1.113.Final.jar -- lib/io.netty-netty-common-4.1.113.Final.jar -- lib/io.netty-netty-handler-4.1.113.Final.jar -- lib/io.netty-netty-handler-proxy-4.1.113.Final.jar -- lib/io.netty-netty-resolver-4.1.113.Final.jar -- lib/io.netty-netty-resolver-dns-4.1.113.Final.jar -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final.jar -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.66.Final-windows-x86_64.jar [11] -- lib/io.netty-netty-tcnative-classes-2.0.66.Final.jar -- lib/io.netty-netty-transport-4.1.113.Final.jar -- lib/io.netty-netty-transport-classes-epoll-4.1.113.Final.jar -- lib/io.netty-netty-transport-native-epoll-4.1.113.Final-linux-aarch_64.jar -- lib/io.netty-netty-transport-native-epoll-4.1.113.Final-linux-x86_64.jar +- lib/io.netty-netty-buffer-4.1.115.Final.jar +- lib/io.netty-netty-codec-4.1.115.Final.jar +- lib/io.netty-netty-codec-dns-4.1.115.Final.jar +- lib/io.netty-netty-codec-http-4.1.115.Final.jar +- lib/io.netty-netty-codec-http2-4.1.115.Final.jar +- lib/io.netty-netty-codec-socks-4.1.115.Final.jar +- lib/io.netty-netty-common-4.1.115.Final.jar +- lib/io.netty-netty-handler-4.1.115.Final.jar +- lib/io.netty-netty-handler-proxy-4.1.115.Final.jar +- lib/io.netty-netty-resolver-4.1.115.Final.jar +- lib/io.netty-netty-resolver-dns-4.1.115.Final.jar +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final.jar +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-osx-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-osx-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-windows-x86_64.jar [11] +- lib/io.netty-netty-tcnative-classes-2.0.69.Final.jar +- lib/io.netty-netty-transport-4.1.115.Final.jar +- lib/io.netty-netty-transport-classes-epoll-4.1.115.Final.jar +- lib/io.netty-netty-transport-native-epoll-4.1.115.Final-linux-aarch_64.jar +- lib/io.netty-netty-transport-native-epoll-4.1.115.Final-linux-x86_64.jar - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-x86_64.jar - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-aarch_64.jar -- lib/io.netty-netty-transport-native-unix-common-4.1.113.Final.jar +- lib/io.netty-netty-transport-native-unix-common-4.1.115.Final.jar The Netty Project diff --git a/pom.xml b/pom.xml index af34d3cfa00..0d64062650f 100644 --- a/pom.xml +++ b/pom.xml @@ -154,7 +154,7 @@ 2.18.0 1.3.0 3.12.4 - 4.1.113.Final + 4.1.115.Final 0.0.25.Final 9.1.3 2.0.9 From bb00a8320f552d335a01fbf457e5b46ae8c68d73 Mon Sep 17 00:00:00 2001 From: Yuri Mizushima Date: Fri, 20 Dec 2024 15:44:59 +0900 Subject: [PATCH 21/55] chore: remove docker tuning from tune-runner-vm action (#4536) (cherry picked from commit e9b1949a9ab06a855482d4895fa492cb1557df84) (cherry picked from commit d1a3bc6a50b2ac5ba468647cdb031ce0d44a52e7) --- .github/actions/tune-runner-vm/action.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/actions/tune-runner-vm/action.yml b/.github/actions/tune-runner-vm/action.yml index 7e5f77f9a83..32b7ed6bba3 100644 --- a/.github/actions/tune-runner-vm/action.yml +++ b/.github/actions/tune-runner-vm/action.yml @@ -76,13 +76,6 @@ runs: # stop Azure Linux agent to save RAM sudo systemctl stop walinuxagent.service || true - - # enable docker experimental mode which is - # required for using "docker build --squash" / "-Ddocker.squash=true" - daemon_json="$(sudo cat /etc/docker/daemon.json | jq '.experimental = true')" - echo "$daemon_json" | sudo tee /etc/docker/daemon.json - # restart docker daemon - sudo systemctl restart docker echo '::endgroup::' # show memory From b18b79d43979e588734814488437ad30ee48a5bf Mon Sep 17 00:00:00 2001 From: Zhangjian He Date: Fri, 10 Jan 2025 21:38:07 +0800 Subject: [PATCH 22/55] Bump vertx.version from 4.5.7 to 4.5.11 to address CVE-2024-8391 (#4545) Signed-off-by: Zhangjian He (cherry picked from commit 3732f4b33f919f58d299dc890f34dc4e42794354) (cherry picked from commit baffa345d3d128905c099776f0e555b327661962) --- bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt | 10 +++++----- .../src/main/resources/LICENSE-server.bin.txt | 10 +++++----- pom.xml | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index 9faf244e122..071272b18a7 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -251,11 +251,11 @@ Apache Software License, Version 2. - lib/io.prometheus-simpleclient_tracer_common-0.15.0.jar [12] - lib/io.prometheus-simpleclient_tracer_otel-0.15.0.jar [12] - lib/io.prometheus-simpleclient_tracer_otel_agent-0.15.0.jar [12] -- lib/io.vertx-vertx-auth-common-4.5.7.jar [13] -- lib/io.vertx-vertx-bridge-common-4.5.7.jar [14] -- lib/io.vertx-vertx-core-4.5.7.jar [15] -- lib/io.vertx-vertx-web-4.5.7.jar [16] -- lib/io.vertx-vertx-web-common-4.5.7.jar [16] +- lib/io.vertx-vertx-auth-common-4.5.11.jar [13] +- lib/io.vertx-vertx-bridge-common-4.5.11.jar [14] +- lib/io.vertx-vertx-core-4.5.11.jar [15] +- lib/io.vertx-vertx-web-4.5.11.jar [16] +- lib/io.vertx-vertx-web-common-4.5.11.jar [16] - lib/org.apache.logging.log4j-log4j-api-2.18.0.jar [17] - lib/org.apache.logging.log4j-log4j-core-2.18.0.jar [17] - lib/org.apache.logging.log4j-log4j-slf4j-impl-2.18.0.jar [17] diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt index e6430f129a9..d7823fa0aa3 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt @@ -251,11 +251,11 @@ Apache Software License, Version 2. - lib/io.prometheus-simpleclient_tracer_common-0.15.0.jar [12] - lib/io.prometheus-simpleclient_tracer_otel-0.15.0.jar [12] - lib/io.prometheus-simpleclient_tracer_otel_agent-0.15.0.jar [12] -- lib/io.vertx-vertx-auth-common-4.5.7.jar [13] -- lib/io.vertx-vertx-bridge-common-4.5.7.jar [14] -- lib/io.vertx-vertx-core-4.5.7.jar [15] -- lib/io.vertx-vertx-web-4.5.7.jar [16] -- lib/io.vertx-vertx-web-common-4.5.7.jar [16] +- lib/io.vertx-vertx-auth-common-4.5.11.jar [13] +- lib/io.vertx-vertx-bridge-common-4.5.11.jar [14] +- lib/io.vertx-vertx-core-4.5.11.jar [15] +- lib/io.vertx-vertx-web-4.5.11.jar [16] +- lib/io.vertx-vertx-web-common-4.5.11.jar [16] - lib/org.apache.logging.log4j-log4j-api-2.18.0.jar [17] - lib/org.apache.logging.log4j-log4j-core-2.18.0.jar [17] - lib/org.apache.logging.log4j-log4j-slf4j-impl-2.18.0.jar [17] diff --git a/pom.xml b/pom.xml index 0d64062650f..009e57e87db 100644 --- a/pom.xml +++ b/pom.xml @@ -173,7 +173,7 @@ 4.6.0 1.3.2 1.19.4 - 4.5.7 + 4.5.11 3.9.3 1.1.10.5 2.1.2 From eeb2697b1b6c1ae3f9ada7841361801b319c8344 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 1 Apr 2025 06:49:35 +0300 Subject: [PATCH 23/55] Add documentation to bk_server.conf about RocksDB config (#4561) (cherry picked from commit 64abf31a299a1aad02d4b6ead3fcf24a1cd73025) (cherry picked from commit e46da064d1fc917a7ec80bdc67954aab51a1fd21) --- conf/bk_server.conf | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/conf/bk_server.conf b/conf/bk_server.conf index db7cd9cfe08..8a475c2fae2 100755 --- a/conf/bk_server.conf +++ b/conf/bk_server.conf @@ -760,17 +760,31 @@ gcEntryLogMetadataCacheEnabled=false # How many entries to pre-fill in cache after a read cache miss # dbStorage_readAheadCacheBatchSize=100 +############################################################################# ## RocksDB specific configurations +############################################################################# ## DbLedgerStorage uses RocksDB to store the indexes from -## (ledgerId, entryId) -> (entryLog, offset) - +## (ledgerId, entryId) -> (entryLog, offset). This is called the entry location database. +## There's also a separate RocksDB database to to store the ledger metadata. +# +# RocksDB configuration can be applied in two ways: by using a configuration file +# or by setting individual properties. +# +# Most individual properties are applied only to the entry location database. +# To fine-tune the ledger metadata database, the configuration file method should be used. +# +# These properties apply to both entry location and ledger metadata databases +############################################################################# +# dbStorage_rocksDB_format_version=5 +# dbStorage_rocksDB_checksum_type=kxxHash +############################################################################# +# Entry location RocksDB database specific configurations: # Size of RocksDB block-cache. For best performance, this cache # should be big enough to hold a significant portion of the index # database which can reach ~2GB in some cases # Default is to use 10% / numberOfLedgers of the direct memory size # dbStorage_rocksDB_blockCacheSize= - -# Other RocksDB specific tunables +# Other RocksDB specific tunables for the entry location database # dbStorage_rocksDB_writeBufferSizeMB=64 # dbStorage_rocksDB_sstSizeInMB=64 # dbStorage_rocksDB_blockSize=65536 @@ -779,7 +793,12 @@ gcEntryLogMetadataCacheEnabled=false # dbStorage_rocksDB_numFilesInLevel0=4 # dbStorage_rocksDB_maxSizeInLevel1MB=256 # dbStorage_rocksDB_logPath= -# dbStorage_rocksDB_format_version=5 +############################################################################# +# Alternative RocksDB configuration by using configuration files. +############################################################################# +# entryLocationRocksdbConf=conf/entry_location_rocksdb.conf +# ledgerMetadataRocksdbConf=conf/ledger_metadata_rocksdb.conf +# defaultRocksdbConf=conf/default_rocksdb.conf ############################################################################# ## DirectIO entry logger configuration From 87915a7989e085263371701c0727037125668e0a Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 1 Apr 2025 17:58:10 +0300 Subject: [PATCH 24/55] Ensure that formatVersion is specified for all RocksDB dbs (#4559) (cherry picked from commit 4d8b927945a4f235c4a91b3fc7387fa7bf939c0d) (cherry picked from commit b00d2e0cd45ee4301ed4af74a734384877ad4e8e) --- .../bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java index 0c13c1c8bff..22c5bc75dbc 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java @@ -182,6 +182,7 @@ private RocksDB initializeRocksDBWithBookieConf(String basePath, String subPath, Options options = new Options(); options.setCreateIfMissing(true); ChecksumType checksumType = ChecksumType.valueOf(conf.getString(ROCKSDB_CHECKSUM_TYPE, "kxxHash")); + int formatVersion = conf.getInt(ROCKSDB_FORMAT_VERSION, 5); if (dbConfigType == DbConfigType.EntryLocation) { /* Set default RocksDB block-cache size to 10% / numberOfLedgers of direct memory, unless override */ @@ -198,7 +199,6 @@ private RocksDB initializeRocksDBWithBookieConf(String basePath, String subPath, int blockSize = conf.getInt(ROCKSDB_BLOCK_SIZE, 64 * 1024); int bloomFilterBitsPerKey = conf.getInt(ROCKSDB_BLOOM_FILTERS_BITS_PER_KEY, 10); boolean lz4CompressionEnabled = conf.getBoolean(ROCKSDB_LZ4_COMPRESSION_ENABLED, true); - int formatVersion = conf.getInt(ROCKSDB_FORMAT_VERSION, 5); if (lz4CompressionEnabled) { options.setCompressionType(CompressionType.LZ4_COMPRESSION); @@ -235,6 +235,7 @@ private RocksDB initializeRocksDBWithBookieConf(String basePath, String subPath, } else { this.cache = null; BlockBasedTableConfig tableOptions = new BlockBasedTableConfig(); + tableOptions.setFormatVersion(formatVersion); tableOptions.setChecksumType(checksumType); options.setTableFormatConfig(tableOptions); } From db2cba90b9abbb3a6d7186aefcdc754dd4e08947 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 2 Apr 2025 10:43:00 +0300 Subject: [PATCH 25/55] Improve locating config files (#4560) (cherry picked from commit f34455b2dafb7ce164b5f81db8b252a7b5cd1a7d) (cherry picked from commit a18dc640ba4f23d24f3bf80787ecf2123a9a0751) --- .../storage/ldb/KeyValueStorageRocksDB.java | 3 +- .../bookkeeper/conf/ServerConfiguration.java | 46 +++++++++++-------- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java index 22c5bc75dbc..c9e72847fbc 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java @@ -40,6 +40,7 @@ import java.util.concurrent.TimeUnit; import org.apache.bookkeeper.bookie.storage.ldb.KeyValueStorageFactory.DbConfigType; import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.commons.lang3.StringUtils; import org.rocksdb.BlockBasedTableConfig; import org.rocksdb.BloomFilter; import org.rocksdb.Cache; @@ -131,7 +132,7 @@ public KeyValueStorageRocksDB(String basePath, String subPath, DbConfigType dbCo dbFilePath = conf.getDefaultRocksDBConf(); } log.info("Searching for a RocksDB configuration file in {}", dbFilePath); - if (Paths.get(dbFilePath).toFile().exists()) { + if (StringUtils.isNotBlank(dbFilePath) && Paths.get(dbFilePath).toFile().exists()) { log.info("Found a RocksDB configuration file and using it to initialize the RocksDB"); db = initializeRocksDBWithConfFile(basePath, subPath, dbConfigType, conf, readOnly, dbFilePath); } else { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java index 1f04e53f575..9a6f0e5b9e3 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java @@ -25,6 +25,7 @@ import java.io.File; import java.net.URL; import java.util.concurrent.TimeUnit; +import lombok.SneakyThrows; import org.apache.bookkeeper.bookie.FileChannelProvider; import org.apache.bookkeeper.bookie.InterleavedLedgerStorage; import org.apache.bookkeeper.bookie.LedgerStorage; @@ -4047,12 +4048,7 @@ public boolean isSkipReplayJournalInvalidRecord() { * @return String configured default rocksdb conf. */ public String getDefaultRocksDBConf() { - String defaultPath = "conf/default_rocksdb.conf"; - URL defURL = getClass().getClassLoader().getResource(defaultPath); - if (defURL != null) { - defaultPath = defURL.getPath(); - } - return getString(DEFAULT_ROCKSDB_CONF, defaultPath); + return getString(DEFAULT_ROCKSDB_CONF, getDefaultFilePath("conf/default_rocksdb.conf")); } /** @@ -4071,12 +4067,7 @@ public ServerConfiguration setDefaultRocksDBConf(String defaultRocksdbConf) { * @return String configured entry Location rocksdb conf. */ public String getEntryLocationRocksdbConf() { - String defaultPath = "conf/entry_location_rocksdb.conf"; - URL defURL = getClass().getClassLoader().getResource(defaultPath); - if (defURL != null) { - defaultPath = defURL.getPath(); - } - return getString(ENTRY_LOCATION_ROCKSDB_CONF, defaultPath); + return getString(ENTRY_LOCATION_ROCKSDB_CONF, getDefaultFilePath("conf/entry_location_rocksdb.conf")); } /** @@ -4095,12 +4086,7 @@ public ServerConfiguration setEntryLocationRocksdbConf(String entryLocationRocks * @return String configured ledger metadata rocksdb conf. */ public String getLedgerMetadataRocksdbConf() { - String defaultPath = "conf/ledger_metadata_rocksdb.conf"; - URL defURL = getClass().getClassLoader().getResource(defaultPath); - if (defURL != null) { - defaultPath = defURL.getPath(); - } - return getString(LEDGER_METADATA_ROCKSDB_CONF, defaultPath); + return getString(LEDGER_METADATA_ROCKSDB_CONF, getDefaultFilePath("conf/ledger_metadata_rocksdb.conf")); } /** @@ -4133,4 +4119,28 @@ public ServerConfiguration setOperationMaxNumbersInSingleRocksDBWriteBatch(int m public int getMaxOperationNumbersInSingleRocksDBBatch() { return getInt(MAX_OPERATION_NUMBERS_IN_SINGLE_ROCKSDB_WRITE_BATCH, 100000); } + + /** + * Retrieves the default file path for the specified file name. + * This method prioritizes a file available in the classpath, which is often used in testing scenarios. + * If the file is not found in the classpath, the original file name is returned. + * + * @param fileName the name of the file for which to retrieve the path. + * @return the path of the file if found in the classpath, otherwise the input file name. + */ + @SneakyThrows + private String getDefaultFilePath(String fileName) { + // Attempt to locate the file in the classpath, used mainly for testing purposes. + URL resourceURL = getClass().getClassLoader().getResource(fileName); + if (resourceURL != null && "file".equals(resourceURL.getProtocol())) { + // Convert the URL to a File object using toURI() for proper URL decoding + // and platform specific file path handling (such as on Windows OS) + File file = new File(resourceURL.toURI()); + if (file.exists()) { + return file.getAbsolutePath(); + } + } + // Return the original file name if no path was found in the classpath + return fileName; + } } From b3d61c714477de471149f21da461fefb0f9a9002 Mon Sep 17 00:00:00 2001 From: lifepuzzlefun Date: Wed, 16 Apr 2025 17:36:57 +0800 Subject: [PATCH 26/55] Reduce metadataLock contention in LedgerHandle (#4549) (cherry picked from commit b10aec21bf35eefb73b96e95028eab7d759209f6) (cherry picked from commit 84ca507b823754085162f7b71a0dbdf8a1e1a5fc) --- .../bookkeeper/client/LedgerHandle.java | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerHandle.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerHandle.java index 0ea5f8fec14..e99d8ba91df 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerHandle.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerHandle.java @@ -1849,19 +1849,24 @@ void notifyWriteFailed(int index, BookieId addr) { } void maybeHandleDelayedWriteBookieFailure() { + Map toReplace = null; synchronized (metadataLock) { if (delayedWriteFailedBookies.isEmpty()) { return; } - Map toReplace = new HashMap<>(delayedWriteFailedBookies); + toReplace = new HashMap<>(delayedWriteFailedBookies); delayedWriteFailedBookies.clear(); + } - // Original intent of this change is to do a best-effort ensemble change. - // But this is not possible until the local metadata is completely immutable. - // Until the feature "Make LedgerMetadata Immutable #610" Is complete we will use - // handleBookieFailure() to handle delayed writes as regular bookie failures. - handleBookieFailure(toReplace); + if (toReplace.isEmpty()) { + return; } + + // Original intent of this change is to do a best-effort ensemble change. + // But this is not possible until the local metadata is completely immutable. + // Until the feature "Make LedgerMetadata Immutable #610" Is complete we will use + // handleBookieFailure() to handle delayed writes as regular bookie failures. + handleBookieFailure(toReplace); } void handleBookieFailure(final Map failedBookies) { @@ -1980,12 +1985,12 @@ void ensembleChangeLoop(List origEnsemble, Map fail List newEnsemble = null; Set replaced = null; + + Map toReplace = null; synchronized (metadataLock) { if (!delayedWriteFailedBookies.isEmpty()) { - Map toReplace = new HashMap<>(delayedWriteFailedBookies); + toReplace = new HashMap<>(delayedWriteFailedBookies); delayedWriteFailedBookies.clear(); - - ensembleChangeLoop(origEnsemble, toReplace); } else { newEnsemble = getCurrentEnsemble(); replaced = EnsembleUtils.diffEnsemble(origEnsemble, newEnsemble); @@ -1994,6 +1999,11 @@ void ensembleChangeLoop(List origEnsemble, Map fail changingEnsemble = false; } } + + if (toReplace != null && !toReplace.isEmpty()) { + ensembleChangeLoop(origEnsemble, toReplace); + } + if (newEnsemble != null) { // unsetSuccess outside of lock unsetSuccessAndSendWriteRequest(newEnsemble, replaced); } From 6ae67feae445d3aa498d5806ef5cc1ca13c3c2c1 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 16 Apr 2025 16:29:41 +0300 Subject: [PATCH 27/55] Upgrade Netty to 4.1.119, tcnative to 2.0.70 and io_uring to 0.0.26 (#4584) (cherry picked from commit 02d7f9fb35ac9799e7c62a31d089762f6483fb2f) (cherry picked from commit 2182932602d1ecb0b24c9d38cde25c13dea1ca75) --- .../src/main/resources/LICENSE-all.bin.txt | 110 +++++++++--------- .../src/main/resources/LICENSE-bkctl.bin.txt | 98 ++++++++-------- .../src/main/resources/LICENSE-server.bin.txt | 110 +++++++++--------- .../src/main/resources/NOTICE-all.bin.txt | 50 ++++---- .../src/main/resources/NOTICE-bkctl.bin.txt | 38 +++--- .../src/main/resources/NOTICE-server.bin.txt | 50 ++++---- pom.xml | 4 +- 7 files changed, 230 insertions(+), 230 deletions(-) diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index 071272b18a7..822a641b006 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -217,32 +217,32 @@ Apache Software License, Version 2. - lib/commons-io-commons-io-2.7.jar [8] - lib/commons-lang-commons-lang-2.6.jar [9] - lib/commons-logging-commons-logging-1.1.1.jar [10] -- lib/io.netty-netty-buffer-4.1.115.Final.jar [11] -- lib/io.netty-netty-codec-4.1.115.Final.jar [11] -- lib/io.netty-netty-codec-dns-4.1.115.Final.jar [11] -- lib/io.netty-netty-codec-http-4.1.115.Final.jar [11] -- lib/io.netty-netty-codec-http2-4.1.115.Final.jar [11] -- lib/io.netty-netty-codec-socks-4.1.115.Final.jar [11] -- lib/io.netty-netty-common-4.1.115.Final.jar [11] -- lib/io.netty-netty-handler-4.1.115.Final.jar [11] -- lib/io.netty-netty-handler-proxy-4.1.115.Final.jar [11] -- lib/io.netty-netty-resolver-4.1.115.Final.jar [11] -- lib/io.netty-netty-resolver-dns-4.1.115.Final.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-linux-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-osx-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-osx-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-windows-x86_64.jar [11] -- lib/io.netty-netty-tcnative-classes-2.0.69.Final.jar [11] -- lib/io.netty-netty-transport-4.1.115.Final.jar [11] -- lib/io.netty-netty-transport-classes-epoll-4.1.115.Final.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.115.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.115.Final-linux-x86_64.jar [11] -- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-x86_64.jar [11] -- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-aarch_64.jar [11] -- lib/io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.25.Final.jar [11] -- lib/io.netty-netty-transport-native-unix-common-4.1.115.Final.jar [11] +- lib/io.netty-netty-buffer-4.1.119.Final.jar [11] +- lib/io.netty-netty-codec-4.1.119.Final.jar [11] +- lib/io.netty-netty-codec-dns-4.1.119.Final.jar [11] +- lib/io.netty-netty-codec-http-4.1.119.Final.jar [11] +- lib/io.netty-netty-codec-http2-4.1.119.Final.jar [11] +- lib/io.netty-netty-codec-socks-4.1.119.Final.jar [11] +- lib/io.netty-netty-common-4.1.119.Final.jar [11] +- lib/io.netty-netty-handler-4.1.119.Final.jar [11] +- lib/io.netty-netty-handler-proxy-4.1.119.Final.jar [11] +- lib/io.netty-netty-resolver-4.1.119.Final.jar [11] +- lib/io.netty-netty-resolver-dns-4.1.119.Final.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-windows-x86_64.jar [11] +- lib/io.netty-netty-tcnative-classes-2.0.70.Final.jar [11] +- lib/io.netty-netty-transport-4.1.119.Final.jar [11] +- lib/io.netty-netty-transport-classes-epoll-4.1.119.Final.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.119.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.119.Final-linux-x86_64.jar [11] +- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar [11] +- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar [11] +- lib/io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.26.Final.jar [11] +- lib/io.netty-netty-transport-native-unix-common-4.1.119.Final.jar [11] - lib/io.prometheus-simpleclient-0.15.0.jar [12] - lib/io.prometheus-simpleclient_common-0.15.0.jar [12] - lib/io.prometheus-simpleclient_hotspot-0.15.0.jar [12] @@ -332,7 +332,7 @@ Apache Software License, Version 2. [8] Source available at https://github.com/apache/commons-io/tree/rel/commons-io-2.7 [9] Source available at https://github.com/apache/commons-lang/tree/LANG_2_6 [10] Source available at https://github.com/apache/commons-logging/tree/commons-logging-1.1.1 -[11] Source available at https://github.com/netty/netty/tree/netty-4.1.115.Final +[11] Source available at https://github.com/netty/netty/tree/netty-4.1.119.Final [12] Source available at https://github.com/prometheus/client_java/tree/parent-0.15.0 [13] Source available at https://github.com/vert-x3/vertx-auth/tree/4.3.2 [14] Source available at https://github.com/vert-x3/vertx-bridge-common/tree/4.3.2 @@ -372,9 +372,9 @@ Apache Software License, Version 2. [53] Source available at https://github.com/carrotsearch/hppc/tree/0.9.1 ------------------------------------------------------------------------------------ -lib/io.netty-netty-codec-4.1.115.Final.jar bundles some 3rd party dependencies +lib/io.netty-netty-codec-4.1.119.Final.jar bundles some 3rd party dependencies -lib/io.netty-netty-codec-4.1.115.Final.jar contains the extensions to Java Collections Framework which has +lib/io.netty-netty-codec-4.1.119.Final.jar contains the extensions to Java Collections Framework which has been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * LICENSE: @@ -383,7 +383,7 @@ been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified version of Robert Harder's Public Domain +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified version of Robert Harder's Public Domain Base64 Encoder and Decoder, which can be obtained at: * LICENSE: @@ -391,7 +391,7 @@ Base64 Encoder and Decoder, which can be obtained at: * HOMEPAGE: * http://iharder.sourceforge.net/current/java/base64/ -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'Webbit', an event based +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'Webbit', an event based WebSocket and HTTP server, which can be obtained at: * LICENSE: @@ -399,7 +399,7 @@ WebSocket and HTTP server, which can be obtained at: * HOMEPAGE: * https://github.com/joewalnes/webbit -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'SLF4J', a simple logging +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'SLF4J', a simple logging facade for Java, which can be obtained at: * LICENSE: @@ -407,7 +407,7 @@ facade for Java, which can be obtained at: * HOMEPAGE: * http://www.slf4j.org/ -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'Apache Harmony', an open source +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'Apache Harmony', an open source Java SE, which can be obtained at: * NOTICE: @@ -417,7 +417,7 @@ Java SE, which can be obtained at: * HOMEPAGE: * http://archive.apache.org/dist/harmony/ -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression and decompression library written by Matthew J. Francis. It can be obtained at: * LICENSE: @@ -425,7 +425,7 @@ and decompression library written by Matthew J. Francis. It can be obtained at: * HOMEPAGE: * https://code.google.com/p/jbzip2/ -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct the suffix array and the Burrows-Wheeler transformed string for any input string of a constant-size alphabet written by Yuta Mori. It can be obtained at: @@ -434,7 +434,7 @@ a constant-size alphabet written by Yuta Mori. It can be obtained at: * HOMEPAGE: * https://github.com/y-256/libdivsufsort -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, which can be obtained at: * LICENSE: @@ -442,7 +442,7 @@ Java Concurrency Tools for the JVM, which can be obtained at: * HOMEPAGE: * https://github.com/JCTools/JCTools -lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in +lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in pure Java, which can be obtained at: * LICENSE: @@ -450,7 +450,7 @@ pure Java, which can be obtained at: * HOMEPAGE: * http://www.jcraft.com/jzlib/ -lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and +lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * LICENSE: @@ -458,7 +458,7 @@ decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * HOMEPAGE: * https://github.com/ning/compress -lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'lz4', a LZ4 Java compression +lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'lz4', a LZ4 Java compression and decompression library written by Adrien Grand. It can be obtained at: * LICENSE: @@ -466,7 +466,7 @@ and decompression library written by Adrien Grand. It can be obtained at: * HOMEPAGE: * https://github.com/jpountz/lz4-java -lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'lzma-java', a LZMA Java compression +lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'lzma-java', a LZMA Java compression and decompression library, which can be obtained at: * LICENSE: @@ -474,7 +474,7 @@ and decompression library, which can be obtained at: * HOMEPAGE: * https://github.com/jponge/lzma-java -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression and decompression library written by William Kinney. It can be obtained at: * LICENSE: @@ -482,7 +482,7 @@ and decompression library written by William Kinney. It can be obtained at: * HOMEPAGE: * https://code.google.com/p/jfastlz/ -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data interchange format, which can be obtained at: * LICENSE: @@ -490,7 +490,7 @@ Google's data interchange format, which can be obtained at: * HOMEPAGE: * https://github.com/google/protobuf -lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate +lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate a temporary self-signed X.509 certificate when the JVM does not provide the equivalent functionality. It can be obtained at: @@ -499,7 +499,7 @@ equivalent functionality. It can be obtained at: * HOMEPAGE: * http://www.bouncycastle.org/ -lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Snappy', a compression library produced +lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Snappy', a compression library produced by Google Inc, which can be obtained at: * LICENSE: @@ -507,7 +507,7 @@ by Google Inc, which can be obtained at: * HOMEPAGE: * https://github.com/google/snappy -lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java +lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java serialization API, which can be obtained at: * LICENSE: @@ -515,7 +515,7 @@ serialization API, which can be obtained at: * HOMEPAGE: * https://github.com/jboss-remoting/jboss-marshalling -lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Caliper', Google's micro- +lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Caliper', Google's micro- benchmarking framework, which can be obtained at: * LICENSE: @@ -523,7 +523,7 @@ benchmarking framework, which can be obtained at: * HOMEPAGE: * https://github.com/google/caliper -lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Apache Commons Logging', a logging +lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Apache Commons Logging', a logging framework, which can be obtained at: * LICENSE: @@ -531,7 +531,7 @@ framework, which can be obtained at: * HOMEPAGE: * http://commons.apache.org/logging/ -lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Apache Log4J', a logging framework, which +lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Apache Log4J', a logging framework, which can be obtained at: * LICENSE: @@ -539,7 +539,7 @@ can be obtained at: * HOMEPAGE: * http://logging.apache.org/log4j/ -lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Aalto XML', an ultra-high performance +lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Aalto XML', an ultra-high performance non-blocking XML processor, which can be obtained at: * LICENSE: @@ -547,7 +547,7 @@ non-blocking XML processor, which can be obtained at: * HOMEPAGE: * http://wiki.fasterxml.com/AaltoHome -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * LICENSE: @@ -555,7 +555,7 @@ the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * HOMEPAGE: * https://github.com/twitter/hpack -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * LICENSE: @@ -563,7 +563,7 @@ the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * HOMEPAGE: * https://github.com/python-hyper/hpack/ -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at: * LICENSE: @@ -571,7 +571,7 @@ the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at * HOMEPAGE: * https://github.com/nghttp2/nghttp2/ -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library provides utilities for the java.lang API, which can be obtained at: * LICENSE: @@ -580,7 +580,7 @@ provides utilities for the java.lang API, which can be obtained at: * https://commons.apache.org/proper/commons-lang/ -lib/io.netty-netty-codec-4.1.115.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', +lib/io.netty-netty-codec-4.1.119.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. * LICENSE: @@ -588,7 +588,7 @@ that provides an easy way to ensure a user has everything necessary to run the M * HOMEPAGE: * https://github.com/takari/maven-wrapper -lib/io.netty-netty-codec-4.1.115.Final.jar contains the dnsinfo.h header file, +lib/io.netty-netty-codec-4.1.119.Final.jar contains the dnsinfo.h header file, that provides a way to retrieve the system DNS configuration on MacOS. This private header is also used by Apple's open source mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/). diff --git a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt index dd20fe2fe22..b80c54515dc 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt @@ -217,26 +217,26 @@ Apache Software License, Version 2. - lib/commons-io-commons-io-2.7.jar [8] - lib/commons-lang-commons-lang-2.6.jar [9] - lib/commons-logging-commons-logging-1.1.1.jar [10] -- lib/io.netty-netty-buffer-4.1.115.Final.jar [11] -- lib/io.netty-netty-codec-4.1.115.Final.jar [11] -- lib/io.netty-netty-common-4.1.115.Final.jar [11] -- lib/io.netty-netty-handler-4.1.115.Final.jar [11] -- lib/io.netty-netty-resolver-4.1.115.Final.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-linux-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-osx-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-osx-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-windows-x86_64.jar [11] -- lib/io.netty-netty-tcnative-classes-2.0.69.Final.jar [11] -- lib/io.netty-netty-transport-4.1.115.Final.jar [11] -- lib/io.netty-netty-transport-classes-epoll-4.1.115.Final.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.115.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.115.Final-linux-x86_64.jar [11] -- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-x86_64.jar [11] -- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-aarch_64.jar [11] -- lib/io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.25.Final.jar [11] -- lib/io.netty-netty-transport-native-unix-common-4.1.115.Final.jar [11] +- lib/io.netty-netty-buffer-4.1.119.Final.jar [11] +- lib/io.netty-netty-codec-4.1.119.Final.jar [11] +- lib/io.netty-netty-common-4.1.119.Final.jar [11] +- lib/io.netty-netty-handler-4.1.119.Final.jar [11] +- lib/io.netty-netty-resolver-4.1.119.Final.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-windows-x86_64.jar [11] +- lib/io.netty-netty-tcnative-classes-2.0.70.Final.jar [11] +- lib/io.netty-netty-transport-4.1.119.Final.jar [11] +- lib/io.netty-netty-transport-classes-epoll-4.1.119.Final.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.119.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.119.Final-linux-x86_64.jar [11] +- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar [11] +- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar [11] +- lib/io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.26.Final.jar [11] +- lib/io.netty-netty-transport-native-unix-common-4.1.119.Final.jar [11] - lib/org.apache.logging.log4j-log4j-api-2.18.0.jar [16] - lib/org.apache.logging.log4j-log4j-core-2.18.0.jar [16] - lib/org.apache.logging.log4j-log4j-slf4j-impl-2.18.0.jar [16] @@ -299,7 +299,7 @@ Apache Software License, Version 2. [8] Source available at https://github.com/apache/commons-io/tree/rel/commons-io-2.7 [9] Source available at https://github.com/apache/commons-lang/tree/LANG_2_6 [10] Source available at https://github.com/apache/commons-logging/tree/commons-logging-1.1.1 -[11] Source available at https://github.com/netty/netty/tree/netty-4.1.115.Final +[11] Source available at https://github.com/netty/netty/tree/netty-4.1.119.Final [16] Source available at https://github.com/apache/logging-log4j2/tree/rel/2.18.0 [18] Source available at https://github.com/apache/commons-collections/tree/collections-4.1 [19] Source available at https://github.com/apache/commons-lang/tree/LANG_3_6 @@ -330,9 +330,9 @@ Apache Software License, Version 2. [52] Source available at https://github.com/carrotsearch/hppc/tree/0.9.1 ------------------------------------------------------------------------------------ -lib/io.netty-netty-codec-4.1.115.Final.jar bundles some 3rd party dependencies +lib/io.netty-netty-codec-4.1.119.Final.jar bundles some 3rd party dependencies -lib/io.netty-netty-codec-4.1.115.Final.jar contains the extensions to Java Collections Framework which has +lib/io.netty-netty-codec-4.1.119.Final.jar contains the extensions to Java Collections Framework which has been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * LICENSE: @@ -341,7 +341,7 @@ been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified version of Robert Harder's Public Domain +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified version of Robert Harder's Public Domain Base64 Encoder and Decoder, which can be obtained at: * LICENSE: @@ -349,7 +349,7 @@ Base64 Encoder and Decoder, which can be obtained at: * HOMEPAGE: * http://iharder.sourceforge.net/current/java/base64/ -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'Webbit', an event based +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'Webbit', an event based WebSocket and HTTP server, which can be obtained at: * LICENSE: @@ -357,7 +357,7 @@ WebSocket and HTTP server, which can be obtained at: * HOMEPAGE: * https://github.com/joewalnes/webbit -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'SLF4J', a simple logging +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'SLF4J', a simple logging facade for Java, which can be obtained at: * LICENSE: @@ -365,7 +365,7 @@ facade for Java, which can be obtained at: * HOMEPAGE: * http://www.slf4j.org/ -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'Apache Harmony', an open source +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'Apache Harmony', an open source Java SE, which can be obtained at: * NOTICE: @@ -375,7 +375,7 @@ Java SE, which can be obtained at: * HOMEPAGE: * http://archive.apache.org/dist/harmony/ -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression and decompression library written by Matthew J. Francis. It can be obtained at: * LICENSE: @@ -383,7 +383,7 @@ and decompression library written by Matthew J. Francis. It can be obtained at: * HOMEPAGE: * https://code.google.com/p/jbzip2/ -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct the suffix array and the Burrows-Wheeler transformed string for any input string of a constant-size alphabet written by Yuta Mori. It can be obtained at: @@ -392,7 +392,7 @@ a constant-size alphabet written by Yuta Mori. It can be obtained at: * HOMEPAGE: * https://github.com/y-256/libdivsufsort -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, which can be obtained at: * LICENSE: @@ -400,7 +400,7 @@ Java Concurrency Tools for the JVM, which can be obtained at: * HOMEPAGE: * https://github.com/JCTools/JCTools -lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in +lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in pure Java, which can be obtained at: * LICENSE: @@ -408,7 +408,7 @@ pure Java, which can be obtained at: * HOMEPAGE: * http://www.jcraft.com/jzlib/ -lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and +lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * LICENSE: @@ -416,7 +416,7 @@ decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * HOMEPAGE: * https://github.com/ning/compress -lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'lz4', a LZ4 Java compression +lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'lz4', a LZ4 Java compression and decompression library written by Adrien Grand. It can be obtained at: * LICENSE: @@ -424,7 +424,7 @@ and decompression library written by Adrien Grand. It can be obtained at: * HOMEPAGE: * https://github.com/jpountz/lz4-java -lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'lzma-java', a LZMA Java compression +lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'lzma-java', a LZMA Java compression and decompression library, which can be obtained at: * LICENSE: @@ -432,7 +432,7 @@ and decompression library, which can be obtained at: * HOMEPAGE: * https://github.com/jponge/lzma-java -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression and decompression library written by William Kinney. It can be obtained at: * LICENSE: @@ -440,7 +440,7 @@ and decompression library written by William Kinney. It can be obtained at: * HOMEPAGE: * https://code.google.com/p/jfastlz/ -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data interchange format, which can be obtained at: * LICENSE: @@ -448,7 +448,7 @@ Google's data interchange format, which can be obtained at: * HOMEPAGE: * https://github.com/google/protobuf -lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate +lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate a temporary self-signed X.509 certificate when the JVM does not provide the equivalent functionality. It can be obtained at: @@ -457,7 +457,7 @@ equivalent functionality. It can be obtained at: * HOMEPAGE: * http://www.bouncycastle.org/ -lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Snappy', a compression library produced +lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Snappy', a compression library produced by Google Inc, which can be obtained at: * LICENSE: @@ -465,7 +465,7 @@ by Google Inc, which can be obtained at: * HOMEPAGE: * https://github.com/google/snappy -lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java +lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java serialization API, which can be obtained at: * LICENSE: @@ -473,7 +473,7 @@ serialization API, which can be obtained at: * HOMEPAGE: * https://github.com/jboss-remoting/jboss-marshalling -lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Caliper', Google's micro- +lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Caliper', Google's micro- benchmarking framework, which can be obtained at: * LICENSE: @@ -481,7 +481,7 @@ benchmarking framework, which can be obtained at: * HOMEPAGE: * https://github.com/google/caliper -lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Apache Commons Logging', a logging +lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Apache Commons Logging', a logging framework, which can be obtained at: * LICENSE: @@ -489,7 +489,7 @@ framework, which can be obtained at: * HOMEPAGE: * http://commons.apache.org/logging/ -lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Apache Log4J', a logging framework, which +lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Apache Log4J', a logging framework, which can be obtained at: * LICENSE: @@ -497,7 +497,7 @@ can be obtained at: * HOMEPAGE: * http://logging.apache.org/log4j/ -lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Aalto XML', an ultra-high performance +lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Aalto XML', an ultra-high performance non-blocking XML processor, which can be obtained at: * LICENSE: @@ -505,7 +505,7 @@ non-blocking XML processor, which can be obtained at: * HOMEPAGE: * http://wiki.fasterxml.com/AaltoHome -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * LICENSE: @@ -513,7 +513,7 @@ the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * HOMEPAGE: * https://github.com/twitter/hpack -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * LICENSE: @@ -521,7 +521,7 @@ the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * HOMEPAGE: * https://github.com/python-hyper/hpack/ -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at: * LICENSE: @@ -529,7 +529,7 @@ the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at * HOMEPAGE: * https://github.com/nghttp2/nghttp2/ -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library provides utilities for the java.lang API, which can be obtained at: * LICENSE: @@ -538,7 +538,7 @@ provides utilities for the java.lang API, which can be obtained at: * https://commons.apache.org/proper/commons-lang/ -lib/io.netty-netty-codec-4.1.115.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', +lib/io.netty-netty-codec-4.1.119.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. * LICENSE: @@ -546,7 +546,7 @@ that provides an easy way to ensure a user has everything necessary to run the M * HOMEPAGE: * https://github.com/takari/maven-wrapper -lib/io.netty-netty-codec-4.1.115.Final.jar contains the dnsinfo.h header file, +lib/io.netty-netty-codec-4.1.119.Final.jar contains the dnsinfo.h header file, that provides a way to retrieve the system DNS configuration on MacOS. This private header is also used by Apple's open source mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/). diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt index d7823fa0aa3..b962cf3842f 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt @@ -217,32 +217,32 @@ Apache Software License, Version 2. - lib/commons-io-commons-io-2.7.jar [8] - lib/commons-lang-commons-lang-2.6.jar [9] - lib/commons-logging-commons-logging-1.1.1.jar [10] -- lib/io.netty-netty-buffer-4.1.115.Final.jar [11] -- lib/io.netty-netty-codec-4.1.115.Final.jar [11] -- lib/io.netty-netty-codec-dns-4.1.115.Final.jar [11] -- lib/io.netty-netty-codec-http-4.1.115.Final.jar [11] -- lib/io.netty-netty-codec-http2-4.1.115.Final.jar [11] -- lib/io.netty-netty-codec-socks-4.1.115.Final.jar [11] -- lib/io.netty-netty-common-4.1.115.Final.jar [11] -- lib/io.netty-netty-handler-4.1.115.Final.jar [11] -- lib/io.netty-netty-handler-proxy-4.1.115.Final.jar [11] -- lib/io.netty-netty-resolver-4.1.115.Final.jar [11] -- lib/io.netty-netty-resolver-dns-4.1.115.Final.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-linux-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-osx-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-osx-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-windows-x86_64.jar [11] -- lib/io.netty-netty-tcnative-classes-2.0.69.Final.jar [11] -- lib/io.netty-netty-transport-4.1.115.Final.jar [11] -- lib/io.netty-netty-transport-classes-epoll-4.1.115.Final.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.115.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.115.Final-linux-x86_64.jar [11] -- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-x86_64.jar [11] -- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-aarch_64.jar [11] -- lib/io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.25.Final.jar [11] -- lib/io.netty-netty-transport-native-unix-common-4.1.115.Final.jar [11] +- lib/io.netty-netty-buffer-4.1.119.Final.jar [11] +- lib/io.netty-netty-codec-4.1.119.Final.jar [11] +- lib/io.netty-netty-codec-dns-4.1.119.Final.jar [11] +- lib/io.netty-netty-codec-http-4.1.119.Final.jar [11] +- lib/io.netty-netty-codec-http2-4.1.119.Final.jar [11] +- lib/io.netty-netty-codec-socks-4.1.119.Final.jar [11] +- lib/io.netty-netty-common-4.1.119.Final.jar [11] +- lib/io.netty-netty-handler-4.1.119.Final.jar [11] +- lib/io.netty-netty-handler-proxy-4.1.119.Final.jar [11] +- lib/io.netty-netty-resolver-4.1.119.Final.jar [11] +- lib/io.netty-netty-resolver-dns-4.1.119.Final.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-windows-x86_64.jar [11] +- lib/io.netty-netty-tcnative-classes-2.0.70.Final.jar [11] +- lib/io.netty-netty-transport-4.1.119.Final.jar [11] +- lib/io.netty-netty-transport-classes-epoll-4.1.119.Final.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.119.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.119.Final-linux-x86_64.jar [11] +- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar [11] +- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar [11] +- lib/io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.26.Final.jar [11] +- lib/io.netty-netty-transport-native-unix-common-4.1.119.Final.jar [11] - lib/io.prometheus-simpleclient-0.15.0.jar [12] - lib/io.prometheus-simpleclient_common-0.15.0.jar [12] - lib/io.prometheus-simpleclient_hotspot-0.15.0.jar [12] @@ -328,7 +328,7 @@ Apache Software License, Version 2. [8] Source available at https://github.com/apache/commons-io/tree/rel/commons-io-2.7 [9] Source available at https://github.com/apache/commons-lang/tree/LANG_2_6 [10] Source available at https://github.com/apache/commons-logging/tree/commons-logging-1.1.1 -[11] Source available at https://github.com/netty/netty/tree/netty-4.1.115.Final +[11] Source available at https://github.com/netty/netty/tree/netty-4.1.119.Final [12] Source available at https://github.com/prometheus/client_java/tree/parent-0.15.0 [13] Source available at https://github.com/vert-x3/vertx-auth/tree/4.3.2 [14] Source available at https://github.com/vert-x3/vertx-bridge-common/tree/4.3.2 @@ -367,9 +367,9 @@ Apache Software License, Version 2. [52] Source available at https://github.com/carrotsearch/hppc/tree/0.9.1 ------------------------------------------------------------------------------------ -lib/io.netty-netty-codec-4.1.115.Final.jar bundles some 3rd party dependencies +lib/io.netty-netty-codec-4.1.119.Final.jar bundles some 3rd party dependencies -lib/io.netty-netty-codec-4.1.115.Final.jar contains the extensions to Java Collections Framework which has +lib/io.netty-netty-codec-4.1.119.Final.jar contains the extensions to Java Collections Framework which has been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * LICENSE: @@ -378,7 +378,7 @@ been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified version of Robert Harder's Public Domain +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified version of Robert Harder's Public Domain Base64 Encoder and Decoder, which can be obtained at: * LICENSE: @@ -386,7 +386,7 @@ Base64 Encoder and Decoder, which can be obtained at: * HOMEPAGE: * http://iharder.sourceforge.net/current/java/base64/ -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'Webbit', an event based +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'Webbit', an event based WebSocket and HTTP server, which can be obtained at: * LICENSE: @@ -394,7 +394,7 @@ WebSocket and HTTP server, which can be obtained at: * HOMEPAGE: * https://github.com/joewalnes/webbit -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'SLF4J', a simple logging +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'SLF4J', a simple logging facade for Java, which can be obtained at: * LICENSE: @@ -402,7 +402,7 @@ facade for Java, which can be obtained at: * HOMEPAGE: * http://www.slf4j.org/ -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'Apache Harmony', an open source +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'Apache Harmony', an open source Java SE, which can be obtained at: * NOTICE: @@ -412,7 +412,7 @@ Java SE, which can be obtained at: * HOMEPAGE: * http://archive.apache.org/dist/harmony/ -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression and decompression library written by Matthew J. Francis. It can be obtained at: * LICENSE: @@ -420,7 +420,7 @@ and decompression library written by Matthew J. Francis. It can be obtained at: * HOMEPAGE: * https://code.google.com/p/jbzip2/ -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct the suffix array and the Burrows-Wheeler transformed string for any input string of a constant-size alphabet written by Yuta Mori. It can be obtained at: @@ -429,7 +429,7 @@ a constant-size alphabet written by Yuta Mori. It can be obtained at: * HOMEPAGE: * https://github.com/y-256/libdivsufsort -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, which can be obtained at: * LICENSE: @@ -437,7 +437,7 @@ Java Concurrency Tools for the JVM, which can be obtained at: * HOMEPAGE: * https://github.com/JCTools/JCTools -lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in +lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in pure Java, which can be obtained at: * LICENSE: @@ -445,7 +445,7 @@ pure Java, which can be obtained at: * HOMEPAGE: * http://www.jcraft.com/jzlib/ -lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and +lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * LICENSE: @@ -453,7 +453,7 @@ decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * HOMEPAGE: * https://github.com/ning/compress -lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'lz4', a LZ4 Java compression +lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'lz4', a LZ4 Java compression and decompression library written by Adrien Grand. It can be obtained at: * LICENSE: @@ -461,7 +461,7 @@ and decompression library written by Adrien Grand. It can be obtained at: * HOMEPAGE: * https://github.com/jpountz/lz4-java -lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'lzma-java', a LZMA Java compression +lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'lzma-java', a LZMA Java compression and decompression library, which can be obtained at: * LICENSE: @@ -469,7 +469,7 @@ and decompression library, which can be obtained at: * HOMEPAGE: * https://github.com/jponge/lzma-java -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression and decompression library written by William Kinney. It can be obtained at: * LICENSE: @@ -477,7 +477,7 @@ and decompression library written by William Kinney. It can be obtained at: * HOMEPAGE: * https://code.google.com/p/jfastlz/ -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data interchange format, which can be obtained at: * LICENSE: @@ -485,7 +485,7 @@ Google's data interchange format, which can be obtained at: * HOMEPAGE: * https://github.com/google/protobuf -lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate +lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate a temporary self-signed X.509 certificate when the JVM does not provide the equivalent functionality. It can be obtained at: @@ -494,7 +494,7 @@ equivalent functionality. It can be obtained at: * HOMEPAGE: * http://www.bouncycastle.org/ -lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Snappy', a compression library produced +lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Snappy', a compression library produced by Google Inc, which can be obtained at: * LICENSE: @@ -502,7 +502,7 @@ by Google Inc, which can be obtained at: * HOMEPAGE: * https://github.com/google/snappy -lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java +lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java serialization API, which can be obtained at: * LICENSE: @@ -510,7 +510,7 @@ serialization API, which can be obtained at: * HOMEPAGE: * https://github.com/jboss-remoting/jboss-marshalling -lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Caliper', Google's micro- +lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Caliper', Google's micro- benchmarking framework, which can be obtained at: * LICENSE: @@ -518,7 +518,7 @@ benchmarking framework, which can be obtained at: * HOMEPAGE: * https://github.com/google/caliper -lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Apache Commons Logging', a logging +lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Apache Commons Logging', a logging framework, which can be obtained at: * LICENSE: @@ -526,7 +526,7 @@ framework, which can be obtained at: * HOMEPAGE: * http://commons.apache.org/logging/ -lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Apache Log4J', a logging framework, which +lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Apache Log4J', a logging framework, which can be obtained at: * LICENSE: @@ -534,7 +534,7 @@ can be obtained at: * HOMEPAGE: * http://logging.apache.org/log4j/ -lib/io.netty-netty-codec-4.1.115.Final.jar optionally depends on 'Aalto XML', an ultra-high performance +lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Aalto XML', an ultra-high performance non-blocking XML processor, which can be obtained at: * LICENSE: @@ -542,7 +542,7 @@ non-blocking XML processor, which can be obtained at: * HOMEPAGE: * http://wiki.fasterxml.com/AaltoHome -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * LICENSE: @@ -550,7 +550,7 @@ the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * HOMEPAGE: * https://github.com/twitter/hpack -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * LICENSE: @@ -558,7 +558,7 @@ the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * HOMEPAGE: * https://github.com/python-hyper/hpack/ -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at: * LICENSE: @@ -566,7 +566,7 @@ the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at * HOMEPAGE: * https://github.com/nghttp2/nghttp2/ -lib/io.netty-netty-codec-4.1.115.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library +lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library provides utilities for the java.lang API, which can be obtained at: * LICENSE: @@ -575,7 +575,7 @@ provides utilities for the java.lang API, which can be obtained at: * https://commons.apache.org/proper/commons-lang/ -lib/io.netty-netty-codec-4.1.115.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', +lib/io.netty-netty-codec-4.1.119.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. * LICENSE: @@ -583,7 +583,7 @@ that provides an easy way to ensure a user has everything necessary to run the M * HOMEPAGE: * https://github.com/takari/maven-wrapper -lib/io.netty-netty-codec-4.1.115.Final.jar contains the dnsinfo.h header file, +lib/io.netty-netty-codec-4.1.119.Final.jar contains the dnsinfo.h header file, that provides a way to retrieve the system DNS configuration on MacOS. This private header is also used by Apple's open source mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/). diff --git a/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt index bacdd97dff5..6a5034b6495 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt @@ -23,31 +23,31 @@ LongAdder), which was released with the following comments: http://creativecommons.org/publicdomain/zero/1.0/ ------------------------------------------------------------------------------------ -- lib/io.netty-netty-buffer-4.1.115.Final.jar -- lib/io.netty-netty-codec-4.1.115.Final.jar -- lib/io.netty-netty-codec-dns-4.1.115.Final.jar -- lib/io.netty-netty-codec-http-4.1.115.Final.jar -- lib/io.netty-netty-codec-http2-4.1.115.Final.jar -- lib/io.netty-netty-codec-socks-4.1.115.Final.jar -- lib/io.netty-netty-common-4.1.115.Final.jar -- lib/io.netty-netty-handler-4.1.115.Final.jar -- lib/io.netty-netty-handler-proxy-4.1.115.Final.jar -- lib/io.netty-netty-resolver-4.1.115.Final.jar -- lib/io.netty-netty-resolver-dns-4.1.115.Final.jar -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final.jar -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-linux-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-osx-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-osx-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-windows-x86_64.jar [11] -- lib/io.netty-netty-tcnative-classes-2.0.69.Final.jar -- lib/io.netty-netty-transport-4.1.115.Final.jar -- lib/io.netty-netty-transport-classes-epoll-4.1.115.Final.jar -- lib/io.netty-netty-transport-native-epoll-4.1.115.Final-linux-aarch_64.jar -- lib/io.netty-netty-transport-native-epoll-4.1.115.Final-linux-x86_64.jar -- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-x86_64.jar -- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-aarch_64.jar -- lib/io.netty-netty-transport-native-unix-common-4.1.115.Final.jar +- lib/io.netty-netty-buffer-4.1.119.Final.jar +- lib/io.netty-netty-codec-4.1.119.Final.jar +- lib/io.netty-netty-codec-dns-4.1.119.Final.jar +- lib/io.netty-netty-codec-http-4.1.119.Final.jar +- lib/io.netty-netty-codec-http2-4.1.119.Final.jar +- lib/io.netty-netty-codec-socks-4.1.119.Final.jar +- lib/io.netty-netty-common-4.1.119.Final.jar +- lib/io.netty-netty-handler-4.1.119.Final.jar +- lib/io.netty-netty-handler-proxy-4.1.119.Final.jar +- lib/io.netty-netty-resolver-4.1.119.Final.jar +- lib/io.netty-netty-resolver-dns-4.1.119.Final.jar +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final.jar +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-windows-x86_64.jar [11] +- lib/io.netty-netty-tcnative-classes-2.0.70.Final.jar +- lib/io.netty-netty-transport-4.1.119.Final.jar +- lib/io.netty-netty-transport-classes-epoll-4.1.119.Final.jar +- lib/io.netty-netty-transport-native-epoll-4.1.119.Final-linux-aarch_64.jar +- lib/io.netty-netty-transport-native-epoll-4.1.119.Final-linux-x86_64.jar +- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar +- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar +- lib/io.netty-netty-transport-native-unix-common-4.1.119.Final.jar The Netty Project diff --git a/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt index 9d65a58dd09..602eb0b5850 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt +++ b/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt @@ -5,25 +5,25 @@ This product includes software developed at The Apache Software Foundation (http://www.apache.org/). ------------------------------------------------------------------------------------ -- lib/io.netty-netty-buffer-4.1.115.Final.jar -- lib/io.netty-netty-codec-4.1.115.Final.jar -- lib/io.netty-netty-common-4.1.115.Final.jar -- lib/io.netty-netty-handler-4.1.115.Final.jar -- lib/io.netty-netty-resolver-4.1.115.Final.jar -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final.jar -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-linux-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-osx-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-osx-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-windows-x86_64.jar [11] -- lib/io.netty-netty-tcnative-classes-2.0.69.Final.jar -- lib/io.netty-netty-transport-4.1.115.Final.jar -- lib/io.netty-netty-transport-classes-epoll-4.1.115.Final.jar -- lib/io.netty-netty-transport-native-epoll-4.1.115.Final-linux-aarch_64.jar -- lib/io.netty-netty-transport-native-epoll-4.1.115.Final-linux-x86_64.jar -- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-x86_64.jar -- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-aarch_64.jar -- lib/io.netty-netty-transport-native-unix-common-4.1.115.Final.jar +- lib/io.netty-netty-buffer-4.1.119.Final.jar +- lib/io.netty-netty-codec-4.1.119.Final.jar +- lib/io.netty-netty-common-4.1.119.Final.jar +- lib/io.netty-netty-handler-4.1.119.Final.jar +- lib/io.netty-netty-resolver-4.1.119.Final.jar +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final.jar +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-windows-x86_64.jar [11] +- lib/io.netty-netty-tcnative-classes-2.0.70.Final.jar +- lib/io.netty-netty-transport-4.1.119.Final.jar +- lib/io.netty-netty-transport-classes-epoll-4.1.119.Final.jar +- lib/io.netty-netty-transport-native-epoll-4.1.119.Final-linux-aarch_64.jar +- lib/io.netty-netty-transport-native-epoll-4.1.119.Final-linux-x86_64.jar +- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar +- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar +- lib/io.netty-netty-transport-native-unix-common-4.1.119.Final.jar The Netty Project diff --git a/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt index 69aef1d677d..54322df1ae2 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt @@ -5,31 +5,31 @@ This product includes software developed at The Apache Software Foundation (http://www.apache.org/). ------------------------------------------------------------------------------------ -- lib/io.netty-netty-buffer-4.1.115.Final.jar -- lib/io.netty-netty-codec-4.1.115.Final.jar -- lib/io.netty-netty-codec-dns-4.1.115.Final.jar -- lib/io.netty-netty-codec-http-4.1.115.Final.jar -- lib/io.netty-netty-codec-http2-4.1.115.Final.jar -- lib/io.netty-netty-codec-socks-4.1.115.Final.jar -- lib/io.netty-netty-common-4.1.115.Final.jar -- lib/io.netty-netty-handler-4.1.115.Final.jar -- lib/io.netty-netty-handler-proxy-4.1.115.Final.jar -- lib/io.netty-netty-resolver-4.1.115.Final.jar -- lib/io.netty-netty-resolver-dns-4.1.115.Final.jar -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final.jar -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-linux-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-osx-aarch_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-osx-x86_64.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.69.Final-windows-x86_64.jar [11] -- lib/io.netty-netty-tcnative-classes-2.0.69.Final.jar -- lib/io.netty-netty-transport-4.1.115.Final.jar -- lib/io.netty-netty-transport-classes-epoll-4.1.115.Final.jar -- lib/io.netty-netty-transport-native-epoll-4.1.115.Final-linux-aarch_64.jar -- lib/io.netty-netty-transport-native-epoll-4.1.115.Final-linux-x86_64.jar -- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-x86_64.jar -- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-aarch_64.jar -- lib/io.netty-netty-transport-native-unix-common-4.1.115.Final.jar +- lib/io.netty-netty-buffer-4.1.119.Final.jar +- lib/io.netty-netty-codec-4.1.119.Final.jar +- lib/io.netty-netty-codec-dns-4.1.119.Final.jar +- lib/io.netty-netty-codec-http-4.1.119.Final.jar +- lib/io.netty-netty-codec-http2-4.1.119.Final.jar +- lib/io.netty-netty-codec-socks-4.1.119.Final.jar +- lib/io.netty-netty-common-4.1.119.Final.jar +- lib/io.netty-netty-handler-4.1.119.Final.jar +- lib/io.netty-netty-handler-proxy-4.1.119.Final.jar +- lib/io.netty-netty-resolver-4.1.119.Final.jar +- lib/io.netty-netty-resolver-dns-4.1.119.Final.jar +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final.jar +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-aarch_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-x86_64.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-windows-x86_64.jar [11] +- lib/io.netty-netty-tcnative-classes-2.0.70.Final.jar +- lib/io.netty-netty-transport-4.1.119.Final.jar +- lib/io.netty-netty-transport-classes-epoll-4.1.119.Final.jar +- lib/io.netty-netty-transport-native-epoll-4.1.119.Final-linux-aarch_64.jar +- lib/io.netty-netty-transport-native-epoll-4.1.119.Final-linux-x86_64.jar +- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar +- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar +- lib/io.netty-netty-transport-native-unix-common-4.1.119.Final.jar The Netty Project diff --git a/pom.xml b/pom.xml index 009e57e87db..82f7d5239dd 100644 --- a/pom.xml +++ b/pom.xml @@ -154,8 +154,8 @@ 2.18.0 1.3.0 3.12.4 - 4.1.115.Final - 0.0.25.Final + 4.1.119.Final + 0.0.26.Final 9.1.3 2.0.9 0.15.0 From 2d2318c5ece63c82f27f3bbf4373648bd798d79c Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 16 Apr 2025 12:42:57 +0300 Subject: [PATCH 28/55] Upgrade Apache Commons libraries to compatible versions (#4582) While working on #4580, I noticed that the Commons Compress version and the Commons Codec library version aren't compatible. It causes a ClassDefNotFoundError in running tests which were using some specific methods of docker-java. While exploring this, I noticed that Apache Commons library versions haven't been kept up-to-date for a long time and it's better to handle that for 4.18.0 release. - upgrade commons-cli from 1.2 to 1.9.0 - upgrade commons-codec from 1.6 to 1.18.0 - upgrade commons-io from 2.17.0 to 2.19.0 - upgrade commons-lang3 from 3.6 to 3.17.0 - upgrade commons-compress from 1.26.0 to 1.27.0 (cherry picked from commit bcd6b52fcaa9b82cd60dbc3a8583df4ca60796e7) (cherry picked from commit a01aa111a4b10c779cd883a120e7ee3a2e3226d5) --- .../src/main/resources/LICENSE-all.bin.txt | 15 ++++++++------- .../src/main/resources/LICENSE-bkctl.bin.txt | 16 ++++++++-------- .../src/main/resources/LICENSE-server.bin.txt | 15 ++++++++------- pom.xml | 10 +++++----- 4 files changed, 29 insertions(+), 27 deletions(-) diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index 822a641b006..863e2cb17de 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -211,10 +211,10 @@ Apache Software License, Version 2. - lib/com.google.guava-guava-32.0.1-jre.jar [4] - lib/com.google.guava-failureaccess-1.0.1.jar [4] - lib/com.google.guava-listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar [4] -- lib/commons-cli-commons-cli-1.2.jar [5] -- lib/commons-codec-commons-codec-1.6.jar [6] +- lib/commons-cli-commons-cli-1.9.0.jar [5] +- lib/commons-codec-commons-codec-1.18.0.jar [6] - lib/commons-configuration-commons-configuration-1.10.jar [7] -- lib/commons-io-commons-io-2.7.jar [8] +- lib/commons-io-commons-io-2.19.0.jar [8] - lib/commons-lang-commons-lang-2.6.jar [9] - lib/commons-logging-commons-logging-1.1.1.jar [10] - lib/io.netty-netty-buffer-4.1.119.Final.jar [11] @@ -260,7 +260,7 @@ Apache Software License, Version 2. - lib/org.apache.logging.log4j-log4j-core-2.18.0.jar [17] - lib/org.apache.logging.log4j-log4j-slf4j-impl-2.18.0.jar [17] - lib/org.apache.commons-commons-collections4-4.1.jar [19] -- lib/org.apache.commons-commons-lang3-3.6.jar [20] +- lib/org.apache.commons-commons-lang3-3.17.0.jar [58] - lib/org.apache.zookeeper-zookeeper-3.9.3.jar [21] - lib/org.apache.zookeeper-zookeeper-jute-3.9.3.jar [21] - lib/org.apache.zookeeper-zookeeper-3.9.3-tests.jar [21] @@ -326,10 +326,10 @@ Apache Software License, Version 2. [2] Source available at https://github.com/FasterXML/jackson-core/tree/jackson-core-2.17.1 [3] Source available at https://github.com/FasterXML/jackson-databind/tree/jackson-databind-2.17.1 [4] Source available at https://github.com/google/guava/tree/v32.0.1 -[5] Source available at https://github.com/apache/commons-cli/tree/cli-1.2 -[6] Source available at https://github.com/apache/commons-codec/tree/commons-codec-1.6-RC2 +[5] Source available at https://github.com/apache/commons-cli/tree/rel/commons-cli-1.9.0 +[6] Source available at https://github.com/apache/commons-codec/tree/rel/commons-codec-1.18.0 [7] Source available at https://github.com/apache/commons-configuration/tree/CONFIGURATION_1_10 -[8] Source available at https://github.com/apache/commons-io/tree/rel/commons-io-2.7 +[8] Source available at https://github.com/apache/commons-io/tree/rel/commons-io-2.19.0 [9] Source available at https://github.com/apache/commons-lang/tree/LANG_2_6 [10] Source available at https://github.com/apache/commons-logging/tree/commons-logging-1.1.1 [11] Source available at https://github.com/netty/netty/tree/netty-4.1.119.Final @@ -370,6 +370,7 @@ Apache Software License, Version 2. [51] Source available at https://github.com/ReactiveX/RxJava/tree/v3.0.1 [52] Source available at https://github.com/HdrHistogram/HdrHistogram/tree/HdrHistogram-2.1.10 [53] Source available at https://github.com/carrotsearch/hppc/tree/0.9.1 +[58] Source available at https://github.com/apache/commons-lang/tree/rel/commons-lang-3.17.0 ------------------------------------------------------------------------------------ lib/io.netty-netty-codec-4.1.119.Final.jar bundles some 3rd party dependencies diff --git a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt index b80c54515dc..95c925e1ff2 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt @@ -211,10 +211,10 @@ Apache Software License, Version 2. - lib/com.google.guava-guava-32.0.1-jre.jar [4] - lib/com.google.guava-failureaccess-1.0.1.jar [4] - lib/com.google.guava-listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar [4] -- lib/commons-cli-commons-cli-1.2.jar [5] -- lib/commons-codec-commons-codec-1.6.jar [6] +- lib/commons-cli-commons-cli-1.9.0.jar [5] +- lib/commons-codec-commons-codec-1.18.0.jar [6] - lib/commons-configuration-commons-configuration-1.10.jar [7] -- lib/commons-io-commons-io-2.7.jar [8] +- lib/commons-io-commons-io-2.19.0.jar [8] - lib/commons-lang-commons-lang-2.6.jar [9] - lib/commons-logging-commons-logging-1.1.1.jar [10] - lib/io.netty-netty-buffer-4.1.119.Final.jar [11] @@ -241,7 +241,7 @@ Apache Software License, Version 2. - lib/org.apache.logging.log4j-log4j-core-2.18.0.jar [16] - lib/org.apache.logging.log4j-log4j-slf4j-impl-2.18.0.jar [16] - lib/org.apache.commons-commons-collections4-4.1.jar [18] -- lib/org.apache.commons-commons-lang3-3.6.jar [19] +- lib/org.apache.commons-commons-lang3-3.17.0.jar [55] - lib/org.apache.zookeeper-zookeeper-3.9.3.jar [20] - lib/org.apache.zookeeper-zookeeper-jute-3.9.3.jar [20] - lib/org.apache.zookeeper-zookeeper-3.9.3-tests.jar [20] @@ -293,10 +293,10 @@ Apache Software License, Version 2. [2] Source available at https://github.com/FasterXML/jackson-core/tree/jackson-core-2.17.1 [3] Source available at https://github.com/FasterXML/jackson-databind/tree/jackson-databind-2.17.1 [4] Source available at https://github.com/google/guava/tree/v32.0.1 -[5] Source available at https://github.com/apache/commons-cli/tree/cli-1.2 -[6] Source available at https://github.com/apache/commons-codec/tree/commons-codec-1.6-RC2 +[5] Source available at https://github.com/apache/commons-cli/tree/rel/commons-cli-1.9.0 +[6] Source available at https://github.com/apache/commons-codec/tree/rel/commons-codec-1.18.0 [7] Source available at https://github.com/apache/commons-configuration/tree/CONFIGURATION_1_10 -[8] Source available at https://github.com/apache/commons-io/tree/rel/commons-io-2.7 +[8] Source available at https://github.com/apache/commons-io/tree/rel/commons-io-2.19.0 [9] Source available at https://github.com/apache/commons-lang/tree/LANG_2_6 [10] Source available at https://github.com/apache/commons-logging/tree/commons-logging-1.1.1 [11] Source available at https://github.com/netty/netty/tree/netty-4.1.119.Final @@ -328,7 +328,7 @@ Apache Software License, Version 2. [50] Source available at https://github.com/xerial/snappy-java/releases/tag/v1.1.10.5 [51] Source available at https://github.com/ReactiveX/RxJava/tree/v3.0.1 [52] Source available at https://github.com/carrotsearch/hppc/tree/0.9.1 - +[55] Source available at https://github.com/apache/commons-lang/tree/rel/commons-lang-3.17.0 ------------------------------------------------------------------------------------ lib/io.netty-netty-codec-4.1.119.Final.jar bundles some 3rd party dependencies diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt index b962cf3842f..923799bd091 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt @@ -211,10 +211,10 @@ Apache Software License, Version 2. - lib/com.google.guava-guava-32.0.1-jre.jar [4] - lib/com.google.guava-failureaccess-1.0.1.jar [4] - lib/com.google.guava-listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar [4] -- lib/commons-cli-commons-cli-1.2.jar [5] -- lib/commons-codec-commons-codec-1.6.jar [6] +- lib/commons-cli-commons-cli-1.9.0.jar [5] +- lib/commons-codec-commons-codec-1.18.0.jar [6] - lib/commons-configuration-commons-configuration-1.10.jar [7] -- lib/commons-io-commons-io-2.7.jar [8] +- lib/commons-io-commons-io-2.19.0.jar [8] - lib/commons-lang-commons-lang-2.6.jar [9] - lib/commons-logging-commons-logging-1.1.1.jar [10] - lib/io.netty-netty-buffer-4.1.119.Final.jar [11] @@ -260,7 +260,7 @@ Apache Software License, Version 2. - lib/org.apache.logging.log4j-log4j-core-2.18.0.jar [17] - lib/org.apache.logging.log4j-log4j-slf4j-impl-2.18.0.jar [17] - lib/org.apache.commons-commons-collections4-4.1.jar [19] -- lib/org.apache.commons-commons-lang3-3.6.jar [20] +- lib/org.apache.commons-commons-lang3-3.17.0.jar [57] - lib/org.apache.zookeeper-zookeeper-3.9.3.jar [21] - lib/org.apache.zookeeper-zookeeper-jute-3.9.3.jar [21] - lib/org.apache.zookeeper-zookeeper-3.9.3-tests.jar [21] @@ -322,10 +322,10 @@ Apache Software License, Version 2. [2] Source available at https://github.com/FasterXML/jackson-core/tree/jackson-core-2.17.1 [3] Source available at https://github.com/FasterXML/jackson-databind/tree/jackson-databind-2.17.1 [4] Source available at https://github.com/google/guava/tree/v32.0.1 -[5] Source available at https://github.com/apache/commons-cli/tree/cli-1.2 -[6] Source available at https://github.com/apache/commons-codec/tree/commons-codec-1.6-RC2 +[5] Source available at https://github.com/apache/commons-cli/tree/rel/commons-cli-1.9.0 +[6] Source available at https://github.com/apache/commons-codec/tree/rel/commons-codec-1.18.0 [7] Source available at https://github.com/apache/commons-configuration/tree/CONFIGURATION_1_10 -[8] Source available at https://github.com/apache/commons-io/tree/rel/commons-io-2.7 +[8] Source available at https://github.com/apache/commons-io/tree/rel/commons-io-2.19.0 [9] Source available at https://github.com/apache/commons-lang/tree/LANG_2_6 [10] Source available at https://github.com/apache/commons-logging/tree/commons-logging-1.1.1 [11] Source available at https://github.com/netty/netty/tree/netty-4.1.119.Final @@ -365,6 +365,7 @@ Apache Software License, Version 2. [50] Source available at https://github.com/xerial/snappy-java/releases/tag/v1.1.10.5 [51] Source available at https://github.com/ReactiveX/RxJava/tree/v3.0.1 [52] Source available at https://github.com/carrotsearch/hppc/tree/0.9.1 +[57] Source available at https://github.com/apache/commons-lang/tree/rel/commons-lang-3.17.0 ------------------------------------------------------------------------------------ lib/io.netty-netty-codec-4.1.119.Final.jar bundles some 3rd party dependencies diff --git a/pom.xml b/pom.xml index 82f7d5239dd..f9cfe6d3043 100644 --- a/pom.xml +++ b/pom.xml @@ -117,14 +117,14 @@ 1.18.2 1.8.0.Final 3.0.1 - 1.2 + 1.9.0 4.1 - 1.6 + 1.18.0 1.10 - 1.26.0 + 1.27.1 2.6 - 3.6 - 2.7 + 3.17.0 + 2.19.0 1.0.2.4 5.7.1 4.1.12.1 From 00517c0da082201c55a9d1254e8c2d1814521283 Mon Sep 17 00:00:00 2001 From: zzb <48124861+zhaizhibo@users.noreply.github.com> Date: Thu, 17 Apr 2025 16:33:04 +0800 Subject: [PATCH 29/55] Fix the coredump that occurs when calling KeyValueStorageRocksDB.count after rocksdb has been closed (#4581) * Fix the coredump that occurs when calling KeyValueStorageRocksDB.count() (possibly triggered by Prometheus) after RocksDB has been closed(#4243) * fix race when count op in process and db gets closed. --------- Co-authored-by: zhaizhibo (cherry picked from commit 2831ed3405069f2212e2c6204f645e503319ba69) (cherry picked from commit 3795f559e006726cc0d500e5f98386693b47bdb5) --- .../storage/ldb/KeyValueStorageRocksDB.java | 25 +++++++++++++++++-- .../ldb/KeyValueStorageRocksDBTest.java | 17 +++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java index c9e72847fbc..b870fb5939d 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java @@ -38,6 +38,7 @@ import java.util.List; import java.util.Map.Entry; import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.bookkeeper.bookie.storage.ldb.KeyValueStorageFactory.DbConfigType; import org.apache.bookkeeper.conf.ServerConfiguration; import org.apache.commons.lang3.StringUtils; @@ -74,6 +75,8 @@ public class KeyValueStorageRocksDB implements KeyValueStorage { static KeyValueStorageFactory factory = (defaultBasePath, subPath, dbConfigType, conf) -> new KeyValueStorageRocksDB(defaultBasePath, subPath, dbConfigType, conf); + private volatile boolean closed = true; + private final ReentrantReadWriteLock closedLock = new ReentrantReadWriteLock(); private final RocksDB db; private RocksObject options; private List columnFamilyDescriptors; @@ -147,6 +150,7 @@ public KeyValueStorageRocksDB(String basePath, String subPath, DbConfigType dbCo optionDontCache.setFillCache(false); this.writeBatchMaxSize = conf.getMaxOperationNumbersInSingleRocksDBBatch(); + this.closed = false; } private RocksDB initializeRocksDBWithConfFile(String basePath, String subPath, DbConfigType dbConfigType, @@ -287,7 +291,13 @@ private RocksDB initializeRocksDBWithBookieConf(String basePath, String subPath, @Override public void close() throws IOException { - db.close(); + try { + closedLock.writeLock().lock(); + closed = true; + db.close(); + } finally { + closedLock.writeLock().unlock(); + } if (cache != null) { cache.close(); } @@ -513,7 +523,18 @@ public void close() { @Override public long count() throws IOException { try { - return db.getLongProperty("rocksdb.estimate-num-keys"); + if (closed) { + throw new IOException("RocksDB is closed"); + } + try { + closedLock.readLock().lock(); + if (!closed) { + return db.getLongProperty("rocksdb.estimate-num-keys"); + } + throw new IOException("RocksDB is closed"); + } finally { + closedLock.readLock().unlock(); + } } catch (RocksDBException e) { throw new IOException("Error in getting records count", e); } diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDBTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDBTest.java index 2ef3e010f8b..97be7ae7c78 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDBTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDBTest.java @@ -21,9 +21,11 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import java.io.File; +import java.io.IOException; import java.net.URL; import java.nio.file.Files; import java.nio.file.Paths; @@ -131,4 +133,19 @@ public void testLevelCompactionDynamicLevelBytesFromConfigurationFile() throws E ColumnFamilyOptions familyOptions = columnFamilyDescriptorList.get(0).getOptions(); assertEquals(true, familyOptions.levelCompactionDynamicLevelBytes()); } + + @Test + public void testCallCountAfterClose() throws IOException { + ServerConfiguration configuration = new ServerConfiguration(); + URL url = getClass().getClassLoader().getResource("test_entry_location_rocksdb.conf"); + configuration.setEntryLocationRocksdbConf(url.getPath()); + File tmpDir = Files.createTempDirectory("bk-kv-rocksdbtest-file").toFile(); + Files.createDirectory(Paths.get(tmpDir.toString(), "subDir")); + KeyValueStorageRocksDB rocksDB = new KeyValueStorageRocksDB(tmpDir.toString(), "subDir", + KeyValueStorageFactory.DbConfigType.EntryLocation, configuration); + assertNotNull(rocksDB.getColumnFamilyDescriptors()); + rocksDB.close(); + IOException exception = assertThrows(IOException.class, rocksDB::count); + assertEquals("RocksDB is closed", exception.getMessage()); + } } From 7683944f881d503698409543a736d329671065f3 Mon Sep 17 00:00:00 2001 From: Yan Zhao Date: Thu, 11 Jan 2024 05:50:48 +0800 Subject: [PATCH 30/55] Improve auto-recovery noise log when some bookie down. (#4118) * Improve auto-recovery noise log when some bookie down. * Address comment. (cherry picked from commit 54168b5d6441f7cbe0b478867d8301a6eee0b891) (cherry picked from commit 82171ba198981695cf5ded4e5420716bfd3ce450) --- .../RackawareEnsemblePlacementPolicyImpl.java | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java index 86a9d507fe5..f71302e3d1d 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java @@ -467,13 +467,26 @@ public PlacementResult replaceBookie(int ensembleSize, int writeQuorum try { excludeBookies = addDefaultRackBookiesIfMinNumRacksIsEnforced(excludeBookies); excludeBookies.addAll(currentEnsemble); + + Set ensembleNodes = new HashSet<>(); + Set excludeNodes = new HashSet<>(); BookieNode bn = knownBookies.get(bookieToReplace); if (null == bn) { bn = createBookieNode(bookieToReplace); } - - Set ensembleNodes = convertBookiesToNodes(currentEnsemble); - Set excludeNodes = convertBookiesToNodes(excludeBookies); + for (BookieId bookieId : currentEnsemble) { + if (bookieId.equals(bookieToReplace)) { + continue; + } + ensembleNodes.add(convertBookieToNode(bookieId)); + } + for (BookieId bookieId : excludeBookies) { + if (bookieId.equals(bookieToReplace)) { + excludeNodes.add(bn); + continue; + } + excludeNodes.add(convertBookieToNode(bookieId)); + } excludeNodes.addAll(ensembleNodes); excludeNodes.add(bn); From 4348854092f39d70c17c9eaf8808db91380b6012 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Thu, 13 Feb 2025 17:42:44 -0800 Subject: [PATCH 31/55] [cli] Fix: recover command doesn't accept rate limit parameter (#4535) * Fix: recover command didn't accept rate limit parameter (cherry picked from commit a3e3668fbd1c5bb024a3ce793867248546f9ad75) (cherry picked from commit 9877db9e29143cfe3671a1faff7cf0d069801b68) --- .../src/main/java/org/apache/bookkeeper/bookie/BookieShell.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java index 944b97e79dc..8cf34f24898 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java @@ -488,7 +488,7 @@ public RecoverCmd() { opts.addOption("sk", "skipOpenLedgers", false, "Skip recovering open ledgers"); opts.addOption("d", "deleteCookie", false, "Delete cookie node for the bookie."); opts.addOption("sku", "skipUnrecoverableLedgers", false, "Skip unrecoverable ledgers."); - opts.addOption("rate", "replicationRate", false, "Replication rate by bytes"); + opts.addOption("rate", "replicationRate", true, "Replication rate by bytes"); } @Override From 8cbf32f568d1fe953fc773053c28e8dc3e422c8a Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Sun, 27 Apr 2025 10:09:05 +0300 Subject: [PATCH 32/55] Upgrade Netty to 4.1.121.Final (#4597) (cherry picked from commit c9dc52a4f1eef9411c1da480b3c49b720b6e081d) --- .../src/main/resources/LICENSE-all.bin.txt | 90 +++++++++---------- .../src/main/resources/LICENSE-bkctl.bin.txt | 78 ++++++++-------- .../src/main/resources/LICENSE-server.bin.txt | 90 +++++++++---------- .../src/main/resources/NOTICE-all.bin.txt | 32 +++---- .../src/main/resources/NOTICE-bkctl.bin.txt | 20 ++--- .../src/main/resources/NOTICE-server.bin.txt | 32 +++---- pom.xml | 2 +- 7 files changed, 172 insertions(+), 172 deletions(-) diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index 863e2cb17de..584308becdc 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -217,17 +217,17 @@ Apache Software License, Version 2. - lib/commons-io-commons-io-2.19.0.jar [8] - lib/commons-lang-commons-lang-2.6.jar [9] - lib/commons-logging-commons-logging-1.1.1.jar [10] -- lib/io.netty-netty-buffer-4.1.119.Final.jar [11] -- lib/io.netty-netty-codec-4.1.119.Final.jar [11] -- lib/io.netty-netty-codec-dns-4.1.119.Final.jar [11] -- lib/io.netty-netty-codec-http-4.1.119.Final.jar [11] -- lib/io.netty-netty-codec-http2-4.1.119.Final.jar [11] -- lib/io.netty-netty-codec-socks-4.1.119.Final.jar [11] -- lib/io.netty-netty-common-4.1.119.Final.jar [11] -- lib/io.netty-netty-handler-4.1.119.Final.jar [11] -- lib/io.netty-netty-handler-proxy-4.1.119.Final.jar [11] -- lib/io.netty-netty-resolver-4.1.119.Final.jar [11] -- lib/io.netty-netty-resolver-dns-4.1.119.Final.jar [11] +- lib/io.netty-netty-buffer-4.1.121.Final.jar [11] +- lib/io.netty-netty-codec-4.1.121.Final.jar [11] +- lib/io.netty-netty-codec-dns-4.1.121.Final.jar [11] +- lib/io.netty-netty-codec-http-4.1.121.Final.jar [11] +- lib/io.netty-netty-codec-http2-4.1.121.Final.jar [11] +- lib/io.netty-netty-codec-socks-4.1.121.Final.jar [11] +- lib/io.netty-netty-common-4.1.121.Final.jar [11] +- lib/io.netty-netty-handler-4.1.121.Final.jar [11] +- lib/io.netty-netty-handler-proxy-4.1.121.Final.jar [11] +- lib/io.netty-netty-resolver-4.1.121.Final.jar [11] +- lib/io.netty-netty-resolver-dns-4.1.121.Final.jar [11] - lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final.jar [11] - lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-aarch_64.jar [11] - lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-x86_64.jar [11] @@ -235,14 +235,14 @@ Apache Software License, Version 2. - lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-x86_64.jar [11] - lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-windows-x86_64.jar [11] - lib/io.netty-netty-tcnative-classes-2.0.70.Final.jar [11] -- lib/io.netty-netty-transport-4.1.119.Final.jar [11] -- lib/io.netty-netty-transport-classes-epoll-4.1.119.Final.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.119.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.119.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-transport-4.1.121.Final.jar [11] +- lib/io.netty-netty-transport-classes-epoll-4.1.121.Final.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.121.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.121.Final-linux-x86_64.jar [11] - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar [11] - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar [11] - lib/io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.26.Final.jar [11] -- lib/io.netty-netty-transport-native-unix-common-4.1.119.Final.jar [11] +- lib/io.netty-netty-transport-native-unix-common-4.1.121.Final.jar [11] - lib/io.prometheus-simpleclient-0.15.0.jar [12] - lib/io.prometheus-simpleclient_common-0.15.0.jar [12] - lib/io.prometheus-simpleclient_hotspot-0.15.0.jar [12] @@ -332,7 +332,7 @@ Apache Software License, Version 2. [8] Source available at https://github.com/apache/commons-io/tree/rel/commons-io-2.19.0 [9] Source available at https://github.com/apache/commons-lang/tree/LANG_2_6 [10] Source available at https://github.com/apache/commons-logging/tree/commons-logging-1.1.1 -[11] Source available at https://github.com/netty/netty/tree/netty-4.1.119.Final +[11] Source available at https://github.com/netty/netty/tree/netty-4.1.121.Final [12] Source available at https://github.com/prometheus/client_java/tree/parent-0.15.0 [13] Source available at https://github.com/vert-x3/vertx-auth/tree/4.3.2 [14] Source available at https://github.com/vert-x3/vertx-bridge-common/tree/4.3.2 @@ -373,9 +373,9 @@ Apache Software License, Version 2. [58] Source available at https://github.com/apache/commons-lang/tree/rel/commons-lang-3.17.0 ------------------------------------------------------------------------------------ -lib/io.netty-netty-codec-4.1.119.Final.jar bundles some 3rd party dependencies +lib/io.netty-netty-codec-4.1.121.Final.jar bundles some 3rd party dependencies -lib/io.netty-netty-codec-4.1.119.Final.jar contains the extensions to Java Collections Framework which has +lib/io.netty-netty-codec-4.1.121.Final.jar contains the extensions to Java Collections Framework which has been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * LICENSE: @@ -384,7 +384,7 @@ been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified version of Robert Harder's Public Domain +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified version of Robert Harder's Public Domain Base64 Encoder and Decoder, which can be obtained at: * LICENSE: @@ -392,7 +392,7 @@ Base64 Encoder and Decoder, which can be obtained at: * HOMEPAGE: * http://iharder.sourceforge.net/current/java/base64/ -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'Webbit', an event based +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'Webbit', an event based WebSocket and HTTP server, which can be obtained at: * LICENSE: @@ -400,7 +400,7 @@ WebSocket and HTTP server, which can be obtained at: * HOMEPAGE: * https://github.com/joewalnes/webbit -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'SLF4J', a simple logging +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'SLF4J', a simple logging facade for Java, which can be obtained at: * LICENSE: @@ -408,7 +408,7 @@ facade for Java, which can be obtained at: * HOMEPAGE: * http://www.slf4j.org/ -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'Apache Harmony', an open source +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'Apache Harmony', an open source Java SE, which can be obtained at: * NOTICE: @@ -418,7 +418,7 @@ Java SE, which can be obtained at: * HOMEPAGE: * http://archive.apache.org/dist/harmony/ -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression and decompression library written by Matthew J. Francis. It can be obtained at: * LICENSE: @@ -426,7 +426,7 @@ and decompression library written by Matthew J. Francis. It can be obtained at: * HOMEPAGE: * https://code.google.com/p/jbzip2/ -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct the suffix array and the Burrows-Wheeler transformed string for any input string of a constant-size alphabet written by Yuta Mori. It can be obtained at: @@ -435,7 +435,7 @@ a constant-size alphabet written by Yuta Mori. It can be obtained at: * HOMEPAGE: * https://github.com/y-256/libdivsufsort -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, which can be obtained at: * LICENSE: @@ -443,7 +443,7 @@ Java Concurrency Tools for the JVM, which can be obtained at: * HOMEPAGE: * https://github.com/JCTools/JCTools -lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in pure Java, which can be obtained at: * LICENSE: @@ -451,7 +451,7 @@ pure Java, which can be obtained at: * HOMEPAGE: * http://www.jcraft.com/jzlib/ -lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * LICENSE: @@ -459,7 +459,7 @@ decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * HOMEPAGE: * https://github.com/ning/compress -lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'lz4', a LZ4 Java compression +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'lz4', a LZ4 Java compression and decompression library written by Adrien Grand. It can be obtained at: * LICENSE: @@ -467,7 +467,7 @@ and decompression library written by Adrien Grand. It can be obtained at: * HOMEPAGE: * https://github.com/jpountz/lz4-java -lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'lzma-java', a LZMA Java compression +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'lzma-java', a LZMA Java compression and decompression library, which can be obtained at: * LICENSE: @@ -475,7 +475,7 @@ and decompression library, which can be obtained at: * HOMEPAGE: * https://github.com/jponge/lzma-java -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression and decompression library written by William Kinney. It can be obtained at: * LICENSE: @@ -483,7 +483,7 @@ and decompression library written by William Kinney. It can be obtained at: * HOMEPAGE: * https://code.google.com/p/jfastlz/ -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data interchange format, which can be obtained at: * LICENSE: @@ -491,7 +491,7 @@ Google's data interchange format, which can be obtained at: * HOMEPAGE: * https://github.com/google/protobuf -lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate a temporary self-signed X.509 certificate when the JVM does not provide the equivalent functionality. It can be obtained at: @@ -500,7 +500,7 @@ equivalent functionality. It can be obtained at: * HOMEPAGE: * http://www.bouncycastle.org/ -lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Snappy', a compression library produced +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Snappy', a compression library produced by Google Inc, which can be obtained at: * LICENSE: @@ -508,7 +508,7 @@ by Google Inc, which can be obtained at: * HOMEPAGE: * https://github.com/google/snappy -lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java serialization API, which can be obtained at: * LICENSE: @@ -516,7 +516,7 @@ serialization API, which can be obtained at: * HOMEPAGE: * https://github.com/jboss-remoting/jboss-marshalling -lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Caliper', Google's micro- +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Caliper', Google's micro- benchmarking framework, which can be obtained at: * LICENSE: @@ -524,7 +524,7 @@ benchmarking framework, which can be obtained at: * HOMEPAGE: * https://github.com/google/caliper -lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Apache Commons Logging', a logging +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Apache Commons Logging', a logging framework, which can be obtained at: * LICENSE: @@ -532,7 +532,7 @@ framework, which can be obtained at: * HOMEPAGE: * http://commons.apache.org/logging/ -lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Apache Log4J', a logging framework, which +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Apache Log4J', a logging framework, which can be obtained at: * LICENSE: @@ -540,7 +540,7 @@ can be obtained at: * HOMEPAGE: * http://logging.apache.org/log4j/ -lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Aalto XML', an ultra-high performance +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Aalto XML', an ultra-high performance non-blocking XML processor, which can be obtained at: * LICENSE: @@ -548,7 +548,7 @@ non-blocking XML processor, which can be obtained at: * HOMEPAGE: * http://wiki.fasterxml.com/AaltoHome -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * LICENSE: @@ -556,7 +556,7 @@ the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * HOMEPAGE: * https://github.com/twitter/hpack -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * LICENSE: @@ -564,7 +564,7 @@ the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * HOMEPAGE: * https://github.com/python-hyper/hpack/ -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at: * LICENSE: @@ -572,7 +572,7 @@ the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at * HOMEPAGE: * https://github.com/nghttp2/nghttp2/ -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library provides utilities for the java.lang API, which can be obtained at: * LICENSE: @@ -581,7 +581,7 @@ provides utilities for the java.lang API, which can be obtained at: * https://commons.apache.org/proper/commons-lang/ -lib/io.netty-netty-codec-4.1.119.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', +lib/io.netty-netty-codec-4.1.121.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. * LICENSE: @@ -589,7 +589,7 @@ that provides an easy way to ensure a user has everything necessary to run the M * HOMEPAGE: * https://github.com/takari/maven-wrapper -lib/io.netty-netty-codec-4.1.119.Final.jar contains the dnsinfo.h header file, +lib/io.netty-netty-codec-4.1.121.Final.jar contains the dnsinfo.h header file, that provides a way to retrieve the system DNS configuration on MacOS. This private header is also used by Apple's open source mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/). diff --git a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt index 95c925e1ff2..20007fd504a 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt @@ -217,11 +217,11 @@ Apache Software License, Version 2. - lib/commons-io-commons-io-2.19.0.jar [8] - lib/commons-lang-commons-lang-2.6.jar [9] - lib/commons-logging-commons-logging-1.1.1.jar [10] -- lib/io.netty-netty-buffer-4.1.119.Final.jar [11] -- lib/io.netty-netty-codec-4.1.119.Final.jar [11] -- lib/io.netty-netty-common-4.1.119.Final.jar [11] -- lib/io.netty-netty-handler-4.1.119.Final.jar [11] -- lib/io.netty-netty-resolver-4.1.119.Final.jar [11] +- lib/io.netty-netty-buffer-4.1.121.Final.jar [11] +- lib/io.netty-netty-codec-4.1.121.Final.jar [11] +- lib/io.netty-netty-common-4.1.121.Final.jar [11] +- lib/io.netty-netty-handler-4.1.121.Final.jar [11] +- lib/io.netty-netty-resolver-4.1.121.Final.jar [11] - lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final.jar [11] - lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-aarch_64.jar [11] - lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-x86_64.jar [11] @@ -229,14 +229,14 @@ Apache Software License, Version 2. - lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-x86_64.jar [11] - lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-windows-x86_64.jar [11] - lib/io.netty-netty-tcnative-classes-2.0.70.Final.jar [11] -- lib/io.netty-netty-transport-4.1.119.Final.jar [11] -- lib/io.netty-netty-transport-classes-epoll-4.1.119.Final.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.119.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.119.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-transport-4.1.121.Final.jar [11] +- lib/io.netty-netty-transport-classes-epoll-4.1.121.Final.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.121.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.121.Final-linux-x86_64.jar [11] - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar [11] - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar [11] - lib/io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.26.Final.jar [11] -- lib/io.netty-netty-transport-native-unix-common-4.1.119.Final.jar [11] +- lib/io.netty-netty-transport-native-unix-common-4.1.121.Final.jar [11] - lib/org.apache.logging.log4j-log4j-api-2.18.0.jar [16] - lib/org.apache.logging.log4j-log4j-core-2.18.0.jar [16] - lib/org.apache.logging.log4j-log4j-slf4j-impl-2.18.0.jar [16] @@ -299,7 +299,7 @@ Apache Software License, Version 2. [8] Source available at https://github.com/apache/commons-io/tree/rel/commons-io-2.19.0 [9] Source available at https://github.com/apache/commons-lang/tree/LANG_2_6 [10] Source available at https://github.com/apache/commons-logging/tree/commons-logging-1.1.1 -[11] Source available at https://github.com/netty/netty/tree/netty-4.1.119.Final +[11] Source available at https://github.com/netty/netty/tree/netty-4.1.121.Final [16] Source available at https://github.com/apache/logging-log4j2/tree/rel/2.18.0 [18] Source available at https://github.com/apache/commons-collections/tree/collections-4.1 [19] Source available at https://github.com/apache/commons-lang/tree/LANG_3_6 @@ -330,9 +330,9 @@ Apache Software License, Version 2. [52] Source available at https://github.com/carrotsearch/hppc/tree/0.9.1 [55] Source available at https://github.com/apache/commons-lang/tree/rel/commons-lang-3.17.0 ------------------------------------------------------------------------------------ -lib/io.netty-netty-codec-4.1.119.Final.jar bundles some 3rd party dependencies +lib/io.netty-netty-codec-4.1.121.Final.jar bundles some 3rd party dependencies -lib/io.netty-netty-codec-4.1.119.Final.jar contains the extensions to Java Collections Framework which has +lib/io.netty-netty-codec-4.1.121.Final.jar contains the extensions to Java Collections Framework which has been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * LICENSE: @@ -341,7 +341,7 @@ been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified version of Robert Harder's Public Domain +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified version of Robert Harder's Public Domain Base64 Encoder and Decoder, which can be obtained at: * LICENSE: @@ -349,7 +349,7 @@ Base64 Encoder and Decoder, which can be obtained at: * HOMEPAGE: * http://iharder.sourceforge.net/current/java/base64/ -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'Webbit', an event based +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'Webbit', an event based WebSocket and HTTP server, which can be obtained at: * LICENSE: @@ -357,7 +357,7 @@ WebSocket and HTTP server, which can be obtained at: * HOMEPAGE: * https://github.com/joewalnes/webbit -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'SLF4J', a simple logging +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'SLF4J', a simple logging facade for Java, which can be obtained at: * LICENSE: @@ -365,7 +365,7 @@ facade for Java, which can be obtained at: * HOMEPAGE: * http://www.slf4j.org/ -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'Apache Harmony', an open source +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'Apache Harmony', an open source Java SE, which can be obtained at: * NOTICE: @@ -375,7 +375,7 @@ Java SE, which can be obtained at: * HOMEPAGE: * http://archive.apache.org/dist/harmony/ -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression and decompression library written by Matthew J. Francis. It can be obtained at: * LICENSE: @@ -383,7 +383,7 @@ and decompression library written by Matthew J. Francis. It can be obtained at: * HOMEPAGE: * https://code.google.com/p/jbzip2/ -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct the suffix array and the Burrows-Wheeler transformed string for any input string of a constant-size alphabet written by Yuta Mori. It can be obtained at: @@ -392,7 +392,7 @@ a constant-size alphabet written by Yuta Mori. It can be obtained at: * HOMEPAGE: * https://github.com/y-256/libdivsufsort -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, which can be obtained at: * LICENSE: @@ -400,7 +400,7 @@ Java Concurrency Tools for the JVM, which can be obtained at: * HOMEPAGE: * https://github.com/JCTools/JCTools -lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in pure Java, which can be obtained at: * LICENSE: @@ -408,7 +408,7 @@ pure Java, which can be obtained at: * HOMEPAGE: * http://www.jcraft.com/jzlib/ -lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * LICENSE: @@ -416,7 +416,7 @@ decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * HOMEPAGE: * https://github.com/ning/compress -lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'lz4', a LZ4 Java compression +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'lz4', a LZ4 Java compression and decompression library written by Adrien Grand. It can be obtained at: * LICENSE: @@ -424,7 +424,7 @@ and decompression library written by Adrien Grand. It can be obtained at: * HOMEPAGE: * https://github.com/jpountz/lz4-java -lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'lzma-java', a LZMA Java compression +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'lzma-java', a LZMA Java compression and decompression library, which can be obtained at: * LICENSE: @@ -432,7 +432,7 @@ and decompression library, which can be obtained at: * HOMEPAGE: * https://github.com/jponge/lzma-java -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression and decompression library written by William Kinney. It can be obtained at: * LICENSE: @@ -440,7 +440,7 @@ and decompression library written by William Kinney. It can be obtained at: * HOMEPAGE: * https://code.google.com/p/jfastlz/ -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data interchange format, which can be obtained at: * LICENSE: @@ -448,7 +448,7 @@ Google's data interchange format, which can be obtained at: * HOMEPAGE: * https://github.com/google/protobuf -lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate a temporary self-signed X.509 certificate when the JVM does not provide the equivalent functionality. It can be obtained at: @@ -457,7 +457,7 @@ equivalent functionality. It can be obtained at: * HOMEPAGE: * http://www.bouncycastle.org/ -lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Snappy', a compression library produced +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Snappy', a compression library produced by Google Inc, which can be obtained at: * LICENSE: @@ -465,7 +465,7 @@ by Google Inc, which can be obtained at: * HOMEPAGE: * https://github.com/google/snappy -lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java serialization API, which can be obtained at: * LICENSE: @@ -473,7 +473,7 @@ serialization API, which can be obtained at: * HOMEPAGE: * https://github.com/jboss-remoting/jboss-marshalling -lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Caliper', Google's micro- +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Caliper', Google's micro- benchmarking framework, which can be obtained at: * LICENSE: @@ -481,7 +481,7 @@ benchmarking framework, which can be obtained at: * HOMEPAGE: * https://github.com/google/caliper -lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Apache Commons Logging', a logging +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Apache Commons Logging', a logging framework, which can be obtained at: * LICENSE: @@ -489,7 +489,7 @@ framework, which can be obtained at: * HOMEPAGE: * http://commons.apache.org/logging/ -lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Apache Log4J', a logging framework, which +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Apache Log4J', a logging framework, which can be obtained at: * LICENSE: @@ -497,7 +497,7 @@ can be obtained at: * HOMEPAGE: * http://logging.apache.org/log4j/ -lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Aalto XML', an ultra-high performance +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Aalto XML', an ultra-high performance non-blocking XML processor, which can be obtained at: * LICENSE: @@ -505,7 +505,7 @@ non-blocking XML processor, which can be obtained at: * HOMEPAGE: * http://wiki.fasterxml.com/AaltoHome -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * LICENSE: @@ -513,7 +513,7 @@ the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * HOMEPAGE: * https://github.com/twitter/hpack -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * LICENSE: @@ -521,7 +521,7 @@ the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * HOMEPAGE: * https://github.com/python-hyper/hpack/ -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at: * LICENSE: @@ -529,7 +529,7 @@ the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at * HOMEPAGE: * https://github.com/nghttp2/nghttp2/ -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library provides utilities for the java.lang API, which can be obtained at: * LICENSE: @@ -538,7 +538,7 @@ provides utilities for the java.lang API, which can be obtained at: * https://commons.apache.org/proper/commons-lang/ -lib/io.netty-netty-codec-4.1.119.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', +lib/io.netty-netty-codec-4.1.121.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. * LICENSE: @@ -546,7 +546,7 @@ that provides an easy way to ensure a user has everything necessary to run the M * HOMEPAGE: * https://github.com/takari/maven-wrapper -lib/io.netty-netty-codec-4.1.119.Final.jar contains the dnsinfo.h header file, +lib/io.netty-netty-codec-4.1.121.Final.jar contains the dnsinfo.h header file, that provides a way to retrieve the system DNS configuration on MacOS. This private header is also used by Apple's open source mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/). diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt index 923799bd091..d0bc4b0a369 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt @@ -217,17 +217,17 @@ Apache Software License, Version 2. - lib/commons-io-commons-io-2.19.0.jar [8] - lib/commons-lang-commons-lang-2.6.jar [9] - lib/commons-logging-commons-logging-1.1.1.jar [10] -- lib/io.netty-netty-buffer-4.1.119.Final.jar [11] -- lib/io.netty-netty-codec-4.1.119.Final.jar [11] -- lib/io.netty-netty-codec-dns-4.1.119.Final.jar [11] -- lib/io.netty-netty-codec-http-4.1.119.Final.jar [11] -- lib/io.netty-netty-codec-http2-4.1.119.Final.jar [11] -- lib/io.netty-netty-codec-socks-4.1.119.Final.jar [11] -- lib/io.netty-netty-common-4.1.119.Final.jar [11] -- lib/io.netty-netty-handler-4.1.119.Final.jar [11] -- lib/io.netty-netty-handler-proxy-4.1.119.Final.jar [11] -- lib/io.netty-netty-resolver-4.1.119.Final.jar [11] -- lib/io.netty-netty-resolver-dns-4.1.119.Final.jar [11] +- lib/io.netty-netty-buffer-4.1.121.Final.jar [11] +- lib/io.netty-netty-codec-4.1.121.Final.jar [11] +- lib/io.netty-netty-codec-dns-4.1.121.Final.jar [11] +- lib/io.netty-netty-codec-http-4.1.121.Final.jar [11] +- lib/io.netty-netty-codec-http2-4.1.121.Final.jar [11] +- lib/io.netty-netty-codec-socks-4.1.121.Final.jar [11] +- lib/io.netty-netty-common-4.1.121.Final.jar [11] +- lib/io.netty-netty-handler-4.1.121.Final.jar [11] +- lib/io.netty-netty-handler-proxy-4.1.121.Final.jar [11] +- lib/io.netty-netty-resolver-4.1.121.Final.jar [11] +- lib/io.netty-netty-resolver-dns-4.1.121.Final.jar [11] - lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final.jar [11] - lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-aarch_64.jar [11] - lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-x86_64.jar [11] @@ -235,14 +235,14 @@ Apache Software License, Version 2. - lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-x86_64.jar [11] - lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-windows-x86_64.jar [11] - lib/io.netty-netty-tcnative-classes-2.0.70.Final.jar [11] -- lib/io.netty-netty-transport-4.1.119.Final.jar [11] -- lib/io.netty-netty-transport-classes-epoll-4.1.119.Final.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.119.Final-linux-aarch_64.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.119.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-transport-4.1.121.Final.jar [11] +- lib/io.netty-netty-transport-classes-epoll-4.1.121.Final.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.121.Final-linux-aarch_64.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.121.Final-linux-x86_64.jar [11] - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar [11] - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar [11] - lib/io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.26.Final.jar [11] -- lib/io.netty-netty-transport-native-unix-common-4.1.119.Final.jar [11] +- lib/io.netty-netty-transport-native-unix-common-4.1.121.Final.jar [11] - lib/io.prometheus-simpleclient-0.15.0.jar [12] - lib/io.prometheus-simpleclient_common-0.15.0.jar [12] - lib/io.prometheus-simpleclient_hotspot-0.15.0.jar [12] @@ -328,7 +328,7 @@ Apache Software License, Version 2. [8] Source available at https://github.com/apache/commons-io/tree/rel/commons-io-2.19.0 [9] Source available at https://github.com/apache/commons-lang/tree/LANG_2_6 [10] Source available at https://github.com/apache/commons-logging/tree/commons-logging-1.1.1 -[11] Source available at https://github.com/netty/netty/tree/netty-4.1.119.Final +[11] Source available at https://github.com/netty/netty/tree/netty-4.1.121.Final [12] Source available at https://github.com/prometheus/client_java/tree/parent-0.15.0 [13] Source available at https://github.com/vert-x3/vertx-auth/tree/4.3.2 [14] Source available at https://github.com/vert-x3/vertx-bridge-common/tree/4.3.2 @@ -368,9 +368,9 @@ Apache Software License, Version 2. [57] Source available at https://github.com/apache/commons-lang/tree/rel/commons-lang-3.17.0 ------------------------------------------------------------------------------------ -lib/io.netty-netty-codec-4.1.119.Final.jar bundles some 3rd party dependencies +lib/io.netty-netty-codec-4.1.121.Final.jar bundles some 3rd party dependencies -lib/io.netty-netty-codec-4.1.119.Final.jar contains the extensions to Java Collections Framework which has +lib/io.netty-netty-codec-4.1.121.Final.jar contains the extensions to Java Collections Framework which has been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * LICENSE: @@ -379,7 +379,7 @@ been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified version of Robert Harder's Public Domain +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified version of Robert Harder's Public Domain Base64 Encoder and Decoder, which can be obtained at: * LICENSE: @@ -387,7 +387,7 @@ Base64 Encoder and Decoder, which can be obtained at: * HOMEPAGE: * http://iharder.sourceforge.net/current/java/base64/ -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'Webbit', an event based +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'Webbit', an event based WebSocket and HTTP server, which can be obtained at: * LICENSE: @@ -395,7 +395,7 @@ WebSocket and HTTP server, which can be obtained at: * HOMEPAGE: * https://github.com/joewalnes/webbit -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'SLF4J', a simple logging +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'SLF4J', a simple logging facade for Java, which can be obtained at: * LICENSE: @@ -403,7 +403,7 @@ facade for Java, which can be obtained at: * HOMEPAGE: * http://www.slf4j.org/ -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'Apache Harmony', an open source +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'Apache Harmony', an open source Java SE, which can be obtained at: * NOTICE: @@ -413,7 +413,7 @@ Java SE, which can be obtained at: * HOMEPAGE: * http://archive.apache.org/dist/harmony/ -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression and decompression library written by Matthew J. Francis. It can be obtained at: * LICENSE: @@ -421,7 +421,7 @@ and decompression library written by Matthew J. Francis. It can be obtained at: * HOMEPAGE: * https://code.google.com/p/jbzip2/ -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct the suffix array and the Burrows-Wheeler transformed string for any input string of a constant-size alphabet written by Yuta Mori. It can be obtained at: @@ -430,7 +430,7 @@ a constant-size alphabet written by Yuta Mori. It can be obtained at: * HOMEPAGE: * https://github.com/y-256/libdivsufsort -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, which can be obtained at: * LICENSE: @@ -438,7 +438,7 @@ Java Concurrency Tools for the JVM, which can be obtained at: * HOMEPAGE: * https://github.com/JCTools/JCTools -lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in pure Java, which can be obtained at: * LICENSE: @@ -446,7 +446,7 @@ pure Java, which can be obtained at: * HOMEPAGE: * http://www.jcraft.com/jzlib/ -lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * LICENSE: @@ -454,7 +454,7 @@ decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * HOMEPAGE: * https://github.com/ning/compress -lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'lz4', a LZ4 Java compression +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'lz4', a LZ4 Java compression and decompression library written by Adrien Grand. It can be obtained at: * LICENSE: @@ -462,7 +462,7 @@ and decompression library written by Adrien Grand. It can be obtained at: * HOMEPAGE: * https://github.com/jpountz/lz4-java -lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'lzma-java', a LZMA Java compression +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'lzma-java', a LZMA Java compression and decompression library, which can be obtained at: * LICENSE: @@ -470,7 +470,7 @@ and decompression library, which can be obtained at: * HOMEPAGE: * https://github.com/jponge/lzma-java -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression and decompression library written by William Kinney. It can be obtained at: * LICENSE: @@ -478,7 +478,7 @@ and decompression library written by William Kinney. It can be obtained at: * HOMEPAGE: * https://code.google.com/p/jfastlz/ -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data interchange format, which can be obtained at: * LICENSE: @@ -486,7 +486,7 @@ Google's data interchange format, which can be obtained at: * HOMEPAGE: * https://github.com/google/protobuf -lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate a temporary self-signed X.509 certificate when the JVM does not provide the equivalent functionality. It can be obtained at: @@ -495,7 +495,7 @@ equivalent functionality. It can be obtained at: * HOMEPAGE: * http://www.bouncycastle.org/ -lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Snappy', a compression library produced +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Snappy', a compression library produced by Google Inc, which can be obtained at: * LICENSE: @@ -503,7 +503,7 @@ by Google Inc, which can be obtained at: * HOMEPAGE: * https://github.com/google/snappy -lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java serialization API, which can be obtained at: * LICENSE: @@ -511,7 +511,7 @@ serialization API, which can be obtained at: * HOMEPAGE: * https://github.com/jboss-remoting/jboss-marshalling -lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Caliper', Google's micro- +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Caliper', Google's micro- benchmarking framework, which can be obtained at: * LICENSE: @@ -519,7 +519,7 @@ benchmarking framework, which can be obtained at: * HOMEPAGE: * https://github.com/google/caliper -lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Apache Commons Logging', a logging +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Apache Commons Logging', a logging framework, which can be obtained at: * LICENSE: @@ -527,7 +527,7 @@ framework, which can be obtained at: * HOMEPAGE: * http://commons.apache.org/logging/ -lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Apache Log4J', a logging framework, which +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Apache Log4J', a logging framework, which can be obtained at: * LICENSE: @@ -535,7 +535,7 @@ can be obtained at: * HOMEPAGE: * http://logging.apache.org/log4j/ -lib/io.netty-netty-codec-4.1.119.Final.jar optionally depends on 'Aalto XML', an ultra-high performance +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'Aalto XML', an ultra-high performance non-blocking XML processor, which can be obtained at: * LICENSE: @@ -543,7 +543,7 @@ non-blocking XML processor, which can be obtained at: * HOMEPAGE: * http://wiki.fasterxml.com/AaltoHome -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * LICENSE: @@ -551,7 +551,7 @@ the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * HOMEPAGE: * https://github.com/twitter/hpack -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * LICENSE: @@ -559,7 +559,7 @@ the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * HOMEPAGE: * https://github.com/python-hyper/hpack/ -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at: * LICENSE: @@ -567,7 +567,7 @@ the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at * HOMEPAGE: * https://github.com/nghttp2/nghttp2/ -lib/io.netty-netty-codec-4.1.119.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library +lib/io.netty-netty-codec-4.1.121.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library provides utilities for the java.lang API, which can be obtained at: * LICENSE: @@ -576,7 +576,7 @@ provides utilities for the java.lang API, which can be obtained at: * https://commons.apache.org/proper/commons-lang/ -lib/io.netty-netty-codec-4.1.119.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', +lib/io.netty-netty-codec-4.1.121.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. * LICENSE: @@ -584,7 +584,7 @@ that provides an easy way to ensure a user has everything necessary to run the M * HOMEPAGE: * https://github.com/takari/maven-wrapper -lib/io.netty-netty-codec-4.1.119.Final.jar contains the dnsinfo.h header file, +lib/io.netty-netty-codec-4.1.121.Final.jar contains the dnsinfo.h header file, that provides a way to retrieve the system DNS configuration on MacOS. This private header is also used by Apple's open source mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/). diff --git a/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt index 6a5034b6495..d8e2d64dea2 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt @@ -23,17 +23,17 @@ LongAdder), which was released with the following comments: http://creativecommons.org/publicdomain/zero/1.0/ ------------------------------------------------------------------------------------ -- lib/io.netty-netty-buffer-4.1.119.Final.jar -- lib/io.netty-netty-codec-4.1.119.Final.jar -- lib/io.netty-netty-codec-dns-4.1.119.Final.jar -- lib/io.netty-netty-codec-http-4.1.119.Final.jar -- lib/io.netty-netty-codec-http2-4.1.119.Final.jar -- lib/io.netty-netty-codec-socks-4.1.119.Final.jar -- lib/io.netty-netty-common-4.1.119.Final.jar -- lib/io.netty-netty-handler-4.1.119.Final.jar -- lib/io.netty-netty-handler-proxy-4.1.119.Final.jar -- lib/io.netty-netty-resolver-4.1.119.Final.jar -- lib/io.netty-netty-resolver-dns-4.1.119.Final.jar +- lib/io.netty-netty-buffer-4.1.121.Final.jar +- lib/io.netty-netty-codec-4.1.121.Final.jar +- lib/io.netty-netty-codec-dns-4.1.121.Final.jar +- lib/io.netty-netty-codec-http-4.1.121.Final.jar +- lib/io.netty-netty-codec-http2-4.1.121.Final.jar +- lib/io.netty-netty-codec-socks-4.1.121.Final.jar +- lib/io.netty-netty-common-4.1.121.Final.jar +- lib/io.netty-netty-handler-4.1.121.Final.jar +- lib/io.netty-netty-handler-proxy-4.1.121.Final.jar +- lib/io.netty-netty-resolver-4.1.121.Final.jar +- lib/io.netty-netty-resolver-dns-4.1.121.Final.jar - lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final.jar - lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-aarch_64.jar [11] - lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-x86_64.jar [11] @@ -41,13 +41,13 @@ LongAdder), which was released with the following comments: - lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-x86_64.jar [11] - lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-windows-x86_64.jar [11] - lib/io.netty-netty-tcnative-classes-2.0.70.Final.jar -- lib/io.netty-netty-transport-4.1.119.Final.jar -- lib/io.netty-netty-transport-classes-epoll-4.1.119.Final.jar -- lib/io.netty-netty-transport-native-epoll-4.1.119.Final-linux-aarch_64.jar -- lib/io.netty-netty-transport-native-epoll-4.1.119.Final-linux-x86_64.jar +- lib/io.netty-netty-transport-4.1.121.Final.jar +- lib/io.netty-netty-transport-classes-epoll-4.1.121.Final.jar +- lib/io.netty-netty-transport-native-epoll-4.1.121.Final-linux-aarch_64.jar +- lib/io.netty-netty-transport-native-epoll-4.1.121.Final-linux-x86_64.jar - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar -- lib/io.netty-netty-transport-native-unix-common-4.1.119.Final.jar +- lib/io.netty-netty-transport-native-unix-common-4.1.121.Final.jar The Netty Project diff --git a/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt index 602eb0b5850..47d5c6a522e 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt +++ b/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt @@ -5,11 +5,11 @@ This product includes software developed at The Apache Software Foundation (http://www.apache.org/). ------------------------------------------------------------------------------------ -- lib/io.netty-netty-buffer-4.1.119.Final.jar -- lib/io.netty-netty-codec-4.1.119.Final.jar -- lib/io.netty-netty-common-4.1.119.Final.jar -- lib/io.netty-netty-handler-4.1.119.Final.jar -- lib/io.netty-netty-resolver-4.1.119.Final.jar +- lib/io.netty-netty-buffer-4.1.121.Final.jar +- lib/io.netty-netty-codec-4.1.121.Final.jar +- lib/io.netty-netty-common-4.1.121.Final.jar +- lib/io.netty-netty-handler-4.1.121.Final.jar +- lib/io.netty-netty-resolver-4.1.121.Final.jar - lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final.jar - lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-aarch_64.jar [11] - lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-x86_64.jar [11] @@ -17,13 +17,13 @@ The Apache Software Foundation (http://www.apache.org/). - lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-x86_64.jar [11] - lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-windows-x86_64.jar [11] - lib/io.netty-netty-tcnative-classes-2.0.70.Final.jar -- lib/io.netty-netty-transport-4.1.119.Final.jar -- lib/io.netty-netty-transport-classes-epoll-4.1.119.Final.jar -- lib/io.netty-netty-transport-native-epoll-4.1.119.Final-linux-aarch_64.jar -- lib/io.netty-netty-transport-native-epoll-4.1.119.Final-linux-x86_64.jar +- lib/io.netty-netty-transport-4.1.121.Final.jar +- lib/io.netty-netty-transport-classes-epoll-4.1.121.Final.jar +- lib/io.netty-netty-transport-native-epoll-4.1.121.Final-linux-aarch_64.jar +- lib/io.netty-netty-transport-native-epoll-4.1.121.Final-linux-x86_64.jar - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar -- lib/io.netty-netty-transport-native-unix-common-4.1.119.Final.jar +- lib/io.netty-netty-transport-native-unix-common-4.1.121.Final.jar The Netty Project diff --git a/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt index 54322df1ae2..488aad4530a 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt @@ -5,17 +5,17 @@ This product includes software developed at The Apache Software Foundation (http://www.apache.org/). ------------------------------------------------------------------------------------ -- lib/io.netty-netty-buffer-4.1.119.Final.jar -- lib/io.netty-netty-codec-4.1.119.Final.jar -- lib/io.netty-netty-codec-dns-4.1.119.Final.jar -- lib/io.netty-netty-codec-http-4.1.119.Final.jar -- lib/io.netty-netty-codec-http2-4.1.119.Final.jar -- lib/io.netty-netty-codec-socks-4.1.119.Final.jar -- lib/io.netty-netty-common-4.1.119.Final.jar -- lib/io.netty-netty-handler-4.1.119.Final.jar -- lib/io.netty-netty-handler-proxy-4.1.119.Final.jar -- lib/io.netty-netty-resolver-4.1.119.Final.jar -- lib/io.netty-netty-resolver-dns-4.1.119.Final.jar +- lib/io.netty-netty-buffer-4.1.121.Final.jar +- lib/io.netty-netty-codec-4.1.121.Final.jar +- lib/io.netty-netty-codec-dns-4.1.121.Final.jar +- lib/io.netty-netty-codec-http-4.1.121.Final.jar +- lib/io.netty-netty-codec-http2-4.1.121.Final.jar +- lib/io.netty-netty-codec-socks-4.1.121.Final.jar +- lib/io.netty-netty-common-4.1.121.Final.jar +- lib/io.netty-netty-handler-4.1.121.Final.jar +- lib/io.netty-netty-handler-proxy-4.1.121.Final.jar +- lib/io.netty-netty-resolver-4.1.121.Final.jar +- lib/io.netty-netty-resolver-dns-4.1.121.Final.jar - lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final.jar - lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-aarch_64.jar [11] - lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-x86_64.jar [11] @@ -23,13 +23,13 @@ The Apache Software Foundation (http://www.apache.org/). - lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-x86_64.jar [11] - lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-windows-x86_64.jar [11] - lib/io.netty-netty-tcnative-classes-2.0.70.Final.jar -- lib/io.netty-netty-transport-4.1.119.Final.jar -- lib/io.netty-netty-transport-classes-epoll-4.1.119.Final.jar -- lib/io.netty-netty-transport-native-epoll-4.1.119.Final-linux-aarch_64.jar -- lib/io.netty-netty-transport-native-epoll-4.1.119.Final-linux-x86_64.jar +- lib/io.netty-netty-transport-4.1.121.Final.jar +- lib/io.netty-netty-transport-classes-epoll-4.1.121.Final.jar +- lib/io.netty-netty-transport-native-epoll-4.1.121.Final-linux-aarch_64.jar +- lib/io.netty-netty-transport-native-epoll-4.1.121.Final-linux-x86_64.jar - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar -- lib/io.netty-netty-transport-native-unix-common-4.1.119.Final.jar +- lib/io.netty-netty-transport-native-unix-common-4.1.121.Final.jar The Netty Project diff --git a/pom.xml b/pom.xml index f9cfe6d3043..679e4b858ca 100644 --- a/pom.xml +++ b/pom.xml @@ -154,7 +154,7 @@ 2.18.0 1.3.0 3.12.4 - 4.1.119.Final + 4.1.121.Final 0.0.26.Final 9.1.3 2.0.9 From 35e8310367288eae8469f14fb927e58a5997e87b Mon Sep 17 00:00:00 2001 From: Yan Zhao Date: Wed, 28 May 2025 10:07:27 +0800 Subject: [PATCH 33/55] Fix check read failed entry memory leak issue. (#4513) * Fix check read failed entry memory leak issue. * address the comments. (cherry picked from commit f0c406b3eb643864dceac40845ad39335803cd04) --- .../org/apache/bookkeeper/replication/ReplicationWorker.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java index cf1f2f2f598..05807455dc4 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java @@ -61,6 +61,7 @@ import org.apache.bookkeeper.client.BookKeeperAdmin; import org.apache.bookkeeper.client.EnsemblePlacementPolicy; import org.apache.bookkeeper.client.LedgerChecker; +import org.apache.bookkeeper.client.LedgerEntry; import org.apache.bookkeeper.client.LedgerFragment; import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.client.api.LedgerMetadata; @@ -351,6 +352,10 @@ private boolean tryReadingFaultyEntries(LedgerHandle lh, LedgerFragment ledgerFr lh.asyncReadEntries(entryIdToRead, entryIdToRead, (rc, ledHan, seq, ctx) -> { long thisEntryId = (Long) ctx; if (rc == BKException.Code.OK) { + while (seq.hasMoreElements()) { + LedgerEntry entry = seq.nextElement(); + entry.getEntryBuffer().release(); + } entriesUnableToReadForThisLedger.remove(thisEntryId); if (numOfResponsesToWaitFor.decrementAndGet() == 0) { multiReadComplete.countDown(); From 4c630e26d46f91e48d5ed6edc1a89d31642e948d Mon Sep 17 00:00:00 2001 From: Yong Zhang Date: Fri, 30 May 2025 10:10:16 +0800 Subject: [PATCH 34/55] Fix the data loss issue that caused by the wrong entry log header (#4607) * Fix the data loss issue that caused by the wrong entry log header --- # Motivation We observed numerous errors in the broker that failed to read the ledger from the bookkeeper; although the ledger metadata still exists, it was unable to read from the bookkeeper. After checking the data, we found the ledger located entry log was deleted by the bookkeeper. We have a data loss issue with the bookkeeper. The entry log file was deleted by the Garbage collector because the entry log file wrote a wrong file header. And there is an example that the shows the header is wrong: ``` Failed to get ledgers map index from: 82.log : Not all ledgers were found in ledgers map index. expected: -1932430239 -- found: 0 -- entryLogId: 82 ``` * Add test (cherry picked from commit 52d779ae4a6b28a1c3c7221ef21c04878257c342) --- .../bookkeeper/bookie/DefaultEntryLogger.java | 4 ++ .../bookie/EntryLoggerAllocator.java | 1 + .../bookie/GarbageCollectorThread.java | 3 +- .../bookie/DefaultEntryLogTest.java | 40 +++++++++++++++++++ 4 files changed, 47 insertions(+), 1 deletion(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/DefaultEntryLogger.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/DefaultEntryLogger.java index ab200b7ca0d..cfe7aac7936 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/DefaultEntryLogger.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/DefaultEntryLogger.java @@ -1152,6 +1152,10 @@ EntryLogMetadata extractEntryLogMetadataFromIndex(long entryLogId) throws IOExce + " -- found: " + meta.getLedgersMap().size() + " -- entryLogId: " + entryLogId); } + if (header.ledgersCount == 0) { + throw new IOException("No ledgers map found in entryLogId " + entryLogId + ", do scan to double confirm"); + } + return meta; } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLoggerAllocator.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLoggerAllocator.java index 68fc1eb3caf..e9ff5030d18 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLoggerAllocator.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLoggerAllocator.java @@ -81,6 +81,7 @@ class EntryLoggerAllocator { // within the same JVM. All of these Bookie instances access this header // so there can be race conditions when entry logs are rolled over and // this header buffer is cleared before writing it into the new logChannel. + logfileHeader.setZero(0, DefaultEntryLogger.LOGFILE_HEADER_SIZE); logfileHeader.writeBytes("BKLO".getBytes(UTF_8)); logfileHeader.writeInt(DefaultEntryLogger.HEADER_CURRENT_VERSION); logfileHeader.writerIndex(DefaultEntryLogger.LOGFILE_HEADER_SIZE); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java index 5abf79b2e64..d70082256f9 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java @@ -791,11 +791,12 @@ protected void extractMetaFromEntryLogs() throws EntryLogMetadataMapException { continue; } - LOG.info("Extracting entry log meta from entryLogId: {}", entryLogId); try { // Read through the entry log file and extract the entry log meta EntryLogMetadata entryLogMeta = entryLogger.getEntryLogMetadata(entryLogId, throttler); + LOG.info("Extracted entry log meta from entryLogId: {}, ledgers {}", + entryLogId, entryLogMeta.getLedgersMap().keys()); removeIfLedgerNotExists(entryLogMeta); if (entryLogMeta.isEmpty()) { // This means the entry log is not associated with any active diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/DefaultEntryLogTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/DefaultEntryLogTest.java index 38a9ebaf213..cbf12511126 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/DefaultEntryLogTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/DefaultEntryLogTest.java @@ -366,6 +366,46 @@ public void testRecoverFromLedgersMap() throws Exception { assertEquals(120, meta.getRemainingSize()); } + @Test + public void testLedgersMapIsEmpty() throws Exception { + // create some entries + entryLogger.addEntry(1L, generateEntry(1, 1).nioBuffer()); + entryLogger.addEntry(3L, generateEntry(3, 1).nioBuffer()); + entryLogger.addEntry(2L, generateEntry(2, 1).nioBuffer()); + entryLogger.addEntry(1L, generateEntry(1, 2).nioBuffer()); + ((EntryLogManagerBase) entryLogger.getEntryLogManager()).createNewLog(DefaultEntryLogger.UNASSIGNED_LEDGERID); + entryLogger.close(); + + // Rewrite the entry log header to be on V0 format + File f = new File(curDir, "0.log"); + RandomAccessFile raf = new RandomAccessFile(f, "rw"); + raf.seek(8); + // Mock that there is a ledgers map offset but the ledgers count is 0 + raf.writeLong(40); + raf.writeInt(0); + raf.close(); + + // now see which ledgers are in the log + entryLogger = new DefaultEntryLogger(conf, dirsMgr); + + try { + entryLogger.extractEntryLogMetadataFromIndex(0L); + fail("Should not be possible to recover from ledgers map index"); + } catch (IOException e) { + assertEquals("No ledgers map found in entryLogId 0, do scan to double confirm", e.getMessage()); + } + + // Public method should succeed by falling back to scanning the file + EntryLogMetadata meta = entryLogger.getEntryLogMetadata(0L); + LOG.info("Extracted Meta From Entry Log {}", meta); + assertEquals(60, meta.getLedgersMap().get(1L)); + assertEquals(30, meta.getLedgersMap().get(2L)); + assertEquals(30, meta.getLedgersMap().get(3L)); + assertFalse(meta.getLedgersMap().containsKey(4L)); + assertEquals(120, meta.getTotalSize()); + assertEquals(120, meta.getRemainingSize()); + } + /** * Explicitly try to recover using the ledgers map index at the end of the entry log. */ From 3c04410ef744ba2bf3cf7681eab97ad9326035f3 Mon Sep 17 00:00:00 2001 From: Shen Liu Date: Fri, 14 Feb 2025 16:19:47 +0800 Subject: [PATCH 35/55] Fix region aware placement policy disk weight dose not update. (#4522) * Fix region aware placement policy disk weight dose not update. (cherry picked from commit 55d274bbb3f4dc119db6f454aff8ae4022f3a74a) (cherry picked from commit f507b1c54a95610bc0874e4620105a98429374a9) --- .../RegionAwareEnsemblePlacementPolicy.java | 8 ++ ...estRegionAwareEnsemblePlacementPolicy.java | 97 +++++++++++++++++++ 2 files changed, 105 insertions(+) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RegionAwareEnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RegionAwareEnsemblePlacementPolicy.java index ef68e024672..eb8784a6219 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RegionAwareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RegionAwareEnsemblePlacementPolicy.java @@ -649,4 +649,12 @@ public PlacementPolicyAdherence isEnsembleAdheringToPlacementPolicy(List bookieInfoMap) { + super.updateBookieInfo(bookieInfoMap); + for (TopologyAwareEnsemblePlacementPolicy policy : perRegionPlacement.values()) { + policy.updateBookieInfo(bookieInfoMap); + } + } } diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRegionAwareEnsemblePlacementPolicy.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRegionAwareEnsemblePlacementPolicy.java index 9d8e36a350d..1f04ffa718b 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRegionAwareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRegionAwareEnsemblePlacementPolicy.java @@ -1780,6 +1780,103 @@ public void testRegionsWithDiskWeight() throws Exception { assertEquals(3, ensemble.size()); } + @Test + public void testRegionsWithDifferentDiskWeight() throws Exception { + repp.uninitalize(); + repp = new RegionAwareEnsemblePlacementPolicy(); + conf.setProperty(REPP_ENABLE_VALIDATION, false); + conf.setDiskWeightBasedPlacementEnabled(true); + repp.initialize(conf, Optional.empty(), timer, DISABLE_ALL, + NullStatsLogger.INSTANCE, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); + BookieSocketAddress addr1 = new BookieSocketAddress("127.0.0.2", 3181); + BookieSocketAddress addr2 = new BookieSocketAddress("127.0.0.3", 3181); + BookieSocketAddress addr3 = new BookieSocketAddress("127.0.0.4", 3181); + BookieSocketAddress addr4 = new BookieSocketAddress("127.0.0.5", 3181); + BookieSocketAddress addr5 = new BookieSocketAddress("127.0.0.6", 3181); + + // update dns mapping + StaticDNSResolver.addNodeToRack(addr1.getHostName(), "/region1/r1"); + StaticDNSResolver.addNodeToRack(addr2.getHostName(), "/region1/r1"); + StaticDNSResolver.addNodeToRack(addr3.getHostName(), "/region2/r2"); + StaticDNSResolver.addNodeToRack(addr4.getHostName(), "/region2/r2"); + StaticDNSResolver.addNodeToRack(addr5.getHostName(), "/region2/r2"); + // Update cluster + Set addrs = new HashSet<>(); + addrs.add(addr1.toBookieId()); + addrs.add(addr2.toBookieId()); + addrs.add(addr3.toBookieId()); + addrs.add(addr4.toBookieId()); + addrs.add(addr5.toBookieId()); + repp.onClusterChanged(addrs, new HashSet<>()); + + // update bookie weight + // due to default BookieMaxWeightMultipleForWeightBasedPlacement=3, the test cases need to be in the range + Map bookieInfoMap = new HashMap<>(); + bookieInfoMap.put(addr1.toBookieId(), new BookieInfoReader.BookieInfo(1000000, 800000)); + bookieInfoMap.put(addr2.toBookieId(), new BookieInfoReader.BookieInfo(1000000, 400000)); + bookieInfoMap.put(addr3.toBookieId(), new BookieInfoReader.BookieInfo(1000000, 200000)); + bookieInfoMap.put(addr4.toBookieId(), new BookieInfoReader.BookieInfo(1000000, 300000)); + bookieInfoMap.put(addr5.toBookieId(), new BookieInfoReader.BookieInfo(1000000, 400000)); + repp.updateBookieInfo(bookieInfoMap); + + List ensemble; + Map countMap = new HashMap<>(); + addrs.forEach(a -> countMap.put(a, 0)); + int loopTimes = 5000; + for (int i = 0; i < loopTimes; ++i) { + ensemble = repp.newEnsemble(2, 2, 2, null, + new HashSet<>()).getResult(); + for (BookieId bookieId : ensemble) { + countMap.put(bookieId, countMap.get(bookieId) + 1); + } + } + + // c1 should be 2x than c2 + // c4 should be 1.5x than c3 + // c5 should be 2x than c3 + // we allow a range of (-50%, 50%) deviation instead of the exact multiples + int c1, c2, c3, c4, c5; + c1 = countMap.get(addr1.toBookieId()); + c2 = countMap.get(addr2.toBookieId()); + c3 = countMap.get(addr3.toBookieId()); + c4 = countMap.get(addr4.toBookieId()); + c5 = countMap.get(addr5.toBookieId()); + assertTrue(Math.abs((double) c1 / c2 - 2.0) < 1.0); + assertTrue(Math.abs((double) c4 / c3 - 1.5) < 1.0); + assertTrue(Math.abs((double) c5 / c3 - 2.0) < 1.0); + + // update bookie weight + // due to default BookieMaxWeightMultipleForWeightBasedPlacement=3, the test cases need to be in the range + bookieInfoMap.put(addr1.toBookieId(), new BookieInfoReader.BookieInfo(1000000, 400000)); + bookieInfoMap.put(addr2.toBookieId(), new BookieInfoReader.BookieInfo(1000000, 800000)); + bookieInfoMap.put(addr3.toBookieId(), new BookieInfoReader.BookieInfo(1000000, 400000)); + bookieInfoMap.put(addr4.toBookieId(), new BookieInfoReader.BookieInfo(1000000, 300000)); + bookieInfoMap.put(addr5.toBookieId(), new BookieInfoReader.BookieInfo(1000000, 200000)); + repp.updateBookieInfo(bookieInfoMap); + + addrs.forEach(a -> countMap.put(a, 0)); + for (int i = 0; i < loopTimes; ++i) { + ensemble = repp.newEnsemble(2, 2, 2, null, + new HashSet<>()).getResult(); + for (BookieId bookieId : ensemble) { + countMap.put(bookieId, countMap.get(bookieId) + 1); + } + } + + // c2 should be 2x than c1 + // c4 should be 1.5x than c5 + // c3 should be 2x than c5 + // we allow a range of (-50%, 50%) deviation instead of the exact multiples + c1 = countMap.get(addr1.toBookieId()); + c2 = countMap.get(addr2.toBookieId()); + c3 = countMap.get(addr3.toBookieId()); + c4 = countMap.get(addr4.toBookieId()); + c5 = countMap.get(addr5.toBookieId()); + assertTrue(Math.abs((double) c2 / c1 - 2.0) < 1.0); + assertTrue(Math.abs((double) c4 / c5 - 1.5) < 1.0); + assertTrue(Math.abs((double) c3 / c5 - 2.0) < 1.0); + } + @Test public void testNotifyRackChangeWithOldRegion() throws Exception { BookieSocketAddress addr1 = new BookieSocketAddress("127.0.1.1", 3181); From 5df87ad82689bdf08267014ce629fbd29e365ca9 Mon Sep 17 00:00:00 2001 From: YingQun Zhong Date: Sun, 16 Feb 2025 07:17:49 +0800 Subject: [PATCH 36/55] remove in address2Region while bookie left to get correct rack info (#4504) (cherry picked from commit 07de650ea03612fb9b81647c2a55fd538d0cda3a) (cherry picked from commit 857959eb8c5cb3c1ceca2da5a36db26419a8a5f3) --- .../RegionAwareEnsemblePlacementPolicy.java | 17 +++ ...estRegionAwareEnsemblePlacementPolicy.java | 121 +++++++++++++++++- 2 files changed, 136 insertions(+), 2 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RegionAwareEnsemblePlacementPolicy.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RegionAwareEnsemblePlacementPolicy.java index eb8784a6219..bc882589b36 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RegionAwareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RegionAwareEnsemblePlacementPolicy.java @@ -117,6 +117,23 @@ protected String parseBookieRegion(BookieId addr) { @Override public void handleBookiesThatLeft(Set leftBookies) { + // case 1: (In some situation, eg, Broker and bookie restart concurrently.) + //1. Bookie X join cluster for the first time, encounters a region exception, and `address2Region` record X's + // region as default-region. + //2. Bookie X left cluster and is removed from knownBookies, but address2Region retains the information of + // bookie X. + //3. update Bookie X's rack info, and calling `onBookieRackChange` will only update address2Region for + // addresses present in knownBookies; therefore, bookie X's region info is not updated. + //4. Bookie X join cluster again, since address2Region contains the previous default-region information, + // getRegion will directly use cached data, resulting of an incorrect region. + + // The bookie region is initialized to "default-region" in address2Region. + // We should ensure that when a bookie leaves the cluster, + // we also clean up the corresponding region information for that bookie in address2Region, + // so that it can update the correct region for the bookie during onBookieRackChange and + // handleBookiesThatJoined. + // to avoid traffic skew in ensemble selection. + leftBookies.forEach(address2Region::remove); super.handleBookiesThatLeft(leftBookies); for (TopologyAwareEnsemblePlacementPolicy policy: perRegionPlacement.values()) { diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRegionAwareEnsemblePlacementPolicy.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRegionAwareEnsemblePlacementPolicy.java index 1f04ffa718b..ca6c074b33e 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRegionAwareEnsemblePlacementPolicy.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRegionAwareEnsemblePlacementPolicy.java @@ -1066,7 +1066,7 @@ public void testEnsembleWithThreeRegionsReplaceDisableDurability() throws Except testEnsembleWithThreeRegionsReplaceInternal(1, true, false); } - public void testEnsembleWithThreeRegionsReplaceInternal(int minDurability, boolean disableDurability, + private void testEnsembleWithThreeRegionsReplaceInternal(int minDurability, boolean disableDurability, boolean disableOneRegion) throws Exception { repp.uninitalize(); repp = new RegionAwareEnsemblePlacementPolicy(); @@ -1208,7 +1208,7 @@ public void testEnsembleDisableDurability() throws Exception { testEnsembleDurabilityDisabledInternal(2, true); } - public void testEnsembleDurabilityDisabledInternal(int minDurability, boolean disableDurability) throws Exception { + private void testEnsembleDurabilityDisabledInternal(int minDurability, boolean disableDurability) throws Exception { repp.uninitalize(); repp = new RegionAwareEnsemblePlacementPolicy(); conf.setProperty(REPP_REGIONS_TO_WRITE, "region1;region2;region3"); @@ -2027,4 +2027,121 @@ public void testNotifyRackChangeWithNewRegion() throws Exception { assertEquals("region2", repp.address2Region.get(addr3.toBookieId())); assertEquals("region3", repp.address2Region.get(addr4.toBookieId())); } + + @Test + public void testBookieLeftThenJoinWithDNSResolveFailed() throws Exception { + + BookieSocketAddress addr1 = new BookieSocketAddress("127.0.1.1", 3181); + BookieSocketAddress addr2 = new BookieSocketAddress("127.0.1.2", 3181); + BookieSocketAddress addr3 = new BookieSocketAddress("127.0.1.3", 3181); + BookieSocketAddress addr4 = new BookieSocketAddress("127.0.1.4", 3181); + + // init dns mapping + // 2. mock dns resolver failed, use default region and rack. + // addr1 rack info. /region-1/default-rack -> /default-region/default-rack. + + // 1. mock addr1 dns resolver failed and use default region and rack. + StaticDNSResolver.addNodeToRack(addr1.getHostName(), "/default-region/default-rack"); + StaticDNSResolver.addNodeToRack(addr2.getHostName(), "/region-1/default-rack"); + StaticDNSResolver.addNodeToRack(addr3.getHostName(), "/region-2/default-rack"); + StaticDNSResolver.addNodeToRack(addr4.getHostName(), "/region-3/default-rack"); + + // init cluster + Set addrs = Sets.newHashSet(addr1.toBookieId(), + addr2.toBookieId(), addr3.toBookieId(), addr4.toBookieId()); + repp.onClusterChanged(addrs, new HashSet<>()); + + assertEquals(4, repp.knownBookies.size()); + assertEquals("/default-region/default-rack", repp.knownBookies.get(addr1.toBookieId()).getNetworkLocation()); + assertEquals("/region-1/default-rack", repp.knownBookies.get(addr2.toBookieId()).getNetworkLocation()); + assertEquals("/region-2/default-rack", repp.knownBookies.get(addr3.toBookieId()).getNetworkLocation()); + assertEquals("/region-3/default-rack", repp.knownBookies.get(addr4.toBookieId()).getNetworkLocation()); + + assertEquals(4, repp.perRegionPlacement.size()); + TopologyAwareEnsemblePlacementPolicy unknownRegionPlacement = repp.perRegionPlacement.get("UnknownRegion"); + assertEquals(1, unknownRegionPlacement.knownBookies.keySet().size()); + assertEquals("/default-region/default-rack", + unknownRegionPlacement.knownBookies.get(addr1.toBookieId()).getNetworkLocation()); + + TopologyAwareEnsemblePlacementPolicy region1Placement = repp.perRegionPlacement.get("region-1"); + assertEquals(1, region1Placement.knownBookies.keySet().size()); + assertEquals("/region-1/default-rack", + region1Placement.knownBookies.get(addr2.toBookieId()).getNetworkLocation()); + + TopologyAwareEnsemblePlacementPolicy region2Placement = repp.perRegionPlacement.get("region-2"); + assertEquals(1, region2Placement.knownBookies.keySet().size()); + assertEquals("/region-2/default-rack", + region2Placement.knownBookies.get(addr3.toBookieId()).getNetworkLocation()); + + TopologyAwareEnsemblePlacementPolicy region3Placement = repp.perRegionPlacement.get("region-3"); + assertEquals(1, region3Placement.knownBookies.keySet().size()); + assertEquals("/region-3/default-rack", + region3Placement.knownBookies.get(addr4.toBookieId()).getNetworkLocation()); + + assertEquals("UnknownRegion", repp.address2Region.get(addr1.toBookieId())); + assertEquals("region-1", repp.address2Region.get(addr2.toBookieId())); + assertEquals("region-2", repp.address2Region.get(addr3.toBookieId())); + assertEquals("region-3", repp.address2Region.get(addr4.toBookieId())); + + // 2. addr1 bookie shutdown and decommission + addrs.remove(addr1.toBookieId()); + repp.onClusterChanged(addrs, new HashSet<>()); + + assertEquals(3, repp.knownBookies.size()); + assertNull(repp.knownBookies.get(addr1.toBookieId())); + assertEquals("/region-1/default-rack", repp.knownBookies.get(addr2.toBookieId()).getNetworkLocation()); + assertEquals("/region-2/default-rack", repp.knownBookies.get(addr3.toBookieId()).getNetworkLocation()); + assertEquals("/region-3/default-rack", repp.knownBookies.get(addr4.toBookieId()).getNetworkLocation()); + + // UnknownRegion,region-1,region-2,region-3 + assertEquals(4, repp.perRegionPlacement.size()); + // after addr1 bookie left, it should remove from locally address2Region + assertNull(repp.address2Region.get(addr1.toBookieId())); + assertEquals("region-1", repp.address2Region.get(addr2.toBookieId())); + assertEquals("region-2", repp.address2Region.get(addr3.toBookieId())); + assertEquals("region-3", repp.address2Region.get(addr4.toBookieId())); + + + // 3. addr1 bookie start and join + addrs.add(addr1.toBookieId()); + repp.onClusterChanged(addrs, new HashSet<>()); + + assertEquals(4, repp.knownBookies.size()); + assertEquals("/default-region/default-rack", repp.knownBookies.get(addr1.toBookieId()).getNetworkLocation()); + assertEquals("/region-1/default-rack", repp.knownBookies.get(addr2.toBookieId()).getNetworkLocation()); + assertEquals("/region-2/default-rack", repp.knownBookies.get(addr3.toBookieId()).getNetworkLocation()); + assertEquals("/region-3/default-rack", repp.knownBookies.get(addr4.toBookieId()).getNetworkLocation()); + + // UnknownRegion,region-1,region-2,region-3 + assertEquals(4, repp.perRegionPlacement.size()); + assertEquals("UnknownRegion", repp.address2Region.get(addr1.toBookieId())); + // addr1 bookie belongs to unknown region + unknownRegionPlacement = repp.perRegionPlacement.get("UnknownRegion"); + assertEquals(1, unknownRegionPlacement.knownBookies.keySet().size()); + assertEquals("/default-region/default-rack", + unknownRegionPlacement.knownBookies.get(addr1.toBookieId()).getNetworkLocation()); + + // 4. Update the correct rack. + // change addr1 rack info. /default-region/default-rack -> /region-1/default-rack. + List bookieAddressList = new ArrayList<>(); + List rackList = new ArrayList<>(); + bookieAddressList.add(addr1); + rackList.add("/region-1/default-rack"); + // onBookieRackChange + StaticDNSResolver.changeRack(bookieAddressList, rackList); + + assertEquals(4, repp.perRegionPlacement.size()); + // addr1 bookie, oldRegion=default-region, newRegion=region-1 + assertEquals("region-1", repp.address2Region.get(addr1.toBookieId())); + + unknownRegionPlacement = repp.perRegionPlacement.get("UnknownRegion"); + assertEquals(0, unknownRegionPlacement.knownBookies.keySet().size()); + assertNotNull(unknownRegionPlacement.historyBookies.get(addr1.toBookieId())); + + + region1Placement = repp.perRegionPlacement.get("region-1"); + assertEquals(2, region1Placement.knownBookies.keySet().size()); + assertEquals("/region-1/default-rack", + region1Placement.knownBookies.get(addr1.toBookieId()).getNetworkLocation()); + } } From bdfeab62530b5796fbb7073956974aa1e5fd91eb Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Wed, 14 May 2025 09:37:52 +0800 Subject: [PATCH 37/55] [fix] Write stuck due to pending add callback by multiple threads (#4557) [fix] Write stuck due to pending add callback by multiple threads (#4557) (cherry picked from commit e47926bc71c8f4b9fc548646b2abb46700923b37) (cherry picked from commit 804b2b611c6a11b42ca0307dd9803c16b9cc3915) --- .../bookkeeper/proto/BookieClientImpl.java | 30 ++-- .../proto/PerChannelBookieClient.java | 8 +- .../proto/ClientSocketDisconnectTest.java | 144 ++++++++++++++++++ 3 files changed, 171 insertions(+), 11 deletions(-) create mode 100644 bookkeeper-server/src/test/java/org/apache/bookkeeper/proto/ClientSocketDisconnectTest.java diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieClientImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieClientImpl.java index 5dde52bc3f5..1e5184d07e8 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieClientImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieClientImpl.java @@ -22,6 +22,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import com.google.protobuf.ExtensionRegistry; import io.netty.buffer.ByteBuf; @@ -355,7 +356,9 @@ private void completeRead(final int rc, } } - private static class ChannelReadyForAddEntryCallback + // Without test, this class should be modifier with "private". + @VisibleForTesting + static class ChannelReadyForAddEntryCallback implements GenericCallback { private final Handle recyclerHandle; @@ -365,7 +368,9 @@ private static class ChannelReadyForAddEntryCallback private long entryId; private BookieId addr; private Object ctx; - private WriteCallback cb; + // Without test, this class should be modifier with "private". + @VisibleForTesting + WriteCallback cb; private int options; private byte[] masterKey; private boolean allowFastFail; @@ -394,17 +399,24 @@ static ChannelReadyForAddEntryCallback create( @Override public void operationComplete(final int rc, PerChannelBookieClient pcbc) { - try { - if (rc != BKException.Code.OK) { - bookieClient.completeAdd(rc, ledgerId, entryId, addr, cb, ctx); - } else { + if (rc != BKException.Code.OK) { + bookieClient.executor.executeOrdered(ledgerId, () -> { + try { + bookieClient.completeAdd(rc, ledgerId, entryId, addr, cb, ctx); + } finally { + ReferenceCountUtil.release(toSend); + } + recycle(); + }); + } else { + try { pcbc.addEntry(ledgerId, masterKey, entryId, toSend, cb, ctx, options, allowFastFail, writeFlags); + } finally { + ReferenceCountUtil.release(toSend); } - } finally { - ReferenceCountUtil.release(toSend); + recycle(); } - recycle(); } private ChannelReadyForAddEntryCallback( diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java index a3475533f1a..80c3a1d9de6 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java @@ -615,11 +615,15 @@ protected void initChannel(Channel ch) throws Exception { } ChannelFuture future = bootstrap.connect(bookieAddr); - future.addListener(contextPreservingListener(new ConnectionFutureListener(startTime))); - future.addListener(x -> makeWritable()); + addChannelListeners(future, startTime); return future; } + protected void addChannelListeners(ChannelFuture future, long connectStartTime) { + future.addListener(contextPreservingListener(new ConnectionFutureListener(connectStartTime))); + future.addListener(x -> makeWritable()); + } + void cleanDisconnectAndClose() { disconnect(); close(); diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/proto/ClientSocketDisconnectTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/proto/ClientSocketDisconnectTest.java new file mode 100644 index 00000000000..2b4eb74397b --- /dev/null +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/proto/ClientSocketDisconnectTest.java @@ -0,0 +1,144 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.bookkeeper.proto; + +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; +import io.netty.util.concurrent.DefaultThreadFactory; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.client.api.BookKeeper; +import org.apache.bookkeeper.client.api.DigestType; +import org.apache.bookkeeper.conf.ClientConfiguration; +import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.test.BookKeeperClusterTestCase; +import org.apache.bookkeeper.tls.SecurityException; +import org.apache.bookkeeper.util.EventLoopUtil; +import org.junit.Assert; +import org.junit.jupiter.api.Test; + +@Slf4j +public class ClientSocketDisconnectTest extends BookKeeperClusterTestCase { + + public ClientSocketDisconnectTest() { + super(1); + this.useUUIDasBookieId = true; + } + + public static class PerChannelBookieClientDecorator extends PerChannelBookieClient { + + private final ThreadCounter threadCounter; + private final AtomicInteger failurePredicate = new AtomicInteger(); + + public PerChannelBookieClientDecorator(PerChannelBookieClient client, BookieId addr, ThreadCounter tCounter) + throws SecurityException { + super(client.executor, client.eventLoopGroup, addr, client.bookieAddressResolver); + this.threadCounter = tCounter; + } + + // Inject a disconnection per two connections. + protected void addChannelListeners(ChannelFuture future, long connectStartTime) { + future.addListener((ChannelFutureListener) future1 -> { + if (failurePredicate.incrementAndGet() % 2 == 1) { + future1.channel().close(); + } + }); + super.addChannelListeners(future, connectStartTime); + } + + // Records the thread who running "PendingAddOp.writeComplete". + @Override + protected void connectIfNeededAndDoOp(BookkeeperInternalCallbacks.GenericCallback op) { + BookieClientImpl.ChannelReadyForAddEntryCallback callback = + (BookieClientImpl.ChannelReadyForAddEntryCallback) op; + BookkeeperInternalCallbacks.WriteCallback originalCallback = callback.cb; + callback.cb = (rc, ledgerId, entryId, addr, ctx) -> { + threadCounter.record(); + originalCallback.writeComplete(rc, ledgerId, entryId, addr, ctx); + }; + super.connectIfNeededAndDoOp(op); + } + } + + private static class ThreadCounter { + + private final Map records = new ConcurrentHashMap<>(); + + public void record() { + Thread currentThread = Thread.currentThread(); + records.computeIfAbsent(currentThread, k -> new AtomicInteger()); + records.get(currentThread).incrementAndGet(); + } + } + + @Test + public void testAddEntriesCallbackWithBKClientThread() throws Exception { + // Create BKC and a ledger handle. + ClientConfiguration conf = new ClientConfiguration(); + conf.setMetadataServiceUri(zkUtil.getMetadataServiceUri()); + org.apache.bookkeeper.client.BookKeeper bkc = + (org.apache.bookkeeper.client.BookKeeper) BookKeeper.newBuilder(conf) + .eventLoopGroup( + EventLoopUtil.getClientEventLoopGroup(conf, new DefaultThreadFactory("test-io"))) + .build(); + final BookieClientImpl bookieClient = (BookieClientImpl) bkc.getClientCtx().getBookieClient(); + LedgerHandle lh = (LedgerHandle) bkc.newCreateLedgerOp() + .withEnsembleSize(1) + .withWriteQuorumSize(1) + .withAckQuorumSize(1) + .withDigestType(DigestType.CRC32C) + .withPassword(new byte[0]) + .execute().join(); + + // Inject two operations. + // 1. Inject a disconnection when connecting successfully. + // 2. Records the thread who running "PendingAddOp.writeComplete". + final ThreadCounter callbackThreadRecorder = new ThreadCounter(); + List ensemble = lh.getLedgerMetadata() + .getAllEnsembles().entrySet().iterator().next().getValue(); + DefaultPerChannelBookieClientPool clientPool = + (DefaultPerChannelBookieClientPool) bookieClient.lookupClient(ensemble.get(0)); + PerChannelBookieClient[] clients = clientPool.clients; + + // Write 100 entries and wait for finishing. + for (int i = 0; i < clients.length; i++) { + clients[i] = new PerChannelBookieClientDecorator(clients[i], ensemble.get(0), callbackThreadRecorder); + } + int addCount = 1000; + CountDownLatch countDownLatch = new CountDownLatch(addCount); + for (int i = 0; i < addCount; i++) { + lh.asyncAddEntry(new byte[]{1}, (rc, lh1, entryId, ctx) -> { + countDownLatch.countDown(); + }, i); + } + countDownLatch.await(); + + // Verify: all callback will run in the "BookKeeperClientWorker" thread. + for (Thread callbackThread : callbackThreadRecorder.records.keySet()) { + Assert.assertTrue(callbackThread.getName(), callbackThread.getName().startsWith("BookKeeperClientWorker")); + } + } +} \ No newline at end of file From 06e144d6c770b2d55f88d20d3257fce1071daa3a Mon Sep 17 00:00:00 2001 From: Hang Chen Date: Wed, 4 Jun 2025 18:46:27 -0700 Subject: [PATCH 38/55] fix branch-4.16 ci (#4612) (cherry picked from commit 3bceb37b41048512bf0db3a0f84253946eb2a308) --- .../org/apache/bookkeeper/proto/ClientSocketDisconnectTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/proto/ClientSocketDisconnectTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/proto/ClientSocketDisconnectTest.java index 2b4eb74397b..709ed771ddc 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/proto/ClientSocketDisconnectTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/proto/ClientSocketDisconnectTest.java @@ -38,7 +38,7 @@ import org.apache.bookkeeper.tls.SecurityException; import org.apache.bookkeeper.util.EventLoopUtil; import org.junit.Assert; -import org.junit.jupiter.api.Test; +import org.junit.Test; @Slf4j public class ClientSocketDisconnectTest extends BookKeeperClusterTestCase { From 5950a411bbdc39cdc271ba2e8c8f3ebcff857548 Mon Sep 17 00:00:00 2001 From: ken <1647023764@qq.com> Date: Fri, 14 Feb 2025 14:15:32 +0800 Subject: [PATCH 39/55] fix pendingDeletedLedgers do not remove ledger error (#4525) * fix pendingDeletedLedgers not remove ledger (cherry picked from commit 0df3caf530dacc41a48fa45f0ebfd8efedf81f95) (cherry picked from commit 0d31202037b81b44fc1f290eef3319ac86b02740) --- .../storage/ldb/LedgerMetadataIndex.java | 1 + .../storage/ldb/DbLedgerStorageTest.java | 59 +++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LedgerMetadataIndex.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LedgerMetadataIndex.java index 9b50479b7d5..edc816ce4b9 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LedgerMetadataIndex.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LedgerMetadataIndex.java @@ -336,6 +336,7 @@ public void removeDeletedLedgers() throws IOException { key.set(ledgerId); ledgersDb.delete(key.array); ++deletedLedgers; + pendingDeletedLedgers.remove(ledgerId); } if (log.isDebugEnabled()) { diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageTest.java index 65f11e5d6a3..dfc2459678b 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageTest.java @@ -33,8 +33,10 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.util.List; +import java.util.Set; import org.apache.bookkeeper.bookie.Bookie; import org.apache.bookkeeper.bookie.Bookie.NoEntryException; import org.apache.bookkeeper.bookie.BookieException; @@ -52,6 +54,7 @@ import org.apache.bookkeeper.conf.TestBKConfiguration; import org.apache.bookkeeper.proto.BookieProtocol; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; @@ -822,4 +825,60 @@ public void testSingleLedgerDirectoryCheckpoint() throws Exception { assertEquals(7, logMark.getLogFileId()); assertEquals(8, logMark.getLogFileOffset()); } + + @Test + public void testSingleLedgerDirectoryCheckpointTriggerRemovePendingDeletedLedgers() + throws Exception { + int gcWaitTime = 1000; + File ledgerDir = new File(tmpDir, "dir"); + ServerConfiguration conf = TestBKConfiguration.newServerConfiguration(); + conf.setGcWaitTime(gcWaitTime); + conf.setProperty(DbLedgerStorage.WRITE_CACHE_MAX_SIZE_MB, 4); + conf.setProperty(DbLedgerStorage.READ_AHEAD_CACHE_MAX_SIZE_MB, 4); + conf.setLedgerStorageClass(DbLedgerStorage.class.getName()); + conf.setLedgerDirNames(new String[] { ledgerDir.getCanonicalPath() }); + + BookieImpl bookie = new TestBookieImpl(conf); + ByteBuf entry1 = Unpooled.buffer(1024); + entry1.writeLong(1); // ledger id + entry1.writeLong(2); // entry id + entry1.writeBytes("entry-1".getBytes()); + bookie.getLedgerStorage().addEntry(entry1); + + bookie.getJournals().get(0).getLastLogMark().getCurMark().setLogMark(1, 2); + ((DbLedgerStorage) bookie.getLedgerStorage()).getLedgerStorageList().get(0).flush(); + + File ledgerDirMark = new File(ledgerDir + "/current", "lastMark"); + try { + LogMark logMark = readLogMark(ledgerDirMark); + assertEquals(1, logMark.getLogFileId()); + assertEquals(2, logMark.getLogFileOffset()); + } catch (Exception e) { + fail(); + } + + ByteBuf entry2 = Unpooled.buffer(1024); + entry2.writeLong(2); // ledger id + entry2.writeLong(1); // entry id + entry2.writeBytes("entry-2".getBytes()); + + bookie.getLedgerStorage().addEntry(entry2); + // write one entry to first ledger directory and flush with logMark(1, 2), + // only the first ledger directory should have lastMark + bookie.getJournals().get(0).getLastLogMark().getCurMark().setLogMark(4, 5); + + SingleDirectoryDbLedgerStorage storage1 = + ((DbLedgerStorage) bookie.getLedgerStorage()).getLedgerStorageList().get(0); + Field field = SingleDirectoryDbLedgerStorage.class.getDeclaredField("ledgerIndex"); + field.setAccessible(true); + LedgerMetadataIndex ledgerMetadataIndex = (LedgerMetadataIndex) field.get(storage1); + Field field1 = LedgerMetadataIndex.class.getDeclaredField("pendingDeletedLedgers"); + field1.setAccessible(true); + Set pendingDeletedLedgers = (Set) field1.get(ledgerMetadataIndex); + + Assert.assertEquals(pendingDeletedLedgers.size(), 0); + pendingDeletedLedgers.add(2L); + bookie.getLedgerStorage().flush(); + Assert.assertEquals(pendingDeletedLedgers.size(), 0); + } } From 196bc1f631695fd3b0817eb528623a7396ca6b0e Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 22 Apr 2025 22:37:55 +0300 Subject: [PATCH 40/55] Adjust DNS cache expiration in tests to reduce test timeout failures (#4586) (cherry picked from commit 7fbb8d89f880579035b5ba2752caf8583d34611a) (cherry picked from commit 065b9feb94838377f85157809a535eb470c04975) --- .github/workflows/bk-ci.yml | 14 ++++++++++++++ pom.xml | 6 +++--- stream/distributedlog/common/pom.xml | 2 +- stream/distributedlog/core/pom.xml | 2 +- stream/distributedlog/pom.xml | 2 +- stream/pom.xml | 2 +- tests/integration-tests-base-groovy/pom.xml | 2 +- 7 files changed, 22 insertions(+), 8 deletions(-) diff --git a/.github/workflows/bk-ci.yml b/.github/workflows/bk-ci.yml index 9a8af608bb5..ca8a3addf41 100644 --- a/.github/workflows/bk-ci.yml +++ b/.github/workflows/bk-ci.yml @@ -157,6 +157,13 @@ jobs: distribution: 'temurin' java-version: 11 + - name: Tune Java DNS TTL settings + run: | + sudo tee -a $JAVA_HOME/conf/security/java.security <org.apache.maven.plugins maven-surefire-plugin - -Xmx2G -Djava.net.preferIPv4Stack=true -Dio.netty.leakDetection.level=paranoid ${test.additional.args} + -Xmx2G -Dsun.net.inetaddr.ttl=1 -Dsun.net.inetaddr.negative.ttl=1 -Djava.net.preferIPv4Stack=true -Dio.netty.leakDetection.level=paranoid ${test.additional.args} ${redirectTestOutputToFile} ${forkCount.variable} false @@ -1297,7 +1297,7 @@ org.apache.maven.plugins maven-surefire-plugin - -Xmx2G -Djava.net.preferIPv4Stack=true + -Xmx2G -Dsun.net.inetaddr.ttl=1 -Dsun.net.inetaddr.negative.ttl=1 -Djava.net.preferIPv4Stack=true false ${forkCount.variable} false @@ -1317,7 +1317,7 @@ org.apache.maven.plugins maven-surefire-plugin - -Xmx2G -Djava.net.preferIPv4Stack=true -Dio.netty.leakDetection.level=paranoid -Dbookkeeper.log.root.level=INFO -Dbookkeeper.log.root.appender=CONSOLE + -Xmx2G -Dsun.net.inetaddr.ttl=1 -Dsun.net.inetaddr.negative.ttl=1 -Djava.net.preferIPv4Stack=true -Dio.netty.leakDetection.level=paranoid -Dbookkeeper.log.root.level=INFO -Dbookkeeper.log.root.appender=CONSOLE false ${forkCount.variable} false diff --git a/stream/distributedlog/common/pom.xml b/stream/distributedlog/common/pom.xml index 58885fbb8e0..a5ab3cbca3e 100644 --- a/stream/distributedlog/common/pom.xml +++ b/stream/distributedlog/common/pom.xml @@ -109,7 +109,7 @@ maven-surefire-plugin ${redirectTestOutputToFile} - -Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G ${test.additional.args} + -Xmx3G -Dsun.net.inetaddr.ttl=1 -Dsun.net.inetaddr.negative.ttl=1 -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G ${test.additional.args} 1 false 1800 diff --git a/stream/distributedlog/core/pom.xml b/stream/distributedlog/core/pom.xml index c6c183c19f4..beae8e20bbb 100644 --- a/stream/distributedlog/core/pom.xml +++ b/stream/distributedlog/core/pom.xml @@ -110,7 +110,7 @@ false ${redirectTestOutputToFile} - -Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G ${test.additional.args} + -Xmx3G -Dsun.net.inetaddr.ttl=1 -Dsun.net.inetaddr.negative.ttl=1 -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G ${test.additional.args} 1 false 1800 diff --git a/stream/distributedlog/pom.xml b/stream/distributedlog/pom.xml index 193476682a5..80cfa966381 100644 --- a/stream/distributedlog/pom.xml +++ b/stream/distributedlog/pom.xml @@ -75,7 +75,7 @@ maven-surefire-plugin ${redirectTestOutputToFile} - -Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G -Dio.netty.leakDetection.level=PARANOID ${test.additional.args} + -Xmx3G -Dsun.net.inetaddr.ttl=1 -Dsun.net.inetaddr.negative.ttl=1 -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G -Dio.netty.leakDetection.level=PARANOID ${test.additional.args} 1 false 1800 diff --git a/stream/pom.xml b/stream/pom.xml index 94af8ee3f85..192dff90fa8 100644 --- a/stream/pom.xml +++ b/stream/pom.xml @@ -57,7 +57,7 @@ true ${redirectTestOutputToFile} - -Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G -Dio.netty.leakDetection.level=PARANOID ${test.additional.args} + -Xmx3G -Dsun.net.inetaddr.ttl=1 -Dsun.net.inetaddr.negative.ttl=1 -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G -Dio.netty.leakDetection.level=PARANOID ${test.additional.args} 1 false 1800 diff --git a/tests/integration-tests-base-groovy/pom.xml b/tests/integration-tests-base-groovy/pom.xml index 5e1919e236d..a56950b4213 100644 --- a/tests/integration-tests-base-groovy/pom.xml +++ b/tests/integration-tests-base-groovy/pom.xml @@ -68,7 +68,7 @@ org.apache.maven.plugins maven-surefire-plugin - -Xmx4G -Djava.net.preferIPv4Stack=true ${test.additional.args} + -Xmx4G -Dsun.net.inetaddr.ttl=1 -Dsun.net.inetaddr.negative.ttl=1 -Djava.net.preferIPv4Stack=true ${test.additional.args} 1 false From 83c6aee46a689bb21b7db1ff127f42a8c6b4707f Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 6 May 2025 10:39:19 +0800 Subject: [PATCH 41/55] Upgrade Jetty to 9.4.57.v20241219 to address CVE-2024-6763 (#4600) ### Motivation & Changes Upgrade Jetty to 9.4.57.v20241219 to address CVE-2024-6763 Jetty 9.4.57.v20241219 contains backported CVE-2024-6763 fix in https://github.com/jetty/jetty.project/pull/12532 although it's not explicitly mentioned and most security scanners don't yet contain the information that it's been addressed in 9.4.57. More details: * https://github.com/jetty/jetty.project/issues/12630 * https://github.com/jetty/jetty.project/releases/tag/jetty-9.4.57.v20241219 Note: The backport is a partial mitigation and Jetty 9.4.57 will continue to be marked as vulnerable. There's a discussion and explanation here: https://gitlab.eclipse.org/security/cve-assignement/-/issues/25#note_2968611 (cherry picked from commit 99eb63a5a417e9fdd7a7df4a5974491d51893c8e) (cherry picked from commit cda3c6b684f95158d238ba121f0770c616103b65) --- .../src/main/resources/LICENSE-all.bin.txt | 16 ++++++++-------- .../src/main/resources/LICENSE-server.bin.txt | 16 ++++++++-------- .../src/main/resources/NOTICE-all.bin.txt | 16 ++++++++-------- .../src/main/resources/NOTICE-server.bin.txt | 16 ++++++++-------- pom.xml | 2 +- 5 files changed, 33 insertions(+), 33 deletions(-) diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index 584308becdc..457888bebe0 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -264,14 +264,14 @@ Apache Software License, Version 2. - lib/org.apache.zookeeper-zookeeper-3.9.3.jar [21] - lib/org.apache.zookeeper-zookeeper-jute-3.9.3.jar [21] - lib/org.apache.zookeeper-zookeeper-3.9.3-tests.jar [21] -- lib/org.eclipse.jetty-jetty-http-9.4.53.v20231009.jar [22] -- lib/org.eclipse.jetty-jetty-io-9.4.53.v20231009.jar [22] -- lib/org.eclipse.jetty-jetty-security-9.4.53.v20231009.jar [22] -- lib/org.eclipse.jetty-jetty-server-9.4.53.v20231009.jar [22] -- lib/org.eclipse.jetty-jetty-servlet-9.4.53.v20231009.jar [22] -- lib/org.eclipse.jetty-jetty-util-9.4.53.v20231009.jar [22] -- lib/org.eclipse.jetty-jetty-util-ajax-9.4.53.v20231009.jar [22] -- lib/org.rocksdb-rocksdbjni-7.9.2.jar [23] +- lib/org.eclipse.jetty-jetty-http-9.4.57.v20241219.jar [22] +- lib/org.eclipse.jetty-jetty-io-9.4.57.v20241219.jar [22] +- lib/org.eclipse.jetty-jetty-security-9.4.57.v20241219.jar [22] +- lib/org.eclipse.jetty-jetty-server-9.4.57.v20241219.jar [22] +- lib/org.eclipse.jetty-jetty-servlet-9.4.57.v20241219.jar [22] +- lib/org.eclipse.jetty-jetty-util-9.4.57.v20241219.jar [22] +- lib/org.eclipse.jetty-jetty-util-ajax-9.4.57.v20241219.jar [22] +- lib/org.rocksdb-rocksdbjni-9.9.3.jar [23] - lib/com.beust-jcommander-1.82.jar [24] - lib/com.yahoo.datasketches-memory-0.8.3.jar [25] - lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25] diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt index d0bc4b0a369..e5ac87b75f7 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt @@ -264,14 +264,14 @@ Apache Software License, Version 2. - lib/org.apache.zookeeper-zookeeper-3.9.3.jar [21] - lib/org.apache.zookeeper-zookeeper-jute-3.9.3.jar [21] - lib/org.apache.zookeeper-zookeeper-3.9.3-tests.jar [21] -- lib/org.eclipse.jetty-jetty-http-9.4.53.v20231009.jar [22] -- lib/org.eclipse.jetty-jetty-io-9.4.53.v20231009.jar [22] -- lib/org.eclipse.jetty-jetty-security-9.4.53.v20231009.jar [22] -- lib/org.eclipse.jetty-jetty-server-9.4.53.v20231009.jar [22] -- lib/org.eclipse.jetty-jetty-servlet-9.4.53.v20231009.jar [22] -- lib/org.eclipse.jetty-jetty-util-9.4.53.v20231009.jar [22] -- lib/org.eclipse.jetty-jetty-util-ajax-9.4.53.v20231009.jar [22] -- lib/org.rocksdb-rocksdbjni-7.9.2.jar [23] +- lib/org.eclipse.jetty-jetty-http-9.4.57.v20241219.jar [22] +- lib/org.eclipse.jetty-jetty-io-9.4.57.v20241219.jar [22] +- lib/org.eclipse.jetty-jetty-security-9.4.57.v20241219.jar [22] +- lib/org.eclipse.jetty-jetty-server-9.4.57.v20241219.jar [22] +- lib/org.eclipse.jetty-jetty-servlet-9.4.57.v20241219.jar [22] +- lib/org.eclipse.jetty-jetty-util-9.4.57.v20241219.jar [22] +- lib/org.eclipse.jetty-jetty-util-ajax-9.4.57.v20241219.jar [22] +- lib/org.rocksdb-rocksdbjni-9.9.3.jar [23] - lib/com.beust-jcommander-1.82.jar [24] - lib/com.yahoo.datasketches-memory-0.8.3.jar [25] - lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25] diff --git a/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt index d8e2d64dea2..168d4a1ff59 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt @@ -93,13 +93,13 @@ SoundCloud Ltd. (http://soundcloud.com/). This product includes software developed as part of the Ocelli project by Netflix Inc. (https://github.com/Netflix/ocelli/). ------------------------------------------------------------------------------------ -- lib/org.eclipse.jetty-jetty-http-9.4.53.v20231009.jar -- lib/org.eclipse.jetty-jetty-io-9.4.53.v20231009.jar -- lib/org.eclipse.jetty-jetty-security-9.4.53.v20231009.jar -- lib/org.eclipse.jetty-jetty-server-9.4.53.v20231009.jar -- lib/org.eclipse.jetty-jetty-servlet-9.4.53.v20231009.jar -- lib/org.eclipse.jetty-jetty-util-9.4.53.v20231009.jar -- lib/org.eclipse.jetty-jetty-util-ajax-9.4.53.v20231009.jar +- lib/org.eclipse.jetty-jetty-http-9.4.57.v20241219.jar +- lib/org.eclipse.jetty-jetty-io-9.4.57.v20241219.jar +- lib/org.eclipse.jetty-jetty-security-9.4.57.v20241219.jar +- lib/org.eclipse.jetty-jetty-server-9.4.57.v20241219.jar +- lib/org.eclipse.jetty-jetty-servlet-9.4.57.v20241219.jar +- lib/org.eclipse.jetty-jetty-util-9.4.57.v20241219.jar +- lib/org.eclipse.jetty-jetty-util-ajax-9.4.57.v20241219.jar ============================================================== Jetty Web Container @@ -121,7 +121,7 @@ Jetty is dual licensed under both Jetty may be distributed under either license. -lib/org.eclipse.jetty-jetty-util-9.4.53.v20231009.jar bundles UnixCrypt +lib/org.eclipse.jetty-jetty-util-9.4.57.v20241219.jar bundles UnixCrypt The UnixCrypt.java code implements the one way cryptography used by Unix systems for simple password protection. Copyright 1996 Aki Yoshida, diff --git a/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt index 488aad4530a..210718ee9fe 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt @@ -75,13 +75,13 @@ SoundCloud Ltd. (http://soundcloud.com/). This product includes software developed as part of the Ocelli project by Netflix Inc. (https://github.com/Netflix/ocelli/). ------------------------------------------------------------------------------------ -- lib/org.eclipse.jetty-jetty-http-9.4.53.v20231009.jar -- lib/org.eclipse.jetty-jetty-io-9.4.53.v20231009.jar -- lib/org.eclipse.jetty-jetty-security-9.4.53.v20231009.jar -- lib/org.eclipse.jetty-jetty-server-9.4.53.v20231009.jar -- lib/org.eclipse.jetty-jetty-servlet-9.4.53.v20231009.jar -- lib/org.eclipse.jetty-jetty-util-9.4.53.v20231009.jar -- lib/org.eclipse.jetty-jetty-util-ajax-9.4.53.v20231009.jar +- lib/org.eclipse.jetty-jetty-http-9.4.57.v20241219.jar +- lib/org.eclipse.jetty-jetty-io-9.4.57.v20241219.jar +- lib/org.eclipse.jetty-jetty-security-9.4.57.v20241219.jar +- lib/org.eclipse.jetty-jetty-server-9.4.57.v20241219.jar +- lib/org.eclipse.jetty-jetty-servlet-9.4.57.v20241219.jar +- lib/org.eclipse.jetty-jetty-util-9.4.57.v20241219.jar +- lib/org.eclipse.jetty-jetty-util-ajax-9.4.57.v20241219.jar ============================================================== Jetty Web Container @@ -103,7 +103,7 @@ Jetty is dual licensed under both Jetty may be distributed under either license. -lib/org.eclipse.jetty-jetty-util-9.4.53.v20231009.jar bundles UnixCrypt +lib/org.eclipse.jetty-jetty-util-9.4.57.v20241219.jar bundles UnixCrypt The UnixCrypt.java code implements the one way cryptography used by Unix systems for simple password protection. Copyright 1996 Aki Yoshida, diff --git a/pom.xml b/pom.xml index 0d5bac6e624..61ba5ba1f92 100644 --- a/pom.xml +++ b/pom.xml @@ -140,7 +140,7 @@ 2.1.10 2.17.1 1.82 - 9.4.53.v20231009 + 9.4.57.v20241219 1.37 2.8.2 1.14.3 From f767b3b843e2e31ae45bf83f0a3dc25b29dabb97 Mon Sep 17 00:00:00 2001 From: lushiji Date: Wed, 18 Jun 2025 09:55:46 +0800 Subject: [PATCH 42/55] Update python version to 4.16.7 (cherry picked from commit 8e0d75ac36ba9132b2c0b33920a1954971d0e6ee) --- stream/clients/python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stream/clients/python/setup.py b/stream/clients/python/setup.py index c8c17ce200b..b5f214960cb 100644 --- a/stream/clients/python/setup.py +++ b/stream/clients/python/setup.py @@ -19,7 +19,7 @@ name = 'apache-bookkeeper-client' description = 'Apache BookKeeper client library' -version = '4.16.6' +version = '4.16.7' # Should be one of: # 'Development Status :: 3 - Alpha' # 'Development Status :: 4 - Beta' From ba65055d704590e9c35f91175ef066d1731a3059 Mon Sep 17 00:00:00 2001 From: StevenLuMT Date: Mon, 7 Jul 2025 09:28:34 +0800 Subject: [PATCH 43/55] fix cherry-pick: Upgrade Jetty to 9.4.57.v20241219 to address CVE-2024-6763 (#4600) (#4631) (cherry picked from commit add680828de960793163e38a9cec0dd2c9ca63ae) --- bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt | 2 +- bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index 457888bebe0..ccf7b55bf1a 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -271,7 +271,7 @@ Apache Software License, Version 2. - lib/org.eclipse.jetty-jetty-servlet-9.4.57.v20241219.jar [22] - lib/org.eclipse.jetty-jetty-util-9.4.57.v20241219.jar [22] - lib/org.eclipse.jetty-jetty-util-ajax-9.4.57.v20241219.jar [22] -- lib/org.rocksdb-rocksdbjni-9.9.3.jar [23] +- lib/org.rocksdb-rocksdbjni-7.9.2.jar [23] - lib/com.beust-jcommander-1.82.jar [24] - lib/com.yahoo.datasketches-memory-0.8.3.jar [25] - lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25] diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt index e5ac87b75f7..61a176243ad 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt @@ -271,7 +271,7 @@ Apache Software License, Version 2. - lib/org.eclipse.jetty-jetty-servlet-9.4.57.v20241219.jar [22] - lib/org.eclipse.jetty-jetty-util-9.4.57.v20241219.jar [22] - lib/org.eclipse.jetty-jetty-util-ajax-9.4.57.v20241219.jar [22] -- lib/org.rocksdb-rocksdbjni-9.9.3.jar [23] +- lib/org.rocksdb-rocksdbjni-7.9.2.jar [23] - lib/com.beust-jcommander-1.82.jar [24] - lib/com.yahoo.datasketches-memory-0.8.3.jar [25] - lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25] From f6ea528f792379b5652a4fd9548a457ff22296a2 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Thu, 13 Feb 2025 11:41:21 -0800 Subject: [PATCH 44/55] Fix: Garbage collection stuck on corrupt entry log file (#4544) (cherry picked from commit 175e29407ece25fbb50d52afbcca384f2182e058) (cherry picked from commit 7733db1c9a673371301d585e3d0e35c44d4110f9) --- .../apache/bookkeeper/bookie/GarbageCollectorThread.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java index d70082256f9..41eb1d941f3 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java @@ -809,8 +809,13 @@ protected void extractMetaFromEntryLogs() throws EntryLogMetadataMapException { entryLogMetaMap.put(entryLogId, entryLogMeta); } } catch (IOException | RuntimeException e) { - LOG.warn("Premature exception when processing " + entryLogId - + " recovery will take care of the problem", e); + LOG.warn("Premature exception when processing {} recovery will take care of the problem", + entryLogId, e); + } catch (OutOfMemoryError oome) { + // somewhat similar to https://github.com/apache/bookkeeper/pull/3901 + // entrylog file can be corrupted but instead having a negative entry size + // it ends up with very large value for the entry size causing OODME + LOG.warn("OutOfMemoryError when processing {} - skipping the entry log", entryLogId, oome); } } } From f09de9f9b44381fb30c6c6ac0086877dc941f883 Mon Sep 17 00:00:00 2001 From: Yong Zhang Date: Tue, 12 Aug 2025 10:18:44 +0800 Subject: [PATCH 45/55] Log all the error in the GarbageCollectorThread (#4649) ### Motivation We met the GarbageCollectionThread was stopped by some runtime error, but we didn't catch it then, causing the GC to stop. Such as: https://github.com/apache/bookkeeper/pull/3901 https://github.com/apache/bookkeeper/pull/4544 In our case, the GC stopped because of the OutOfDirectMemoryException then the process stopped and the files can not be deleted. But we didn't see any error logs. This PR enhance the log info when an unhandled error happens. We already have the [PR](https://github.com/apache/bookkeeper/pull/4544) fixed that. And another fix in this PR is to change the Exception to the Throwable in the getEntryLogMetadata. Here is the error stack: ``` io.netty.util.internal.OutOfDirectMemoryError: failed to allocate 213909504 byte(s) of direct memory (used: 645922847, max: 858783744) at io.netty.util.internal.PlatformDependent.incrementMemoryCounter(PlatformDependent.java:880) at io.netty.util.internal.PlatformDependent.allocateDirectNoCleaner(PlatformDependent.java:809) at io.netty.buffer.PoolArena$DirectArena.allocateDirect(PoolArena.java:718) at io.netty.buffer.PoolArena$DirectArena.newUnpooledChunk(PoolArena.java:707) at io.netty.buffer.PoolArena.allocateHuge(PoolArena.java:224) at io.netty.buffer.PoolArena.allocate(PoolArena.java:142) at io.netty.buffer.PoolArena.reallocate(PoolArena.java:317) at io.netty.buffer.PooledByteBuf.capacity(PooledByteBuf.java:123) at io.netty.buffer.AbstractByteBuf.ensureWritable0(AbstractByteBuf.java:305) at io.netty.buffer.AbstractByteBuf.ensureWritable(AbstractByteBuf.java:280) at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1103) at org.apache.bookkeeper.bookie.BufferedReadChannel.read(BufferedReadChannel.java:104) at org.apache.bookkeeper.bookie.DefaultEntryLogger.extractEntryLogMetadataFromIndex(DefaultEntryLogger.java:1109) at org.apache.bookkeeper.bookie.DefaultEntryLogger.getEntryLogMetadata(DefaultEntryLogger.java:1060) at org.apache.bookkeeper.bookie.GarbageCollectorThread.extractMetaFromEntryLogs(GarbageCollectorThread.java:678) at org.apache.bookkeeper.bookie.GarbageCollectorThread.runWithFlags(GarbageCollectorThread.java:365) at org.apache.bookkeeper.bookie.GarbageCollectorThread.lambda$triggerGC$4(GarbageCollectorThread.java:268) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.run(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Unknown Source) ``` You can see it get much more memory used here extractEntryLogMetadataFromIndex(DefaultEntryLogger.java:1109). The reason is that the header has the wrong data of the header, which should already be fixed by https://github.com/apache/bookkeeper/pull/4607. Then it reading with a wrong map size which could take a lot of memory. (cherry picked from commit e80d0318cfdebbc79f37d9120de8df28c8c1c13a) (cherry picked from commit 35b8f137b49e730efc3c1c0d5e294d2ff827bd42) --- .../java/org/apache/bookkeeper/bookie/DefaultEntryLogger.java | 2 +- .../org/apache/bookkeeper/bookie/GarbageCollectorThread.java | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/DefaultEntryLogger.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/DefaultEntryLogger.java index cfe7aac7936..a8065411eed 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/DefaultEntryLogger.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/DefaultEntryLogger.java @@ -1062,7 +1062,7 @@ public EntryLogMetadata getEntryLogMetadata(long entryLogId, AbstractLogCompacto } catch (FileNotFoundException fne) { LOG.warn("Cannot find entry log file {}.log : {}", Long.toHexString(entryLogId), fne.getMessage()); throw fne; - } catch (Exception e) { + } catch (Throwable e) { LOG.info("Failed to get ledgers map index from: {}.log : {}", entryLogId, e.getMessage()); // Fall-back to scanning diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java index 41eb1d941f3..b1c4b0ab2d8 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java @@ -500,6 +500,10 @@ public void runWithFlags(boolean force, boolean suspendMajor, boolean suspendMin e.getMessage(), e); gcStats.getGcThreadRuntime().registerFailedEvent( MathUtils.nowInNano() - threadStart, TimeUnit.NANOSECONDS); + } catch (Throwable e) { + LOG.error("Error in garbage collector thread, Failed to complete GC/Compaction due to {}", + e.getMessage(), e); + gcStats.getGcThreadRuntime().registerFailedEvent(MathUtils.elapsedNanos(threadStart), TimeUnit.NANOSECONDS); } finally { if (force && forceGarbageCollection.compareAndSet(true, false)) { LOG.info("{} Set forceGarbageCollection to false after force GC to make it forceGC-able again.", From 860e37029b0c529e501f29329fc3cd8317771a89 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Tue, 2 Sep 2025 12:26:50 +0800 Subject: [PATCH 46/55] [fix]Wrong error code(-107) of opening a deleted ledger (#4657) (cherry picked from commit 5870922754b9585f7d3592dcd0714a4da65d9d80) (cherry picked from commit 2b2661de68ad0d9b02fd964c1520def07d044911) --- .../bookkeeper/bookie/BookieException.java | 12 ++ .../bookkeeper/bookie/HandleFactoryImpl.java | 2 +- .../proto/PerChannelBookieClient.java | 5 + .../bookkeeper/proto/ReadEntryProcessor.java | 2 +- .../proto/ReadEntryProcessorV3.java | 5 +- .../bookkeeper/proto/WriteEntryProcessor.java | 4 +- .../proto/WriteEntryProcessorV3.java | 4 +- .../bookkeeper/proto/WriteLacProcessorV3.java | 4 + .../apache/bookkeeper/client/TestFencing.java | 113 ++++++++++++++++++ 9 files changed, 143 insertions(+), 8 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieException.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieException.java index 2b85961cf4b..d88673d0076 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieException.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieException.java @@ -69,6 +69,8 @@ public static BookieException create(int code) { return new MetadataStoreException(); case Code.UnknownBookieIdException: return new UnknownBookieIdException(); + case Code.LedgerFencedAndDeletedException: + return new LedgerFencedAndDeletedException(); case Code.DataUnknownException: return new DataUnknownException(); default: @@ -95,6 +97,7 @@ public interface Code { int CookieExistsException = -109; int EntryLogMetadataMapException = -110; int DataUnknownException = -111; + int LedgerFencedAndDeletedException = -112; } public int getCode() { @@ -199,6 +202,15 @@ public LedgerFencedException() { } } + /** + * Signals that a ledger has been fenced in a bookie. No more entries can be appended to that ledger. + */ + public static class LedgerFencedAndDeletedException extends BookieException { + public LedgerFencedAndDeletedException() { + super(Code.LedgerFencedException); + } + } + /** * Signals that a ledger's operation has been rejected by an internal component because of the resource saturation. */ diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/HandleFactoryImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/HandleFactoryImpl.java index b331f125061..ac87c3aed45 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/HandleFactoryImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/HandleFactoryImpl.java @@ -57,7 +57,7 @@ public LedgerDescriptor getHandle(final long ledgerId, final byte[] masterKey, b if (handle == null) { if (!journalReplay && recentlyFencedAndDeletedLedgers.getIfPresent(ledgerId) != null) { - throw BookieException.create(BookieException.Code.LedgerFencedException); + throw BookieException.create(BookieException.Code.LedgerFencedAndDeletedException); } handle = LedgerDescriptor.create(masterKey, ledgerId, ledgerStorage); ledgers.putIfAbsent(ledgerId, handle); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java index 80c3a1d9de6..7bce38aae2c 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java @@ -1165,6 +1165,11 @@ private void writeAndFlush(final Channel channel, completion.setOutstanding(); } } else { + try { + future.get(); + } catch (Exception ex) { + LOG.warn("Failed to request to the bookie: {}", bookieId, ex); + } nettyOpLogger.registerFailedEvent(MathUtils.elapsedNanos(startTime), TimeUnit.NANOSECONDS); errorOut(key); } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/ReadEntryProcessor.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/ReadEntryProcessor.java index 04efd9634b2..9c6d672f480 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/ReadEntryProcessor.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/ReadEntryProcessor.java @@ -93,7 +93,7 @@ protected void processPacket() { handleReadResultForFenceRead(fenceResult, data, startTimeNanos); return; } - } catch (Bookie.NoLedgerException e) { + } catch (Bookie.NoLedgerException | BookieException.LedgerFencedAndDeletedException e) { if (LOG.isDebugEnabled()) { LOG.debug("Error reading {}", request, e); } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/ReadEntryProcessorV3.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/ReadEntryProcessorV3.java index 999b8095db6..05f9de0ea3a 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/ReadEntryProcessorV3.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/ReadEntryProcessorV3.java @@ -215,9 +215,10 @@ protected ReadResponse getReadResponse() { } } return readEntry(readResponse, entryId, startTimeSw); - } catch (Bookie.NoLedgerException e) { + } catch (Bookie.NoLedgerException | BookieException.LedgerFencedAndDeletedException e) { if (RequestUtils.isFenceRequest(readRequest)) { - LOG.info("No ledger found reading entry {} when fencing ledger {}", entryId, ledgerId); + LOG.info("No ledger found(or it has been deleted) reading entry {} when fencing ledger {}", + entryId, ledgerId); } else if (entryId != BookieProtocol.LAST_ADD_CONFIRMED) { LOG.info("No ledger found while reading entry: {} from ledger: {}", entryId, ledgerId); } else if (LOG.isDebugEnabled()) { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteEntryProcessor.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteEntryProcessor.java index d611ab963af..665e2d3a1c1 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteEntryProcessor.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteEntryProcessor.java @@ -90,8 +90,8 @@ protected void processPacket() { } catch (IOException e) { LOG.error("Error writing {}", request, e); rc = BookieProtocol.EIO; - } catch (BookieException.LedgerFencedException lfe) { - LOG.warn("Write attempt on fenced ledger {} by client {}", request.getLedgerId(), + } catch (BookieException.LedgerFencedException | BookieException.LedgerFencedAndDeletedException lfe) { + LOG.warn("Write attempt on fenced/deleted ledger {} by client {}", request.getLedgerId(), requestHandler.ctx().channel().remoteAddress()); rc = BookieProtocol.EFENCED; } catch (BookieException e) { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteEntryProcessorV3.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteEntryProcessorV3.java index 36aff7ad924..79d9f41d5c3 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteEntryProcessorV3.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteEntryProcessorV3.java @@ -136,8 +136,8 @@ public void writeComplete(int rc, long ledgerId, long entryId, logger.error("Error writing entry:{} to ledger:{}", entryId, ledgerId, e); status = StatusCode.EIO; - } catch (BookieException.LedgerFencedException e) { - logger.error("Ledger fenced while writing entry:{} to ledger:{}", + } catch (BookieException.LedgerFencedException | BookieException.LedgerFencedAndDeletedException e) { + logger.error("Ledger fenced/deleted while writing entry:{} to ledger:{}", entryId, ledgerId, e); status = StatusCode.EFENCED; } catch (BookieException e) { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteLacProcessorV3.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteLacProcessorV3.java index 293cea3bb0c..81d1bc66050 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteLacProcessorV3.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/WriteLacProcessorV3.java @@ -105,6 +105,10 @@ public void writeComplete(int rc, long ledgerId, long entryId, BookieId addr, Ob requestProcessor.bookie.setExplicitLac(Unpooled.wrappedBuffer(lacToAdd), writeCallback, requestHandler, masterKey); status = StatusCode.EOK; + } catch (BookieException.LedgerFencedAndDeletedException e) { + logger.error("Error saving lac {} for ledger:{}, which has been deleted", + lac, ledgerId, e); + status = StatusCode.ENOLEDGER; } catch (IOException e) { logger.error("Error saving lac {} for ledger:{}", lac, ledgerId, e); diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestFencing.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestFencing.java index 77382b4ebde..9f7a81450bb 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestFencing.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestFencing.java @@ -24,6 +24,9 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.UnpooledByteBufAllocator; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; import lombok.extern.slf4j.Slf4j; @@ -33,10 +36,16 @@ import org.apache.bookkeeper.bookie.SortedLedgerStorage; import org.apache.bookkeeper.bookie.storage.ldb.SingleDirectoryDbLedgerStorage; import org.apache.bookkeeper.client.BookKeeper.DigestType; +import org.apache.bookkeeper.client.api.WriteFlag; import org.apache.bookkeeper.conf.ClientConfiguration; import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.proto.BookieClientImpl; +import org.apache.bookkeeper.proto.BookieProtocol; +import org.apache.bookkeeper.proto.BookkeeperInternalCallbacks; +import org.apache.bookkeeper.proto.PerChannelBookieClient; import org.apache.bookkeeper.test.BookKeeperClusterTestCase; import org.apache.bookkeeper.test.TestStatsProvider; +import org.apache.bookkeeper.util.ByteBufList; import org.awaitility.reflect.WhiteboxImpl; import org.junit.Test; import org.slf4j.Logger; @@ -152,6 +161,110 @@ private void triggerGC(Bookie bookie) { } } + @Test(timeout = 3000 * 1000) + public void testConcurrentFenceAndDeleteLedger() throws Exception { + LedgerHandle writeLedger; + writeLedger = bkc.createLedger(digestType, "password".getBytes()); + + String tmp = "BookKeeper is cool!"; + long lac = 0; + for (int i = 0; i < 10; i++) { + long entryId = writeLedger.addEntry(tmp.getBytes()); + LOG.info("entryId: {}", entryId); + lac = entryId; + } + + // Fence and delete. + final BookieId bookieId = writeLedger.getLedgerMetadata().getEnsembleAt(0).get(0); + ClientConfiguration clientConfiguration2 = newClientConfiguration(); + clientConfiguration2.setUseV2WireProtocol(true); + ClientConfiguration clientConfiguration3 = newClientConfiguration(); + BookKeeperTestClient bkcV2 = new BookKeeperTestClient(clientConfiguration2, new TestStatsProvider()); + LedgerHandle writeLedgerV2 = bkcV2.createLedger(digestType, "password".getBytes()); + BookKeeperTestClient bkcV3 = new BookKeeperTestClient(clientConfiguration3, new TestStatsProvider()); + LedgerHandle writeLedgerV3 = bkcV3.createLedger(digestType, "password".getBytes()); + ReadOnlyLedgerHandle readLedgerV2 = + (ReadOnlyLedgerHandle) bkcV2.openLedger(writeLedger.getId(), digestType, "password".getBytes()); + ReadOnlyLedgerHandle readLedgerV3 = + (ReadOnlyLedgerHandle) bkcV3.openLedger(writeLedger.getId(), digestType, "password".getBytes()); + BookieClientImpl bookieClientV2 = (BookieClientImpl) readLedgerV2.clientCtx.getBookieClient(); + BookieClientImpl bookieClientV3 = (BookieClientImpl) readLedgerV3.clientCtx.getBookieClient(); + // Trigger opening connection. + CompletableFuture obtainV2 = new CompletableFuture<>(); + bookieClientV2.lookupClient(bookieId).obtain( + new BookkeeperInternalCallbacks.GenericCallback() { + @Override + public void operationComplete(int rc, PerChannelBookieClient result) { + obtainV2.complete(rc); + } + }, writeLedger.getId()); + assertEquals(obtainV2.get().intValue(), BKException.Code.OK); + CompletableFuture obtainV3 = new CompletableFuture<>(); + bookieClientV3.lookupClient(bookieId).obtain( + new BookkeeperInternalCallbacks.GenericCallback() { + @Override + public void operationComplete(int rc, PerChannelBookieClient result) { + obtainV3.complete(rc); + } + }, writeLedger.getId()); + assertEquals(obtainV3.get().intValue(), BKException.Code.OK); + bkcV3.deleteLedger(readLedgerV3.ledgerId); + + // Waiting for GC. + for (ServerTester server : servers) { + triggerGC(server.getServer().getBookie()); + } + + // Verify: read requests with V2 protocol will receive a NoSuchLedgerException. + final byte readEntryFlagFencing = 1; + CompletableFuture readResV2 = new CompletableFuture<>(); + bookieClientV2.readEntry(bookieId, + writeLedger.getId(), 0, (rc, ledgerId, entryId1, buffer, ctx) -> { + readResV2.complete(rc); + }, null, readEntryFlagFencing, readLedgerV2.ledgerKey); + assertEquals(BKException.Code.NoSuchLedgerExistsException, readResV2.get().intValue()); + // Verify: read requests with V3 protocol will receive a NoSuchLedgerException. + CompletableFuture readResV3 = new CompletableFuture<>(); + bookieClientV3.readEntry(bookieId, + writeLedger.getId(), 0, (rc, ledgerId, entryId1, buffer, ctx) -> { + readResV3.complete(rc); + }, null, readEntryFlagFencing, readLedgerV3.ledgerKey); + assertEquals(BKException.Code.NoSuchLedgerExistsException, readResV3.get().intValue()); + // Verify: add requests with V2 protocol will receive a NoSuchLedgerException. + log.info("Try to add the next entry: {}:{}", writeLedger.getId(), lac + 1); + final ByteBuf dataV2 = UnpooledByteBufAllocator.DEFAULT.heapBuffer(); + // Combine add request, and rewrite ledgerId of the request. + dataV2.writeByte(1); + final ByteBuf toSendV2 = (ByteBuf) writeLedgerV2.macManager.computeDigestAndPackageForSending( + lac + 1, lac, 1, dataV2, writeLedger.ledgerKey, BookieProtocol.FLAG_NONE); + toSendV2.setLong(28, writeLedger.getId()); + CompletableFuture addResV2 = new CompletableFuture<>(); + bookieClientV2.addEntry(bookieId, writeLedger.getId(), writeLedger.ledgerKey, lac + 1, toSendV2, + (rc, ledgerId, entryId1, addr, ctx) -> { + addResV2.complete(rc); + }, null, BookieProtocol.FLAG_NONE, false, WriteFlag.NONE); + assertEquals(BKException.Code.LedgerFencedException, addResV2.get().intValue()); + // Verify: read requests with V3 protocol will receive a NoSuchLedgerException. + final ByteBuf dataV3 = UnpooledByteBufAllocator.DEFAULT.heapBuffer(); + dataV3.writeByte(1); + // Combine add request, and rewrite ledgerId of the request. + final ByteBufList toSendV3 = (ByteBufList) writeLedgerV3.macManager.computeDigestAndPackageForSending( + lac + 1, lac, 1, dataV3, writeLedger.ledgerKey, BookieProtocol.FLAG_NONE); + toSendV3.getBuffer(0).setLong(0, writeLedger.getId()); + CompletableFuture addResV3 = new CompletableFuture<>(); + bookieClientV3.addEntry(bookieId, writeLedger.getId(), writeLedger.ledgerKey, lac + 1, toSendV3, + (rc, ledgerId, entryId1, addr, ctx) -> { + addResV3.complete(rc); + }, null, BookieProtocol.FLAG_NONE, false, WriteFlag.NONE); + assertEquals(BKException.Code.LedgerFencedException, addResV3.get().intValue()); + + // cleanup. + writeLedgerV2.close(); + writeLedgerV3.close(); + bkcV2.close(); + bkcV3.close(); + } + private static int threadCount = 0; class LedgerOpenThread extends Thread { From 95da62e8ce4b6515f6ae5274f7413d34839ed3dc Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Wed, 17 Sep 2025 12:29:19 +0800 Subject: [PATCH 47/55] [fix]BK stays at read_only state even if the disk is empty (#4640) * [fix]BK stay at read_only state even if the disk is empty * correct the javadoc * improve test (cherry picked from commit bde9ff2503ae71c4ef03b212577a1a8be673123a) (cherry picked from commit c39837965872572d45293578decc9081fafc04bb) --- .../bookkeeper/bookie/BookieStateManager.java | 21 ++- .../bookkeeper/bookie/BookieStatus.java | 19 ++- .../bookkeeper/bookie/LedgerDirsMonitor.java | 17 +- .../bookkeeper/bookie/StateManager.java | 12 +- .../service/BookieStateReadOnlyService.java | 4 +- .../bookie/BookieInitializationTest.java | 153 ++++++++++++++++-- .../bookkeeper/bookie/LedgerCacheTest.java | 2 +- .../replication/AuditorLedgerCheckerTest.java | 4 +- 8 files changed, 202 insertions(+), 30 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStateManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStateManager.java index 620a1c2aaa4..c50d51a96aa 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStateManager.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStateManager.java @@ -165,7 +165,7 @@ private boolean isRegistrationManagerDisabled() { @Override public void initState(){ if (forceReadOnly.get()) { - this.bookieStatus.setToReadOnlyMode(); + this.bookieStatus.setToReadOnlyMode(false); } else if (conf.isPersistBookieStatusEnabled()) { this.bookieStatus.readFromDirectories(statusDirs); } @@ -256,22 +256,22 @@ public Void call() throws IOException { } @Override - public Future transitionToWritableMode() { + public Future transitionToWritableMode(boolean isManuallyModify) { return stateService.submit(new Callable() { @Override public Void call() throws Exception{ - doTransitionToWritableMode(); + doTransitionToWritableMode(isManuallyModify); return null; } }); } @Override - public Future transitionToReadOnlyMode() { + public Future transitionToReadOnlyMode(boolean isManuallyModify) { return stateService.submit(new Callable() { @Override public Void call() throws Exception{ - doTransitionToReadOnlyMode(); + doTransitionToReadOnlyMode(isManuallyModify); return null; } }); @@ -298,10 +298,15 @@ private void doRegisterBookie(boolean isReadOnly) throws IOException { } @VisibleForTesting - public void doTransitionToWritableMode() { + public void doTransitionToWritableMode(boolean isManuallyModify) { if (shuttingdown || forceReadOnly.get()) { return; } + if (!isManuallyModify && bookieStatus.isInReadOnlyMode() && bookieStatus.isManuallyModifiedToReadOnly()) { + LOG.info("Skip to transition Bookie to Writable mode automatically because it is manually set to read-only" + + " mode, which can only be changed manually."); + return; + } if (!bookieStatus.setToWritableMode()) { // do nothing if already in writable mode @@ -334,11 +339,11 @@ public void doTransitionToWritableMode() { } @VisibleForTesting - public void doTransitionToReadOnlyMode() { + public void doTransitionToReadOnlyMode(boolean isManuallyModify) { if (shuttingdown) { return; } - if (!bookieStatus.setToReadOnlyMode()) { + if (!bookieStatus.setToReadOnlyMode(isManuallyModify)) { return; } if (!conf.isReadOnlyModeEnabled()) { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStatus.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStatus.java index 7b2d0aad990..ccbba36b532 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStatus.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStatus.java @@ -31,6 +31,7 @@ import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.util.List; +import lombok.Getter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,6 +54,8 @@ enum BookieMode { private int layoutVersion; private long lastUpdateTime; private volatile BookieMode bookieMode; + @Getter + private boolean isManuallyModifiedToReadOnly; BookieStatus() { this.bookieMode = BookieMode.READ_WRITE; @@ -72,6 +75,7 @@ synchronized boolean setToWritableMode() { if (!bookieMode.equals(BookieMode.READ_WRITE)) { bookieMode = BookieMode.READ_WRITE; this.lastUpdateTime = System.currentTimeMillis(); + this.isManuallyModifiedToReadOnly = false; return true; } return false; @@ -81,10 +85,11 @@ boolean isInReadOnlyMode() { return bookieMode.equals(BookieMode.READ_ONLY); } - synchronized boolean setToReadOnlyMode() { + synchronized boolean setToReadOnlyMode(boolean isManuallyModify) { if (!bookieMode.equals(BookieMode.READ_ONLY)) { bookieMode = BookieMode.READ_ONLY; this.lastUpdateTime = System.currentTimeMillis(); + this.isManuallyModifiedToReadOnly = isManuallyModify; return true; } return false; @@ -147,6 +152,7 @@ void readFromDirectories(List directories) { this.lastUpdateTime = status.lastUpdateTime; this.layoutVersion = status.layoutVersion; this.bookieMode = status.bookieMode; + this.isManuallyModifiedToReadOnly = status.isManuallyModifiedToReadOnly; success = true; } } @@ -216,6 +222,15 @@ public BookieStatus parse(BufferedReader reader) if (status.layoutVersion == 1 && parts.length == 3) { status.bookieMode = BookieMode.valueOf(parts[1]); status.lastUpdateTime = Long.parseLong(parts[2].trim()); + status.isManuallyModifiedToReadOnly = true; + return status; + } + // Since we should guarantee the compatibility of downgrade. We do not change the layoutVersion, otherwise, + // the string can not be parsed by the lower version. + if (status.layoutVersion == 1 && parts.length == 4) { + status.bookieMode = BookieMode.valueOf(parts[1]); + status.lastUpdateTime = Long.parseLong(parts[2].trim()); + status.isManuallyModifiedToReadOnly = Boolean.parseBoolean(parts[3].trim()); return status; } } @@ -231,6 +246,8 @@ public String toString() { builder.append(getBookieMode()); builder.append(","); builder.append(System.currentTimeMillis()); + builder.append(","); + builder.append(isManuallyModifiedToReadOnly); builder.append("\n"); return builder.toString(); } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsMonitor.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsMonitor.java index 7ad8ba1e0c5..73e2ba18712 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsMonitor.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsMonitor.java @@ -21,6 +21,7 @@ package org.apache.bookkeeper.bookie; +import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.io.File; import java.io.IOException; @@ -54,6 +55,7 @@ class LedgerDirsMonitor { private final long minUsableSizeForHighPriorityWrites; private ScheduledExecutorService executor; private ScheduledFuture checkTask; + private boolean isFirstLoopOfCheckTask = true; public LedgerDirsMonitor(final ServerConfiguration conf, final DiskChecker diskChecker, @@ -67,6 +69,10 @@ public LedgerDirsMonitor(final ServerConfiguration conf, } private void check(final LedgerDirsManager ldm) { + final boolean isFirstLoopOfCheckTaskLocalValue = this.isFirstLoopOfCheckTask; + if (isFirstLoopOfCheckTaskLocalValue) { + this.isFirstLoopOfCheckTask = false; + } final ConcurrentMap diskUsages = ldm.getDiskUsages(); boolean someDiskFulled = false; boolean highPriorityWritesAllowed = true; @@ -175,6 +181,14 @@ private void check(final LedgerDirsManager ldm) { } } + if (isFirstLoopOfCheckTaskLocalValue && ldm.getFullFilledLedgerDirs().isEmpty()) { + // notify no disk full. + for (LedgerDirsListener listener : ldm.getListeners()) { + listener.allDisksWritable(); + } + return; + } + if (conf.isReadOnlyModeOnAnyDiskFullEnabled()) { if (someDiskFulled && !ldm.getFullFilledLedgerDirs().isEmpty()) { // notify any disk full. @@ -192,7 +206,8 @@ private void check(final LedgerDirsManager ldm) { } } - private void check() { + @VisibleForTesting + void check() { dirsManagers.forEach(this::check); } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/StateManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/StateManager.java index 7ed3f0b6572..9ef1f384e77 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/StateManager.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/StateManager.java @@ -100,12 +100,20 @@ public interface StateManager extends AutoCloseable { /** * Change the state of bookie to Writable mode. */ - Future transitionToWritableMode(); + Future transitionToWritableMode(boolean isManuallyModify); + + default Future transitionToWritableMode() { + return transitionToWritableMode(false); + } /** * Change the state of bookie to ReadOnly mode. */ - Future transitionToReadOnlyMode(); + Future transitionToReadOnlyMode(boolean isManuallyModify); + + default Future transitionToReadOnlyMode() { + return transitionToReadOnlyMode(false); + } /** * ShutdownHandler used to shutdown bookie. diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/BookieStateReadOnlyService.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/BookieStateReadOnlyService.java index d32074e2bb3..73f57c19598 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/BookieStateReadOnlyService.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/BookieStateReadOnlyService.java @@ -58,14 +58,14 @@ public HttpServiceResponse handle(HttpServiceRequest request) throws Exception { response.setBody("Bookie is in forceReadOnly mode, cannot transit to writable mode"); return response; } - stateManager.transitionToWritableMode().get(); + stateManager.transitionToWritableMode(true).get(); } else if (!stateManager.isReadOnly() && inState.isReadOnly()) { if (!stateManager.isReadOnlyModeEnabled()) { response.setCode(HttpServer.StatusCode.BAD_REQUEST); response.setBody("Bookie is disabled ReadOnly mode, cannot transit to readOnly mode"); return response; } - stateManager.transitionToReadOnlyMode().get(); + stateManager.transitionToReadOnlyMode(true).get(); } } else if (!HttpServer.Method.GET.equals(request.getMethod())) { response.setCode(HttpServer.StatusCode.NOT_FOUND); diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java index 5188eb816bb..bd0a12ca6cb 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java @@ -77,6 +77,7 @@ import org.apache.bookkeeper.common.component.ComponentStarter; import org.apache.bookkeeper.common.component.Lifecycle; import org.apache.bookkeeper.common.component.LifecycleComponent; +import org.apache.bookkeeper.common.util.JsonUtil; import org.apache.bookkeeper.conf.ClientConfiguration; import org.apache.bookkeeper.conf.ServerConfiguration; import org.apache.bookkeeper.conf.TestBKConfiguration; @@ -84,7 +85,10 @@ import org.apache.bookkeeper.discover.BookieServiceInfo.Endpoint; import org.apache.bookkeeper.discover.RegistrationManager; import org.apache.bookkeeper.http.HttpRouter; +import org.apache.bookkeeper.http.HttpServer; import org.apache.bookkeeper.http.HttpServerLoader; +import org.apache.bookkeeper.http.service.HttpServiceRequest; +import org.apache.bookkeeper.http.service.HttpServiceResponse; import org.apache.bookkeeper.meta.MetadataBookieDriver; import org.apache.bookkeeper.meta.exceptions.MetadataException; import org.apache.bookkeeper.meta.zk.ZKMetadataBookieDriver; @@ -96,6 +100,7 @@ import org.apache.bookkeeper.replication.ReplicationStats; import org.apache.bookkeeper.server.Main; import org.apache.bookkeeper.server.conf.BookieConfiguration; +import org.apache.bookkeeper.server.http.service.BookieStateReadOnlyService; import org.apache.bookkeeper.server.service.AutoRecoveryService; import org.apache.bookkeeper.server.service.BookieService; import org.apache.bookkeeper.stats.NullStatsLogger; @@ -107,8 +112,10 @@ import org.apache.bookkeeper.versioning.Version; import org.apache.bookkeeper.versioning.Versioned; import org.apache.bookkeeper.zookeeper.ZooKeeperClient; +import org.apache.commons.io.FileUtils; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.data.Stat; +import org.awaitility.Awaitility; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestName; @@ -1445,30 +1452,150 @@ public void testRetrieveBookieStatusWhenStatusFileIsCorrupted() throws Exception .setPersistBookieStatusEnabled(true) .setMetadataServiceUri(metadataServiceUri); // start a new bookie - BookieServer bookieServer = new BookieServer( + BookieServer bookieServer1 = new BookieServer( conf, new TestBookieImpl(conf), NullStatsLogger.INSTANCE, UnpooledByteBufAllocator.DEFAULT, new MockUncleanShutdownDetection()); - bookieServer.start(); + bookieServer1.start(); // transition in to read only and persist the status on disk - Bookie bookie = (BookieImpl) bookieServer.getBookie(); - assertFalse(bookie.isReadOnly()); - bookie.getStateManager().transitionToReadOnlyMode().get(); - assertTrue(bookie.isReadOnly()); + Bookie bookie1 = (BookieImpl) bookieServer1.getBookie(); + assertFalse(bookie1.isReadOnly()); + bookie1.getStateManager().transitionToReadOnlyMode().get(); + assertTrue(bookie1.isReadOnly()); // corrupt status file - List ledgerDirs = ((BookieImpl) bookie).getLedgerDirsManager().getAllLedgerDirs(); + List ledgerDirs = ((BookieImpl) bookie1).getLedgerDirsManager().getAllLedgerDirs(); corruptFile(new File(ledgerDirs.get(0), BOOKIE_STATUS_FILENAME)); corruptFile(new File(ledgerDirs.get(1), BOOKIE_STATUS_FILENAME)); // restart the bookie should be in read only mode - bookieServer.shutdown(); - bookieServer = new BookieServer( + bookieServer1.shutdown(); + BookieServer bookieServer2 = new BookieServer( conf, new TestBookieImpl(conf), NullStatsLogger.INSTANCE, UnpooledByteBufAllocator.DEFAULT, new MockUncleanShutdownDetection()); - bookieServer.start(); - bookie = bookieServer.getBookie(); - assertTrue(bookie.isReadOnly()); - bookieServer.shutdown(); + bookieServer2.start(); + BookieImpl bookie2 = (BookieImpl) bookieServer2.getBookie(); + assertTrue(bookie2.isReadOnly()); + // After a disk check, the bookie should switch to writable mode because the disk usage is fine. + bookie2.dirsMonitor.check(); + Awaitility.await().untilAsserted(() -> { + assertFalse(bookie2.isReadOnly()); + }); + bookieServer2.shutdown(); + } + + + private void setBookieToReadOnly(Bookie bookie, boolean readOnly) throws Exception { + BookieStateReadOnlyService.ReadOnlyState state = new BookieStateReadOnlyService.ReadOnlyState(); + state.setReadOnly(readOnly); + HttpServiceRequest request = new HttpServiceRequest(JsonUtil.toJson(state), HttpServer.Method.PUT, null); + BookieStateReadOnlyService service = new BookieStateReadOnlyService(bookie); + HttpServiceResponse response1 = service.handle(request); + assertEquals(HttpServer.StatusCode.OK.getValue(), response1.getStatusCode()); + } + + /** + * Verify: once the state is manually modified to read-only by Admin API, it should not be changed to writable + * by the monitor task. + * But it can be changed to read-only by monitor task if it was manually set to writable by Admin API. + */ + @Test(timeout = 1000 * 30) + public void testRetrieveBookieStatusAdminModifiedWhenStatusFileIsCorrupted() throws Exception { + File[] tmpLedgerDirs = new File[3]; + String[] filePath = new String[tmpLedgerDirs.length]; + for (int i = 0; i < tmpLedgerDirs.length; i++) { + tmpLedgerDirs[i] = tmpDirs.createNew("bookie", "test" + i); + filePath[i] = tmpLedgerDirs[i].getPath(); + } + final ServerConfiguration conf = TestBKConfiguration.newServerConfiguration(); + conf.setJournalDirName(filePath[0]) + .setLedgerDirNames(filePath) + .setReadOnlyModeEnabled(true) + .setPersistBookieStatusEnabled(true) + .setMetadataServiceUri(metadataServiceUri); + // start a new bookie + BookieServer bookieServer1 = new BookieServer( + conf, new TestBookieImpl(conf), + NullStatsLogger.INSTANCE, UnpooledByteBufAllocator.DEFAULT, + new MockUncleanShutdownDetection()); + bookieServer1.start(); + // transition in to read only and persist the status on disk + Bookie bookie1 = (BookieImpl) bookieServer1.getBookie(); + assertFalse(bookie1.isReadOnly()); + setBookieToReadOnly(bookie1, true); + assertTrue(bookie1.isReadOnly()); + // corrupt status file + List ledgerDirs = ((BookieImpl) bookie1).getLedgerDirsManager().getAllLedgerDirs(); + corruptFile(new File(ledgerDirs.get(0), BOOKIE_STATUS_FILENAME)); + corruptFile(new File(ledgerDirs.get(1), BOOKIE_STATUS_FILENAME)); + // restart the bookie should be in read only mode + bookieServer1.shutdown(); + BookieServer bookieServer2 = new BookieServer( + conf, new TestBookieImpl(conf), + NullStatsLogger.INSTANCE, UnpooledByteBufAllocator.DEFAULT, + new MockUncleanShutdownDetection()); + bookieServer2.start(); + BookieImpl bookie2 = (BookieImpl) bookieServer2.getBookie(); + assertTrue(bookie2.isReadOnly()); + // After a disk check, the bookie should not switch to writable mode because the state was set to read-only + // by admin api manually. + bookie2.dirsMonitor.check(); + Thread.sleep(2000); + Awaitility.await().untilAsserted(() -> { + assertTrue(bookie2.isReadOnly()); + }); + // We can use Admin Api to change the state back to writable. + setBookieToReadOnly(bookie2, false); + assertFalse(bookie2.isReadOnly()); + // The state can be changed to read-only by monitor task if it was manually set to writable by Admin API. + bookie2.getStateManager().transitionToReadOnlyMode().get(); + assertTrue(bookie2.isReadOnly()); + bookieServer2.shutdown(); + } + + /** + * Verify: the newest version can read the old version payload of persisted bookie status. + * old payload: "1,READ_ONLY,1752809349613" + * new payload: "1,READ_ONLY,1752809349613,false" + */ + @Test(timeout = 30000) + public void testPersistBookieStatusCompatibility() throws Exception { + File[] tmpLedgerDirs = new File[1]; + String[] filePath = new String[1]; + tmpLedgerDirs[0] = tmpDirs.createNew("bookie", "test"); + filePath[0] = tmpLedgerDirs[0].getPath(); + final ServerConfiguration conf = TestBKConfiguration.newServerConfiguration(); + conf.setJournalDirName(filePath[0]) + .setLedgerDirNames(filePath) + .setReadOnlyModeEnabled(true) + .setPersistBookieStatusEnabled(true) + .setMetadataServiceUri(metadataServiceUri); + // start a new bookie + BookieServer bookieServer1 = new BookieServer( + conf, new TestBookieImpl(conf), + NullStatsLogger.INSTANCE, UnpooledByteBufAllocator.DEFAULT, + new MockUncleanShutdownDetection()); + bookieServer1.start(); + Bookie bookie1 = (BookieImpl) bookieServer1.getBookie(); + // restart the bookie should be in read only mode + bookieServer1.shutdown(); + // Rewrite file to the old version payload. + List ledgerDirs = ((BookieImpl) bookie1).getLedgerDirsManager().getAllLedgerDirs(); + FileUtils.writeStringToFile(new File(ledgerDirs.get(0), BOOKIE_STATUS_FILENAME), + "1,READ_ONLY,1752809349613"); + BookieServer bookieServer2 = new BookieServer( + conf, new TestBookieImpl(conf), + NullStatsLogger.INSTANCE, UnpooledByteBufAllocator.DEFAULT, + new MockUncleanShutdownDetection()); + bookieServer2.start(); + BookieImpl bookie2 = (BookieImpl) bookieServer2.getBookie(); + assertTrue(bookie2.isReadOnly()); + // After a disk check, the bookie should not switch to writable mode because the state was set to read-only + // by admin api manually(the default value of previous version is true). + bookie2.dirsMonitor.check(); + Awaitility.await().untilAsserted(() -> { + assertTrue(bookie2.isReadOnly()); + }); + bookieServer2.shutdown(); } /** diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java index 4e8f06fe7f4..9e2090d7975 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java @@ -711,7 +711,7 @@ public void onSizeLimitReached(final CheckpointSource.Checkpoint cp) throws IOEx LOG.info("Started flushing mem table."); memTable.flush(FlushTestSortedLedgerStorage.this); } catch (IOException e) { - getStateManager().doTransitionToReadOnlyMode(); + getStateManager().doTransitionToReadOnlyMode(false); LOG.error("Exception thrown while flushing skip list cache.", e); } } diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java index 2e3e09012fb..bbeb7e758fd 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java @@ -326,7 +326,7 @@ public void testReadOnlyBookieExclusionFromURLedgersCheck() throws Exception { BookieServer bk = serverByIndex(bkIndex); bookieConf.setReadOnlyModeEnabled(true); - ((BookieImpl) bk.getBookie()).getStateManager().doTransitionToReadOnlyMode(); + ((BookieImpl) bk.getBookie()).getStateManager().doTransitionToReadOnlyMode(false); bkc.waitForReadOnlyBookie(BookieImpl.getBookieId(confByIndex(bkIndex))) .get(30, TimeUnit.SECONDS); @@ -361,7 +361,7 @@ public void testReadOnlyBookieShutdown() throws Exception { BookieServer bk = serverByIndex(bkIndex); bookieConf.setReadOnlyModeEnabled(true); - ((BookieImpl) bk.getBookie()).getStateManager().doTransitionToReadOnlyMode(); + ((BookieImpl) bk.getBookie()).getStateManager().doTransitionToReadOnlyMode(false); bkc.waitForReadOnlyBookie(BookieImpl.getBookieId(confByIndex(bkIndex))) .get(30, TimeUnit.SECONDS); From d9a7a8f2999dcbf9e4ac36bae709ca6779b9c525 Mon Sep 17 00:00:00 2001 From: Yong Zhang Date: Mon, 24 Nov 2025 11:32:24 +0800 Subject: [PATCH 48/55] Bring back the old public method name (#4682) (cherry picked from commit 1a509e4f736a75bc4570748324afba94ba51e471) (cherry picked from commit dd0ed9d65bdb88b39bcee5030e7fa024c939f48a) --- .../org/apache/bookkeeper/bookie/BookieStateManager.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStateManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStateManager.java index c50d51a96aa..070897b9473 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStateManager.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStateManager.java @@ -297,6 +297,10 @@ private void doRegisterBookie(boolean isReadOnly) throws IOException { } } + public void doTransitionToWritableMode() { + doTransitionToWritableMode(false); + } + @VisibleForTesting public void doTransitionToWritableMode(boolean isManuallyModify) { if (shuttingdown || forceReadOnly.get()) { @@ -338,6 +342,10 @@ public void doTransitionToWritableMode(boolean isManuallyModify) { } } + public void doTransitionToReadOnlyMode() { + doTransitionToReadOnlyMode(false); + } + @VisibleForTesting public void doTransitionToReadOnlyMode(boolean isManuallyModify) { if (shuttingdown) { From a8708eb0142f8e7bd8b6ff6f8175efcd90a082b9 Mon Sep 17 00:00:00 2001 From: mukesh-ctds Date: Mon, 10 Feb 2025 11:45:01 +0530 Subject: [PATCH 49/55] Skipped license check for com.datastax.oss binaries (cherry picked from commit ecb7603b2ca4bb9ba77f8a5e1111da402269ce10) --- dev/check-binary-license | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dev/check-binary-license b/dev/check-binary-license index 5abc5b11a7b..af46f0b95fa 100755 --- a/dev/check-binary-license +++ b/dev/check-binary-license @@ -84,6 +84,11 @@ for J in $JARS; do continue fi + echo $J | grep -q "com.datastax.oss" + if [ $? == 0 ]; then + continue + fi + echo "$LICENSE" | grep -q $J if [ $? != 0 ]; then echo $J unaccounted for in LICENSE From 2e1f4d2c654fb06029f2b2e1b96c3339166852ad Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Wed, 22 Oct 2025 12:53:14 +0800 Subject: [PATCH 50/55] [fix]compile error of the file native_io_jni.c for the env jdk11 & windows (#4665) Co-authored-by: fengyubiao (cherry picked from commit 5154149936e3f8187ae9e2755f43ea9c0ecb866e) --- .../src/main/native-io-jni/cpp/native_io_jni.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/native-io/src/main/native-io-jni/cpp/native_io_jni.c b/native-io/src/main/native-io-jni/cpp/native_io_jni.c index d3bc164bec9..7be468518b0 100644 --- a/native-io/src/main/native-io-jni/cpp/native_io_jni.c +++ b/native-io/src/main/native-io-jni/cpp/native_io_jni.c @@ -20,11 +20,16 @@ */ #define _GNU_SOURCE +#include #include #include #include #include +#ifdef _WIN32 +#include +#else #include +#endif #include @@ -165,7 +170,14 @@ JNIEXPORT jint JNICALL Java_org_apache_bookkeeper_common_util_nativeio_NativeIOJni_fsync(JNIEnv * env, jclass clazz, jint fd) { - int res = fsync(fd); + int res; + + // Guarantee compatibility for winsows. + #ifdef _WIN32 + res = _commit((int)fd); + #else + res = fsync((int)fd); + #endif if (res == -1) { throwExceptionWithErrno(env, "Failed to fsync"); From 609a987ea970798ef0ad273e297bd910ed5f2e04 Mon Sep 17 00:00:00 2001 From: Masahiro Sakamoto Date: Mon, 12 Jan 2026 17:07:10 +0900 Subject: [PATCH 51/55] Replace net.jpountz.lz4:lz4 with at.yawk.lz4:lz4-java (#4696) (cherry picked from commit b88fb5f0390a37947b33f1ba151de820f10452c5) --- bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt | 8 ++++---- bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt | 8 ++++---- bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt | 8 ++++---- pom.xml | 8 ++++---- shaded/distributedlog-core-shaded/pom.xml | 2 +- stream/distributedlog/common/pom.xml | 4 ++-- .../org/apache/distributedlog/io/LZ4CompressionCodec.java | 8 ++++---- 7 files changed, 23 insertions(+), 23 deletions(-) diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index ccf7b55bf1a..790e67297fd 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -275,7 +275,7 @@ Apache Software License, Version 2. - lib/com.beust-jcommander-1.82.jar [24] - lib/com.yahoo.datasketches-memory-0.8.3.jar [25] - lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25] -- lib/net.jpountz.lz4-lz4-1.3.0.jar [26] +- lib/at.yawk.lz4-lz4-java-1.10.2.jar [26] - lib/com.google.api.grpc-proto-google-common-protos-2.9.0.jar [28] - lib/com.google.code.gson-gson-2.9.0.jar [29] - lib/io.opencensus-opencensus-api-0.28.0.jar [30] @@ -346,7 +346,7 @@ Apache Software License, Version 2. [23] Source available at https://github.com/facebook/rocksdb/tree/v6.29.4.1 [24] Source available at https://github.com/cbeust/jcommander/tree/1.82 [25] Source available at https://github.com/DataSketches/sketches-core/tree/sketches-0.8.3 -[26] Source available at https://github.com/lz4/lz4-java/tree/1.3.0 +[26] Source available at https://github.com/yawkat/lz4-java/tree/v1.10.2 [28] Source available at https://github.com/googleapis/java-common-protos/tree/v2.17.0 [29] Source available at https://github.com/google/gson/tree/gson-parent-2.10.1 [30] Source available at https://github.com/census-instrumentation/opencensus-java/tree/v0.28.0 @@ -459,13 +459,13 @@ decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * HOMEPAGE: * https://github.com/ning/compress -lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'lz4', a LZ4 Java compression +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'lz4-java', a LZ4 Java compression and decompression library written by Adrien Grand. It can be obtained at: * LICENSE: * deps/netty/LICENSE.lz4.txt (Apache License 2.0) * HOMEPAGE: - * https://github.com/jpountz/lz4-java + * https://github.com/yawkat/lz4-java lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'lzma-java', a LZMA Java compression and decompression library, which can be obtained at: diff --git a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt index 20007fd504a..9ca54cc559b 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt @@ -246,7 +246,7 @@ Apache Software License, Version 2. - lib/org.apache.zookeeper-zookeeper-jute-3.9.3.jar [20] - lib/org.apache.zookeeper-zookeeper-3.9.3-tests.jar [20] - lib/com.beust-jcommander-1.82.jar [23] -- lib/net.jpountz.lz4-lz4-1.3.0.jar [25] +- lib/at.yawk.lz4-lz4-java-1.10.2.jar [25] - lib/com.google.api.grpc-proto-google-common-protos-2.9.0.jar [27] - lib/com.google.code.gson-gson-2.9.0.jar [28] - lib/io.opencensus-opencensus-api-0.28.0.jar [29] @@ -305,7 +305,7 @@ Apache Software License, Version 2. [19] Source available at https://github.com/apache/commons-lang/tree/LANG_3_6 [20] Source available at https://github.com/apache/zookeeper/tree/release-3.8.0 [23] Source available at https://github.com/cbeust/jcommander/tree/1.82 -[25] Source available at https://github.com/lz4/lz4-java/tree/1.3.0 +[25] Source available at https://github.com/yawkat/lz4-java/tree/v1.10.2 [27] Source available at https://github.com/googleapis/java-common-protos/tree/v2.17.0 [28] Source available at https://github.com/google/gson/tree/gson-parent-2.10.1 [29] Source available at https://github.com/census-instrumentation/opencensus-java/tree/v0.28.0 @@ -416,13 +416,13 @@ decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * HOMEPAGE: * https://github.com/ning/compress -lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'lz4', a LZ4 Java compression +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'lz4-java', a LZ4 Java compression and decompression library written by Adrien Grand. It can be obtained at: * LICENSE: * deps/netty/LICENSE.lz4.txt (Apache License 2.0) * HOMEPAGE: - * https://github.com/jpountz/lz4-java + * https://github.com/yawkat/lz4-java lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'lzma-java', a LZMA Java compression and decompression library, which can be obtained at: diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt index 61a176243ad..a6fa25a9cbe 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt @@ -275,7 +275,7 @@ Apache Software License, Version 2. - lib/com.beust-jcommander-1.82.jar [24] - lib/com.yahoo.datasketches-memory-0.8.3.jar [25] - lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25] -- lib/net.jpountz.lz4-lz4-1.3.0.jar [26] +- lib/at.yawk.lz4-lz4-java-1.10.2.jar [26] - lib/com.google.api.grpc-proto-google-common-protos-2.9.0.jar [28] - lib/com.google.code.gson-gson-2.9.0.jar [29] - lib/io.opencensus-opencensus-api-0.28.0.jar [30] @@ -342,7 +342,7 @@ Apache Software License, Version 2. [23] Source available at https://github.com/facebook/rocksdb/tree/v6.29.4.1 [24] Source available at https://github.com/cbeust/jcommander/tree/1.82 [25] Source available at https://github.com/DataSketches/sketches-core/tree/sketches-0.8.3 -[26] Source available at https://github.com/lz4/lz4-java/tree/1.3.0 +[26] Source available at https://github.com/yawkat/lz4-java/tree/v1.10.2 [28] Source available at https://github.com/googleapis/java-common-protos/tree/v2.17.0 [29] Source available at https://github.com/google/gson/tree/gson-parent-2.10.1 [30] Source available at https://github.com/census-instrumentation/opencensus-java/tree/v0.28.0 @@ -454,13 +454,13 @@ decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * HOMEPAGE: * https://github.com/ning/compress -lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'lz4', a LZ4 Java compression +lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'lz4-java', a LZ4 Java compression and decompression library written by Adrien Grand. It can be obtained at: * LICENSE: * deps/netty/LICENSE.lz4.txt (Apache License 2.0) * HOMEPAGE: - * https://github.com/jpountz/lz4-java + * https://github.com/yawkat/lz4-java lib/io.netty-netty-codec-4.1.121.Final.jar optionally depends on 'lzma-java', a LZMA Java compression and decompression library, which can be obtained at: diff --git a/pom.xml b/pom.xml index 61ba5ba1f92..f4de5687930 100644 --- a/pom.xml +++ b/pom.xml @@ -152,7 +152,7 @@ 0.14.2 1.18.30 2.18.0 - 1.3.0 + 1.10.2 3.12.4 4.1.121.Final 0.0.26.Final @@ -319,9 +319,9 @@ - net.jpountz.lz4 - lz4 - ${lz4.version} + at.yawk.lz4 + lz4-java + ${lz4-java.version} diff --git a/shaded/distributedlog-core-shaded/pom.xml b/shaded/distributedlog-core-shaded/pom.xml index 263b923ba84..4b35c10e1df 100644 --- a/shaded/distributedlog-core-shaded/pom.xml +++ b/shaded/distributedlog-core-shaded/pom.xml @@ -74,7 +74,7 @@ com.google.guava:failureaccess com.google.guava:guava com.google.protobuf:protobuf-java - net.jpountz.lz4:lz4 + at.yawk.lz4:lz4-java com.datastax.oss:bookkeeper-common com.datastax.oss:bookkeeper-common-allocator com.datastax.oss:cpu-affinity diff --git a/stream/distributedlog/common/pom.xml b/stream/distributedlog/common/pom.xml index a5ab3cbca3e..46c1da617f1 100644 --- a/stream/distributedlog/common/pom.xml +++ b/stream/distributedlog/common/pom.xml @@ -74,8 +74,8 @@ netty-buffer - net.jpountz.lz4 - lz4 + at.yawk.lz4 + lz4-java org.jmock diff --git a/stream/distributedlog/common/src/main/java/org/apache/distributedlog/io/LZ4CompressionCodec.java b/stream/distributedlog/common/src/main/java/org/apache/distributedlog/io/LZ4CompressionCodec.java index 48439d23a1a..a02322f4cc8 100644 --- a/stream/distributedlog/common/src/main/java/org/apache/distributedlog/io/LZ4CompressionCodec.java +++ b/stream/distributedlog/common/src/main/java/org/apache/distributedlog/io/LZ4CompressionCodec.java @@ -25,7 +25,7 @@ import java.nio.ByteBuffer; import net.jpountz.lz4.LZ4Compressor; import net.jpountz.lz4.LZ4Factory; -import net.jpountz.lz4.LZ4FastDecompressor; +import net.jpountz.lz4.LZ4SafeDecompressor; /** * An {@code lz4} based {@link CompressionCodec} implementation. @@ -44,7 +44,7 @@ public static LZ4CompressionCodec of() { // Used for compression private static final LZ4Compressor compressor = factory.fastCompressor(); // Used to decompress when the size of the output is known - private static final LZ4FastDecompressor decompressor = factory.fastDecompressor(); + private static final LZ4SafeDecompressor decompressor = factory.safeDecompressor(); @Override public ByteBuf compress(ByteBuf uncompressed, int headerLen) { @@ -69,7 +69,7 @@ public ByteBuf compress(ByteBuf uncompressed, int headerLen) { } @Override - // length parameter is ignored here because of the way the fastDecompressor works. + // length parameter is ignored here because of the way the safeDecompressor works. public ByteBuf decompress(ByteBuf compressed, int decompressedSize) { checkNotNull(compressed); checkArgument(compressed.readableBytes() >= 0); @@ -80,7 +80,7 @@ public ByteBuf decompress(ByteBuf compressed, int decompressedSize) { ByteBuffer compressedNio = compressed.nioBuffer(compressed.readerIndex(), compressed.readableBytes()); decompressor.decompress( - compressedNio, compressedNio.position(), + compressedNio, compressedNio.position(), compressedNio.remaining(), uncompressedNio, uncompressedNio.position(), uncompressedNio.remaining()); uncompressed.writerIndex(decompressedSize); return uncompressed; From 4a471bdfe7050364024659d0f52880608dbef1fb Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 17 Apr 2025 13:56:23 +0300 Subject: [PATCH 52/55] Upgrade to grpc 1.72.0 (#4591) (cherry picked from commit 95277b646d97846668bcdcf2aba5ed47303b3289) --- .../src/main/resources/LICENSE-all.bin.txt | 40 ++++++++++--------- .../src/main/resources/LICENSE-bkctl.bin.txt | 40 ++++++++++--------- .../src/main/resources/LICENSE-server.bin.txt | 40 ++++++++++--------- .../src/main/resources/NOTICE-all.bin.txt | 18 ++++----- .../src/main/resources/NOTICE-bkctl.bin.txt | 18 ++++----- .../src/main/resources/NOTICE-server.bin.txt | 18 ++++----- pom.xml | 2 +- 7 files changed, 94 insertions(+), 82 deletions(-) diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index 790e67297fd..dfd23e5c09b 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -281,21 +281,25 @@ Apache Software License, Version 2. - lib/io.opencensus-opencensus-api-0.28.0.jar [30] - lib/io.opencensus-opencensus-contrib-http-util-0.28.0.jar [30] - lib/io.opencensus-opencensus-proto-0.2.0.jar [30] -- lib/io.grpc-grpc-all-1.54.1.jar [33] -- lib/io.grpc-grpc-alts-1.54.1.jar [33] -- lib/io.grpc-grpc-api-1.54.1.jar [33] -- lib/io.grpc-grpc-auth-1.54.1.jar [33] -- lib/io.grpc-grpc-context-1.54.1.jar [33] -- lib/io.grpc-grpc-core-1.54.1.jar [33] -- lib/io.grpc-grpc-grpclb-1.54.1.jar [33] -- lib/io.grpc-grpc-protobuf-1.54.1.jar [33] -- lib/io.grpc-grpc-protobuf-lite-1.54.1.jar [33] -- lib/io.grpc-grpc-netty-shaded-1.54.1.jar [33] -- lib/io.grpc-grpc-services-1.54.1.jar [33] -- lib/io.grpc-grpc-stub-1.54.1.jar [33] -- lib/io.grpc-grpc-testing-1.54.1.jar [33] -- lib/io.grpc-grpc-xds-1.54.1.jar [33] -- lib/io.grpc-grpc-rls-1.54.1.jar[33] +- lib/io.grpc-grpc-all-1.72.0.jar [33] +- lib/io.grpc-grpc-alts-1.72.0.jar [33] +- lib/io.grpc-grpc-api-1.72.0.jar [33] +- lib/io.grpc-grpc-auth-1.72.0.jar [33] +- lib/io.grpc-grpc-context-1.72.0.jar [33] +- lib/io.grpc-grpc-core-1.72.0.jar [33] +- lib/io.grpc-grpc-grpclb-1.72.0.jar [33] +- lib/io.grpc-grpc-inprocess-1.72.0.jar [33] +- lib/io.grpc-grpc-opentelemetry-1.72.0.jar [33] +- lib/io.grpc-grpc-netty-shaded-1.72.0.jar [33] +- lib/io.grpc-grpc-protobuf-1.72.0.jar [33] +- lib/io.grpc-grpc-protobuf-lite-1.72.0.jar [33] +- lib/io.grpc-grpc-services-1.72.0.jar [33] +- lib/io.grpc-grpc-stub-1.72.0.jar [33] +- lib/io.grpc-grpc-testing-1.72.0.jar [33] +- lib/io.grpc-grpc-util-1.72.0.jar [33] +- lib/io.grpc-grpc-xds-1.72.0.jar [33] +- lib/io.grpc-grpc-rls-1.72.0.jar[33] +- lib/io.grpc-grpc-gcp-csm-observability-1.72.0.jar [33] - lib/org.apache.curator-curator-client-5.7.1.jar [34] - lib/org.apache.curator-curator-framework-5.7.1.jar [34] - lib/org.apache.curator-curator-recipes-5.7.1.jar [34] @@ -310,7 +314,7 @@ Apache Software License, Version 2. - lib/com.google.http-client-google-http-client-gson-1.41.0.jar [43] - lib/com.google.auto.value-auto-value-annotations-1.9.jar [44] - lib/com.google.j2objc-j2objc-annotations-2.8.jar [45] -- lib/com.google.re2j-re2j-1.6.jar [46] +- lib/com.google.re2j-re2j-1.8.jar [46] - lib/io.dropwizard.metrics-metrics-core-4.1.12.1.jar [47] - lib/io.dropwizard.metrics-metrics-graphite-4.1.12.1.jar [47] - lib/io.dropwizard.metrics-metrics-jmx-4.1.12.1.jar [47] @@ -350,7 +354,7 @@ Apache Software License, Version 2. [28] Source available at https://github.com/googleapis/java-common-protos/tree/v2.17.0 [29] Source available at https://github.com/google/gson/tree/gson-parent-2.10.1 [30] Source available at https://github.com/census-instrumentation/opencensus-java/tree/v0.28.0 -[33] Source available at https://github.com/grpc/grpc-java/tree/v1.56.0 +[33] Source available at https://github.com/grpc/grpc-java/tree/v1.72.0 [34] Source available at https://github.com/apache/curator/releases/tag/apache.curator-5.7.1 [36] Source available at https://github.com/google/error-prone/tree/v2.9.0 [37] Source available at https://github.com/apache/yetus/tree/rel/0.12.0 @@ -362,7 +366,7 @@ Apache Software License, Version 2. [43] Source available at https://github.com/googleapis/google-http-java-client/releases/tag/v1.41.0 [44] Source available at https://github.com/google/auto/releases/tag/auto-value-1.10.1 [45] Source available at https://github.com/google/j2objc/releases/tag/1.3 -[46] Source available at https://github.com/google/re2j/releases/tag/re2j-1.7 +[46] Source available at https://github.com/google/re2j/releases/tag/re2j-1.8 [47] Source available at https://github.com/dropwizard/metrics/releases/tag/v4.1.12.1 [48] Source available at https://github.com/perfmark/perfmark/releases/tag/v0.26.0 [49] Source available at https://github.com/google/conscrypt/releases/tag/2.5.2 diff --git a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt index 9ca54cc559b..e8d50b6b0e0 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt @@ -252,21 +252,25 @@ Apache Software License, Version 2. - lib/io.opencensus-opencensus-api-0.28.0.jar [29] - lib/io.opencensus-opencensus-contrib-http-util-0.28.0.jar [29] - lib/io.opencensus-opencensus-proto-0.2.0.jar [29] -- lib/io.grpc-grpc-all-1.54.1.jar [32] -- lib/io.grpc-grpc-alts-1.54.1.jar [32] -- lib/io.grpc-grpc-api-1.54.1.jar [32] -- lib/io.grpc-grpc-auth-1.54.1.jar [32] -- lib/io.grpc-grpc-context-1.54.1.jar [32] -- lib/io.grpc-grpc-core-1.54.1.jar [32] -- lib/io.grpc-grpc-grpclb-1.54.1.jar [32] -- lib/io.grpc-grpc-protobuf-1.54.1.jar [32] -- lib/io.grpc-grpc-protobuf-lite-1.54.1.jar [32] -- lib/io.grpc-grpc-netty-shaded-1.54.1.jar [32] -- lib/io.grpc-grpc-services-1.54.1.jar [32] -- lib/io.grpc-grpc-stub-1.54.1.jar [32] -- lib/io.grpc-grpc-testing-1.54.1.jar [32] -- lib/io.grpc-grpc-xds-1.54.1.jar [32] -- lib/io.grpc-grpc-rls-1.54.1.jar[32] +- lib/io.grpc-grpc-all-1.72.0.jar [32] +- lib/io.grpc-grpc-alts-1.72.0.jar [32] +- lib/io.grpc-grpc-api-1.72.0.jar [32] +- lib/io.grpc-grpc-auth-1.72.0.jar [32] +- lib/io.grpc-grpc-context-1.72.0.jar [32] +- lib/io.grpc-grpc-core-1.72.0.jar [32] +- lib/io.grpc-grpc-grpclb-1.72.0.jar [32] +- lib/io.grpc-grpc-inprocess-1.72.0.jar [32] +- lib/io.grpc-grpc-opentelemetry-1.72.0.jar [32] +- lib/io.grpc-grpc-netty-shaded-1.72.0.jar [32] +- lib/io.grpc-grpc-protobuf-1.72.0.jar [32] +- lib/io.grpc-grpc-protobuf-lite-1.72.0.jar [32] +- lib/io.grpc-grpc-services-1.72.0.jar [32] +- lib/io.grpc-grpc-stub-1.72.0.jar [32] +- lib/io.grpc-grpc-testing-1.72.0.jar [32] +- lib/io.grpc-grpc-util-1.72.0.jar [32] +- lib/io.grpc-grpc-xds-1.72.0.jar [32] +- lib/io.grpc-grpc-rls-1.72.0.jar[32] +- lib/io.grpc-grpc-gcp-csm-observability-1.72.0.jar [32] - lib/org.apache.curator-curator-client-5.7.1.jar [33] - lib/org.apache.curator-curator-framework-5.7.1.jar [33] - lib/org.apache.curator-curator-recipes-5.7.1.jar [33] @@ -281,7 +285,7 @@ Apache Software License, Version 2. - lib/com.google.http-client-google-http-client-1.41.0.jar [43] - lib/com.google.http-client-google-http-client-gson-1.41.0.jar [43] - lib/com.google.j2objc-j2objc-annotations-2.8.jar [44] -- lib/com.google.re2j-re2j-1.6.jar [45] +- lib/com.google.re2j-re2j-1.8.jar [45] - lib/io.dropwizard.metrics-metrics-core-4.1.12.1.jar [46] - lib/io.perfmark-perfmark-api-0.25.0.jar [47] - lib/org.conscrypt-conscrypt-openjdk-uber-2.5.2.jar [49] @@ -309,7 +313,7 @@ Apache Software License, Version 2. [27] Source available at https://github.com/googleapis/java-common-protos/tree/v2.17.0 [28] Source available at https://github.com/google/gson/tree/gson-parent-2.10.1 [29] Source available at https://github.com/census-instrumentation/opencensus-java/tree/v0.28.0 -[32] Source available at https://github.com/grpc/grpc-java/tree/v1.56.0 +[32] Source available at https://github.com/grpc/grpc-java/tree/v1.72.0 [33] Source available at https://github.com/apache/curator/tree/apache-curator-5.7.1 [35] Source available at https://github.com/google/error-prone/tree/v2.9.0 [36] Source available at https://github.com/apache/yetus/tree/rel/0.12.0 @@ -321,7 +325,7 @@ Apache Software License, Version 2. [42] Source available at https://github.com/google/auto/releases/tag/auto-value-1.10.1 [43] Source available at https://github.com/googleapis/google-http-java-client/releases/tag/v1.41.0 [44] Source available at https://github.com/google/j2objc/releases/tag/1.3 -[45] Source available at https://github.com/google/re2j/releases/tag/re2j-1.7 +[45] Source available at https://github.com/google/re2j/releases/tag/re2j-1.8 [46] Source available at https://github.com/dropwizard/metrics/releases/tag/v4.1.12.1 [47] Source available at https://github.com/perfmark/perfmark/releases/tag/v0.26.0 [49] Source available at https://github.com/google/conscrypt/releases/tag/2.5.2 diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt index a6fa25a9cbe..96436826ee1 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt @@ -281,21 +281,25 @@ Apache Software License, Version 2. - lib/io.opencensus-opencensus-api-0.28.0.jar [30] - lib/io.opencensus-opencensus-contrib-http-util-0.28.0.jar [30] - lib/io.opencensus-opencensus-proto-0.2.0.jar [30] -- lib/io.grpc-grpc-all-1.54.1.jar [33] -- lib/io.grpc-grpc-alts-1.54.1.jar [33] -- lib/io.grpc-grpc-api-1.54.1.jar [33] -- lib/io.grpc-grpc-auth-1.54.1.jar [33] -- lib/io.grpc-grpc-context-1.54.1.jar [33] -- lib/io.grpc-grpc-core-1.54.1.jar [33] -- lib/io.grpc-grpc-grpclb-1.54.1.jar [33] -- lib/io.grpc-grpc-protobuf-1.54.1.jar [33] -- lib/io.grpc-grpc-protobuf-lite-1.54.1.jar [33] -- lib/io.grpc-grpc-netty-shaded-1.54.1.jar [33] -- lib/io.grpc-grpc-services-1.54.1.jar [33] -- lib/io.grpc-grpc-stub-1.54.1.jar [33] -- lib/io.grpc-grpc-testing-1.54.1.jar [33] -- lib/io.grpc-grpc-xds-1.54.1.jar [33] -- lib/io.grpc-grpc-rls-1.54.1.jar[33] +- lib/io.grpc-grpc-all-1.72.0.jar [33] +- lib/io.grpc-grpc-alts-1.72.0.jar [33] +- lib/io.grpc-grpc-api-1.72.0.jar [33] +- lib/io.grpc-grpc-auth-1.72.0.jar [33] +- lib/io.grpc-grpc-context-1.72.0.jar [33] +- lib/io.grpc-grpc-core-1.72.0.jar [33] +- lib/io.grpc-grpc-grpclb-1.72.0.jar [33] +- lib/io.grpc-grpc-inprocess-1.72.0.jar [33] +- lib/io.grpc-grpc-opentelemetry-1.72.0.jar [33] +- lib/io.grpc-grpc-netty-shaded-1.72.0.jar [33] +- lib/io.grpc-grpc-protobuf-1.72.0.jar [33] +- lib/io.grpc-grpc-protobuf-lite-1.72.0.jar [33] +- lib/io.grpc-grpc-services-1.72.0.jar [33] +- lib/io.grpc-grpc-stub-1.72.0.jar [33] +- lib/io.grpc-grpc-testing-1.72.0.jar [33] +- lib/io.grpc-grpc-util-1.72.0.jar [33] +- lib/io.grpc-grpc-xds-1.72.0.jar [33] +- lib/io.grpc-grpc-rls-1.72.0.jar[33] +- lib/io.grpc-grpc-gcp-csm-observability-1.72.0.jar [33] - lib/org.apache.curator-curator-client-5.7.1.jar [34] - lib/org.apache.curator-curator-framework-5.7.1.jar [34] - lib/org.apache.curator-curator-recipes-5.7.1.jar [34] @@ -310,7 +314,7 @@ Apache Software License, Version 2. - lib/com.google.http-client-google-http-client-gson-1.41.0.jar [43] - lib/com.google.auto.value-auto-value-annotations-1.9.jar [44] - lib/com.google.j2objc-j2objc-annotations-2.8.jar [45] -- lib/com.google.re2j-re2j-1.6.jar [46] +- lib/com.google.re2j-re2j-1.8.jar [46] - lib/io.dropwizard.metrics-metrics-core-4.1.12.1.jar [47] - lib/io.perfmark-perfmark-api-0.25.0.jar [48] - lib/org.conscrypt-conscrypt-openjdk-uber-2.5.2.jar [49] @@ -346,7 +350,7 @@ Apache Software License, Version 2. [28] Source available at https://github.com/googleapis/java-common-protos/tree/v2.17.0 [29] Source available at https://github.com/google/gson/tree/gson-parent-2.10.1 [30] Source available at https://github.com/census-instrumentation/opencensus-java/tree/v0.28.0 -[33] Source available at https://github.com/grpc/grpc-java/tree/v1.56.0 +[33] Source available at https://github.com/grpc/grpc-java/tree/v1.72.0 [34] Source available at https://github.com/apache/curator/releases/tag/apache.curator-5.7.1 [36] Source available at https://github.com/google/error-prone/tree/v2.9.0 [37] Source available at https://github.com/apache/yetus/tree/rel/0.12.0 @@ -358,7 +362,7 @@ Apache Software License, Version 2. [43] Source available at https://github.com/googleapis/google-http-java-client/releases/tag/v1.41.0 [44] Source available at https://github.com/google/auto/releases/tag/auto-value-1.10.1 [45] Source available at https://github.com/google/j2objc/releases/tag/1.3 -[46] Source available at https://github.com/google/re2j/releases/tag/re2j-1.7 +[46] Source available at https://github.com/google/re2j/releases/tag/re2j-1.8 [47] Source available at https://github.com/dropwizard/metrics/releases/tag/v4.1.12.1 [48] Source available at https://github.com/perfmark/perfmark/releases/tag/v0.26.0 [49] Source available at https://github.com/google/conscrypt/releases/tag/2.5.2 diff --git a/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt index 168d4a1ff59..71db5d93929 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt @@ -135,15 +135,15 @@ granted provided that the copyright notice appears in all copies. Copyright 2010 Cedric Beust cedric@beust.com ------------------------------------------------------------------------------------ -- lib/io.grpc-grpc-all-1.54.1.jar -- lib/io.grpc-grpc-auth-1.54.1.jar -- lib/io.grpc-grpc-context-1.54.1.jar -- lib/io.grpc-grpc-core-1.54.1.jar -- lib/io.grpc-grpc-netty-shaded-1.54.1.jar -- lib/io.grpc-grpc-protobuf-1.54.1.jar -- lib/io.grpc-grpc-protobuf-lite-1.54.1.jar -- lib/io.grpc-grpc-stub-1.54.1.jar -- lib/io.grpc-grpc-testing-1.54.1.jar +- lib/io.grpc-grpc-all-1.72.0.jar +- lib/io.grpc-grpc-auth-1.72.0.jar +- lib/io.grpc-grpc-context-1.72.0.jar +- lib/io.grpc-grpc-core-1.72.0.jar +- lib/io.grpc-grpc-netty-shaded-1.72.0.jar +- lib/io.grpc-grpc-protobuf-1.72.0.jar +- lib/io.grpc-grpc-protobuf-lite-1.72.0.jar +- lib/io.grpc-grpc-stub-1.72.0.jar +- lib/io.grpc-grpc-testing-1.72.0.jar Copyright 2014, gRPC Authors All rights reserved. diff --git a/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt index 47d5c6a522e..935ba8ff657 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt +++ b/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt @@ -53,15 +53,15 @@ under the License. Copyright 2010 Cedric Beust cedric@beust.com ------------------------------------------------------------------------------------ -- lib/io.grpc-grpc-all-1.54.1.jar -- lib/io.grpc-grpc-auth-1.54.1.jar -- lib/io.grpc-grpc-context-1.54.1.jar -- lib/io.grpc-grpc-core-1.54.1.jar -- lib/io.grpc-grpc-netty-shaded-1.54.1.jar -- lib/io.grpc-grpc-protobuf-1.54.1.jar -- lib/io.grpc-grpc-protobuf-lite-1.54.1.jar -- lib/io.grpc-grpc-stub-1.54.1.jar -- lib/io.grpc-grpc-testing-1.54.1.jar +- lib/io.grpc-grpc-all-1.72.0.jar +- lib/io.grpc-grpc-auth-1.72.0.jar +- lib/io.grpc-grpc-context-1.72.0.jar +- lib/io.grpc-grpc-core-1.72.0.jar +- lib/io.grpc-grpc-netty-shaded-1.72.0.jar +- lib/io.grpc-grpc-protobuf-1.72.0.jar +- lib/io.grpc-grpc-protobuf-lite-1.72.0.jar +- lib/io.grpc-grpc-stub-1.72.0.jar +- lib/io.grpc-grpc-testing-1.72.0.jar Copyright 2014, gRPC Authors All rights reserved. diff --git a/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt index 210718ee9fe..7c93a8c2475 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt @@ -117,15 +117,15 @@ granted provided that the copyright notice appears in all copies. Copyright 2010 Cedric Beust cedric@beust.com ------------------------------------------------------------------------------------ -- lib/io.grpc-grpc-all-1.54.1.jar -- lib/io.grpc-grpc-auth-1.54.1.jar -- lib/io.grpc-grpc-context-1.54.1.jar -- lib/io.grpc-grpc-core-1.54.1.jar -- lib/io.grpc-grpc-netty-shaded-1.54.1.jar -- lib/io.grpc-grpc-protobuf-1.54.1.jar -- lib/io.grpc-grpc-protobuf-lite-1.54.1.jar -- lib/io.grpc-grpc-stub-1.54.1.jar -- lib/io.grpc-grpc-testing-1.54.1.jar +- lib/io.grpc-grpc-all-1.72.0.jar +- lib/io.grpc-grpc-auth-1.72.0.jar +- lib/io.grpc-grpc-context-1.72.0.jar +- lib/io.grpc-grpc-core-1.72.0.jar +- lib/io.grpc-grpc-netty-shaded-1.72.0.jar +- lib/io.grpc-grpc-protobuf-1.72.0.jar +- lib/io.grpc-grpc-protobuf-lite-1.72.0.jar +- lib/io.grpc-grpc-stub-1.72.0.jar +- lib/io.grpc-grpc-testing-1.72.0.jar Copyright 2014, gRPC Authors All rights reserved. diff --git a/pom.xml b/pom.xml index f4de5687930..11b7090ee59 100644 --- a/pom.xml +++ b/pom.xml @@ -133,7 +133,7 @@ 2.8.0 3.0.2 2.9.0 - 1.54.1 + 1.72.0 32.0.1-jre 1.1.1 3.3.5 From f7ea01222d419fe8c8db8df0c88394ee1f0d01be Mon Sep 17 00:00:00 2001 From: manas-ctds Date: Fri, 27 Feb 2026 18:01:37 +0530 Subject: [PATCH 53/55] Revert "Upgrade to grpc 1.72.0 (#4591)" This reverts commit 4a471bdfe7050364024659d0f52880608dbef1fb. --- .../src/main/resources/LICENSE-all.bin.txt | 40 +++++++++---------- .../src/main/resources/LICENSE-bkctl.bin.txt | 40 +++++++++---------- .../src/main/resources/LICENSE-server.bin.txt | 40 +++++++++---------- .../src/main/resources/NOTICE-all.bin.txt | 18 ++++----- .../src/main/resources/NOTICE-bkctl.bin.txt | 18 ++++----- .../src/main/resources/NOTICE-server.bin.txt | 18 ++++----- pom.xml | 2 +- 7 files changed, 82 insertions(+), 94 deletions(-) diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index dfd23e5c09b..790e67297fd 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -281,25 +281,21 @@ Apache Software License, Version 2. - lib/io.opencensus-opencensus-api-0.28.0.jar [30] - lib/io.opencensus-opencensus-contrib-http-util-0.28.0.jar [30] - lib/io.opencensus-opencensus-proto-0.2.0.jar [30] -- lib/io.grpc-grpc-all-1.72.0.jar [33] -- lib/io.grpc-grpc-alts-1.72.0.jar [33] -- lib/io.grpc-grpc-api-1.72.0.jar [33] -- lib/io.grpc-grpc-auth-1.72.0.jar [33] -- lib/io.grpc-grpc-context-1.72.0.jar [33] -- lib/io.grpc-grpc-core-1.72.0.jar [33] -- lib/io.grpc-grpc-grpclb-1.72.0.jar [33] -- lib/io.grpc-grpc-inprocess-1.72.0.jar [33] -- lib/io.grpc-grpc-opentelemetry-1.72.0.jar [33] -- lib/io.grpc-grpc-netty-shaded-1.72.0.jar [33] -- lib/io.grpc-grpc-protobuf-1.72.0.jar [33] -- lib/io.grpc-grpc-protobuf-lite-1.72.0.jar [33] -- lib/io.grpc-grpc-services-1.72.0.jar [33] -- lib/io.grpc-grpc-stub-1.72.0.jar [33] -- lib/io.grpc-grpc-testing-1.72.0.jar [33] -- lib/io.grpc-grpc-util-1.72.0.jar [33] -- lib/io.grpc-grpc-xds-1.72.0.jar [33] -- lib/io.grpc-grpc-rls-1.72.0.jar[33] -- lib/io.grpc-grpc-gcp-csm-observability-1.72.0.jar [33] +- lib/io.grpc-grpc-all-1.54.1.jar [33] +- lib/io.grpc-grpc-alts-1.54.1.jar [33] +- lib/io.grpc-grpc-api-1.54.1.jar [33] +- lib/io.grpc-grpc-auth-1.54.1.jar [33] +- lib/io.grpc-grpc-context-1.54.1.jar [33] +- lib/io.grpc-grpc-core-1.54.1.jar [33] +- lib/io.grpc-grpc-grpclb-1.54.1.jar [33] +- lib/io.grpc-grpc-protobuf-1.54.1.jar [33] +- lib/io.grpc-grpc-protobuf-lite-1.54.1.jar [33] +- lib/io.grpc-grpc-netty-shaded-1.54.1.jar [33] +- lib/io.grpc-grpc-services-1.54.1.jar [33] +- lib/io.grpc-grpc-stub-1.54.1.jar [33] +- lib/io.grpc-grpc-testing-1.54.1.jar [33] +- lib/io.grpc-grpc-xds-1.54.1.jar [33] +- lib/io.grpc-grpc-rls-1.54.1.jar[33] - lib/org.apache.curator-curator-client-5.7.1.jar [34] - lib/org.apache.curator-curator-framework-5.7.1.jar [34] - lib/org.apache.curator-curator-recipes-5.7.1.jar [34] @@ -314,7 +310,7 @@ Apache Software License, Version 2. - lib/com.google.http-client-google-http-client-gson-1.41.0.jar [43] - lib/com.google.auto.value-auto-value-annotations-1.9.jar [44] - lib/com.google.j2objc-j2objc-annotations-2.8.jar [45] -- lib/com.google.re2j-re2j-1.8.jar [46] +- lib/com.google.re2j-re2j-1.6.jar [46] - lib/io.dropwizard.metrics-metrics-core-4.1.12.1.jar [47] - lib/io.dropwizard.metrics-metrics-graphite-4.1.12.1.jar [47] - lib/io.dropwizard.metrics-metrics-jmx-4.1.12.1.jar [47] @@ -354,7 +350,7 @@ Apache Software License, Version 2. [28] Source available at https://github.com/googleapis/java-common-protos/tree/v2.17.0 [29] Source available at https://github.com/google/gson/tree/gson-parent-2.10.1 [30] Source available at https://github.com/census-instrumentation/opencensus-java/tree/v0.28.0 -[33] Source available at https://github.com/grpc/grpc-java/tree/v1.72.0 +[33] Source available at https://github.com/grpc/grpc-java/tree/v1.56.0 [34] Source available at https://github.com/apache/curator/releases/tag/apache.curator-5.7.1 [36] Source available at https://github.com/google/error-prone/tree/v2.9.0 [37] Source available at https://github.com/apache/yetus/tree/rel/0.12.0 @@ -366,7 +362,7 @@ Apache Software License, Version 2. [43] Source available at https://github.com/googleapis/google-http-java-client/releases/tag/v1.41.0 [44] Source available at https://github.com/google/auto/releases/tag/auto-value-1.10.1 [45] Source available at https://github.com/google/j2objc/releases/tag/1.3 -[46] Source available at https://github.com/google/re2j/releases/tag/re2j-1.8 +[46] Source available at https://github.com/google/re2j/releases/tag/re2j-1.7 [47] Source available at https://github.com/dropwizard/metrics/releases/tag/v4.1.12.1 [48] Source available at https://github.com/perfmark/perfmark/releases/tag/v0.26.0 [49] Source available at https://github.com/google/conscrypt/releases/tag/2.5.2 diff --git a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt index e8d50b6b0e0..9ca54cc559b 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt @@ -252,25 +252,21 @@ Apache Software License, Version 2. - lib/io.opencensus-opencensus-api-0.28.0.jar [29] - lib/io.opencensus-opencensus-contrib-http-util-0.28.0.jar [29] - lib/io.opencensus-opencensus-proto-0.2.0.jar [29] -- lib/io.grpc-grpc-all-1.72.0.jar [32] -- lib/io.grpc-grpc-alts-1.72.0.jar [32] -- lib/io.grpc-grpc-api-1.72.0.jar [32] -- lib/io.grpc-grpc-auth-1.72.0.jar [32] -- lib/io.grpc-grpc-context-1.72.0.jar [32] -- lib/io.grpc-grpc-core-1.72.0.jar [32] -- lib/io.grpc-grpc-grpclb-1.72.0.jar [32] -- lib/io.grpc-grpc-inprocess-1.72.0.jar [32] -- lib/io.grpc-grpc-opentelemetry-1.72.0.jar [32] -- lib/io.grpc-grpc-netty-shaded-1.72.0.jar [32] -- lib/io.grpc-grpc-protobuf-1.72.0.jar [32] -- lib/io.grpc-grpc-protobuf-lite-1.72.0.jar [32] -- lib/io.grpc-grpc-services-1.72.0.jar [32] -- lib/io.grpc-grpc-stub-1.72.0.jar [32] -- lib/io.grpc-grpc-testing-1.72.0.jar [32] -- lib/io.grpc-grpc-util-1.72.0.jar [32] -- lib/io.grpc-grpc-xds-1.72.0.jar [32] -- lib/io.grpc-grpc-rls-1.72.0.jar[32] -- lib/io.grpc-grpc-gcp-csm-observability-1.72.0.jar [32] +- lib/io.grpc-grpc-all-1.54.1.jar [32] +- lib/io.grpc-grpc-alts-1.54.1.jar [32] +- lib/io.grpc-grpc-api-1.54.1.jar [32] +- lib/io.grpc-grpc-auth-1.54.1.jar [32] +- lib/io.grpc-grpc-context-1.54.1.jar [32] +- lib/io.grpc-grpc-core-1.54.1.jar [32] +- lib/io.grpc-grpc-grpclb-1.54.1.jar [32] +- lib/io.grpc-grpc-protobuf-1.54.1.jar [32] +- lib/io.grpc-grpc-protobuf-lite-1.54.1.jar [32] +- lib/io.grpc-grpc-netty-shaded-1.54.1.jar [32] +- lib/io.grpc-grpc-services-1.54.1.jar [32] +- lib/io.grpc-grpc-stub-1.54.1.jar [32] +- lib/io.grpc-grpc-testing-1.54.1.jar [32] +- lib/io.grpc-grpc-xds-1.54.1.jar [32] +- lib/io.grpc-grpc-rls-1.54.1.jar[32] - lib/org.apache.curator-curator-client-5.7.1.jar [33] - lib/org.apache.curator-curator-framework-5.7.1.jar [33] - lib/org.apache.curator-curator-recipes-5.7.1.jar [33] @@ -285,7 +281,7 @@ Apache Software License, Version 2. - lib/com.google.http-client-google-http-client-1.41.0.jar [43] - lib/com.google.http-client-google-http-client-gson-1.41.0.jar [43] - lib/com.google.j2objc-j2objc-annotations-2.8.jar [44] -- lib/com.google.re2j-re2j-1.8.jar [45] +- lib/com.google.re2j-re2j-1.6.jar [45] - lib/io.dropwizard.metrics-metrics-core-4.1.12.1.jar [46] - lib/io.perfmark-perfmark-api-0.25.0.jar [47] - lib/org.conscrypt-conscrypt-openjdk-uber-2.5.2.jar [49] @@ -313,7 +309,7 @@ Apache Software License, Version 2. [27] Source available at https://github.com/googleapis/java-common-protos/tree/v2.17.0 [28] Source available at https://github.com/google/gson/tree/gson-parent-2.10.1 [29] Source available at https://github.com/census-instrumentation/opencensus-java/tree/v0.28.0 -[32] Source available at https://github.com/grpc/grpc-java/tree/v1.72.0 +[32] Source available at https://github.com/grpc/grpc-java/tree/v1.56.0 [33] Source available at https://github.com/apache/curator/tree/apache-curator-5.7.1 [35] Source available at https://github.com/google/error-prone/tree/v2.9.0 [36] Source available at https://github.com/apache/yetus/tree/rel/0.12.0 @@ -325,7 +321,7 @@ Apache Software License, Version 2. [42] Source available at https://github.com/google/auto/releases/tag/auto-value-1.10.1 [43] Source available at https://github.com/googleapis/google-http-java-client/releases/tag/v1.41.0 [44] Source available at https://github.com/google/j2objc/releases/tag/1.3 -[45] Source available at https://github.com/google/re2j/releases/tag/re2j-1.8 +[45] Source available at https://github.com/google/re2j/releases/tag/re2j-1.7 [46] Source available at https://github.com/dropwizard/metrics/releases/tag/v4.1.12.1 [47] Source available at https://github.com/perfmark/perfmark/releases/tag/v0.26.0 [49] Source available at https://github.com/google/conscrypt/releases/tag/2.5.2 diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt index 96436826ee1..a6fa25a9cbe 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt @@ -281,25 +281,21 @@ Apache Software License, Version 2. - lib/io.opencensus-opencensus-api-0.28.0.jar [30] - lib/io.opencensus-opencensus-contrib-http-util-0.28.0.jar [30] - lib/io.opencensus-opencensus-proto-0.2.0.jar [30] -- lib/io.grpc-grpc-all-1.72.0.jar [33] -- lib/io.grpc-grpc-alts-1.72.0.jar [33] -- lib/io.grpc-grpc-api-1.72.0.jar [33] -- lib/io.grpc-grpc-auth-1.72.0.jar [33] -- lib/io.grpc-grpc-context-1.72.0.jar [33] -- lib/io.grpc-grpc-core-1.72.0.jar [33] -- lib/io.grpc-grpc-grpclb-1.72.0.jar [33] -- lib/io.grpc-grpc-inprocess-1.72.0.jar [33] -- lib/io.grpc-grpc-opentelemetry-1.72.0.jar [33] -- lib/io.grpc-grpc-netty-shaded-1.72.0.jar [33] -- lib/io.grpc-grpc-protobuf-1.72.0.jar [33] -- lib/io.grpc-grpc-protobuf-lite-1.72.0.jar [33] -- lib/io.grpc-grpc-services-1.72.0.jar [33] -- lib/io.grpc-grpc-stub-1.72.0.jar [33] -- lib/io.grpc-grpc-testing-1.72.0.jar [33] -- lib/io.grpc-grpc-util-1.72.0.jar [33] -- lib/io.grpc-grpc-xds-1.72.0.jar [33] -- lib/io.grpc-grpc-rls-1.72.0.jar[33] -- lib/io.grpc-grpc-gcp-csm-observability-1.72.0.jar [33] +- lib/io.grpc-grpc-all-1.54.1.jar [33] +- lib/io.grpc-grpc-alts-1.54.1.jar [33] +- lib/io.grpc-grpc-api-1.54.1.jar [33] +- lib/io.grpc-grpc-auth-1.54.1.jar [33] +- lib/io.grpc-grpc-context-1.54.1.jar [33] +- lib/io.grpc-grpc-core-1.54.1.jar [33] +- lib/io.grpc-grpc-grpclb-1.54.1.jar [33] +- lib/io.grpc-grpc-protobuf-1.54.1.jar [33] +- lib/io.grpc-grpc-protobuf-lite-1.54.1.jar [33] +- lib/io.grpc-grpc-netty-shaded-1.54.1.jar [33] +- lib/io.grpc-grpc-services-1.54.1.jar [33] +- lib/io.grpc-grpc-stub-1.54.1.jar [33] +- lib/io.grpc-grpc-testing-1.54.1.jar [33] +- lib/io.grpc-grpc-xds-1.54.1.jar [33] +- lib/io.grpc-grpc-rls-1.54.1.jar[33] - lib/org.apache.curator-curator-client-5.7.1.jar [34] - lib/org.apache.curator-curator-framework-5.7.1.jar [34] - lib/org.apache.curator-curator-recipes-5.7.1.jar [34] @@ -314,7 +310,7 @@ Apache Software License, Version 2. - lib/com.google.http-client-google-http-client-gson-1.41.0.jar [43] - lib/com.google.auto.value-auto-value-annotations-1.9.jar [44] - lib/com.google.j2objc-j2objc-annotations-2.8.jar [45] -- lib/com.google.re2j-re2j-1.8.jar [46] +- lib/com.google.re2j-re2j-1.6.jar [46] - lib/io.dropwizard.metrics-metrics-core-4.1.12.1.jar [47] - lib/io.perfmark-perfmark-api-0.25.0.jar [48] - lib/org.conscrypt-conscrypt-openjdk-uber-2.5.2.jar [49] @@ -350,7 +346,7 @@ Apache Software License, Version 2. [28] Source available at https://github.com/googleapis/java-common-protos/tree/v2.17.0 [29] Source available at https://github.com/google/gson/tree/gson-parent-2.10.1 [30] Source available at https://github.com/census-instrumentation/opencensus-java/tree/v0.28.0 -[33] Source available at https://github.com/grpc/grpc-java/tree/v1.72.0 +[33] Source available at https://github.com/grpc/grpc-java/tree/v1.56.0 [34] Source available at https://github.com/apache/curator/releases/tag/apache.curator-5.7.1 [36] Source available at https://github.com/google/error-prone/tree/v2.9.0 [37] Source available at https://github.com/apache/yetus/tree/rel/0.12.0 @@ -362,7 +358,7 @@ Apache Software License, Version 2. [43] Source available at https://github.com/googleapis/google-http-java-client/releases/tag/v1.41.0 [44] Source available at https://github.com/google/auto/releases/tag/auto-value-1.10.1 [45] Source available at https://github.com/google/j2objc/releases/tag/1.3 -[46] Source available at https://github.com/google/re2j/releases/tag/re2j-1.8 +[46] Source available at https://github.com/google/re2j/releases/tag/re2j-1.7 [47] Source available at https://github.com/dropwizard/metrics/releases/tag/v4.1.12.1 [48] Source available at https://github.com/perfmark/perfmark/releases/tag/v0.26.0 [49] Source available at https://github.com/google/conscrypt/releases/tag/2.5.2 diff --git a/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt index 71db5d93929..168d4a1ff59 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt @@ -135,15 +135,15 @@ granted provided that the copyright notice appears in all copies. Copyright 2010 Cedric Beust cedric@beust.com ------------------------------------------------------------------------------------ -- lib/io.grpc-grpc-all-1.72.0.jar -- lib/io.grpc-grpc-auth-1.72.0.jar -- lib/io.grpc-grpc-context-1.72.0.jar -- lib/io.grpc-grpc-core-1.72.0.jar -- lib/io.grpc-grpc-netty-shaded-1.72.0.jar -- lib/io.grpc-grpc-protobuf-1.72.0.jar -- lib/io.grpc-grpc-protobuf-lite-1.72.0.jar -- lib/io.grpc-grpc-stub-1.72.0.jar -- lib/io.grpc-grpc-testing-1.72.0.jar +- lib/io.grpc-grpc-all-1.54.1.jar +- lib/io.grpc-grpc-auth-1.54.1.jar +- lib/io.grpc-grpc-context-1.54.1.jar +- lib/io.grpc-grpc-core-1.54.1.jar +- lib/io.grpc-grpc-netty-shaded-1.54.1.jar +- lib/io.grpc-grpc-protobuf-1.54.1.jar +- lib/io.grpc-grpc-protobuf-lite-1.54.1.jar +- lib/io.grpc-grpc-stub-1.54.1.jar +- lib/io.grpc-grpc-testing-1.54.1.jar Copyright 2014, gRPC Authors All rights reserved. diff --git a/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt index 935ba8ff657..47d5c6a522e 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt +++ b/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt @@ -53,15 +53,15 @@ under the License. Copyright 2010 Cedric Beust cedric@beust.com ------------------------------------------------------------------------------------ -- lib/io.grpc-grpc-all-1.72.0.jar -- lib/io.grpc-grpc-auth-1.72.0.jar -- lib/io.grpc-grpc-context-1.72.0.jar -- lib/io.grpc-grpc-core-1.72.0.jar -- lib/io.grpc-grpc-netty-shaded-1.72.0.jar -- lib/io.grpc-grpc-protobuf-1.72.0.jar -- lib/io.grpc-grpc-protobuf-lite-1.72.0.jar -- lib/io.grpc-grpc-stub-1.72.0.jar -- lib/io.grpc-grpc-testing-1.72.0.jar +- lib/io.grpc-grpc-all-1.54.1.jar +- lib/io.grpc-grpc-auth-1.54.1.jar +- lib/io.grpc-grpc-context-1.54.1.jar +- lib/io.grpc-grpc-core-1.54.1.jar +- lib/io.grpc-grpc-netty-shaded-1.54.1.jar +- lib/io.grpc-grpc-protobuf-1.54.1.jar +- lib/io.grpc-grpc-protobuf-lite-1.54.1.jar +- lib/io.grpc-grpc-stub-1.54.1.jar +- lib/io.grpc-grpc-testing-1.54.1.jar Copyright 2014, gRPC Authors All rights reserved. diff --git a/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt index 7c93a8c2475..210718ee9fe 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt @@ -117,15 +117,15 @@ granted provided that the copyright notice appears in all copies. Copyright 2010 Cedric Beust cedric@beust.com ------------------------------------------------------------------------------------ -- lib/io.grpc-grpc-all-1.72.0.jar -- lib/io.grpc-grpc-auth-1.72.0.jar -- lib/io.grpc-grpc-context-1.72.0.jar -- lib/io.grpc-grpc-core-1.72.0.jar -- lib/io.grpc-grpc-netty-shaded-1.72.0.jar -- lib/io.grpc-grpc-protobuf-1.72.0.jar -- lib/io.grpc-grpc-protobuf-lite-1.72.0.jar -- lib/io.grpc-grpc-stub-1.72.0.jar -- lib/io.grpc-grpc-testing-1.72.0.jar +- lib/io.grpc-grpc-all-1.54.1.jar +- lib/io.grpc-grpc-auth-1.54.1.jar +- lib/io.grpc-grpc-context-1.54.1.jar +- lib/io.grpc-grpc-core-1.54.1.jar +- lib/io.grpc-grpc-netty-shaded-1.54.1.jar +- lib/io.grpc-grpc-protobuf-1.54.1.jar +- lib/io.grpc-grpc-protobuf-lite-1.54.1.jar +- lib/io.grpc-grpc-stub-1.54.1.jar +- lib/io.grpc-grpc-testing-1.54.1.jar Copyright 2014, gRPC Authors All rights reserved. diff --git a/pom.xml b/pom.xml index 11b7090ee59..f4de5687930 100644 --- a/pom.xml +++ b/pom.xml @@ -133,7 +133,7 @@ 2.8.0 3.0.2 2.9.0 - 1.72.0 + 1.54.1 32.0.1-jre 1.1.1 3.3.5 From 0da907546bb41f02c94c6786ad6e9f2f052c88ee Mon Sep 17 00:00:00 2001 From: priyanshu-ctds Date: Mon, 21 Jul 2025 17:28:19 +0530 Subject: [PATCH 54/55] fix cherry-pick : corrected required module names and removed dead link (cherry picked from commit dca89a720a77e7e59663b3cd737cf7ba6fe1c06a) --- bin/common.sh | 4 ++-- stream/README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/common.sh b/bin/common.sh index 72779396a63..2f474a0382b 100755 --- a/bin/common.sh +++ b/bin/common.sh @@ -133,8 +133,8 @@ CLI_GC_OPTS=${CLI_GC_OPTS:-"${DEFAULT_CLI_GC_OPTS}"} CLI_GC_LOGGING_OPTS=${CLI_GC_LOGGING_OPTS:-"${DEFAULT_CLI_GC_LOGGING_OPTS}"} # module names -BOOKIE_SERVER_MODULE_NAME="(org.apache.bookkeeper-)?bookkeeper-server" -TABLE_SERVICE_MODULE_NAME="(org.apache.bookkeeper-)?stream-storage-server" +BOOKIE_SERVER_MODULE_NAME="(com.datastax.oss-)?bookkeeper-server" +TABLE_SERVICE_MODULE_NAME="(com.datastax.oss-)?stream-storage-server" is_released_binary() { if [ -d ${BK_HOME}/lib ]; then diff --git a/stream/README.md b/stream/README.md index 1f57950cd36..4b828bc3c85 100644 --- a/stream/README.md +++ b/stream/README.md @@ -4,7 +4,7 @@ BookKeeper Table Service is a contrib module added to BookKeeper, providing a ta ## Detail Design -[BP-30](https://docs.google.com/document/d/155xAwWv5IdOitHh1NVMEwCMGgB28M3FyMiQSxEpjE-Y/edit#heading=h.56rbh52koe3f) +[BP-30] ## Build From e06349665469931d15833ac2aabe58e7210a23b8 Mon Sep 17 00:00:00 2001 From: priyanshu-ctds Date: Tue, 22 Jul 2025 13:57:25 +0530 Subject: [PATCH 55/55] fix cherry-pick: corrected module names and removed dead link (cherry picked from commit 1d3c5f9b3867f22e844e6e95d9638a78c03c757c) --- bin/bkctl | 2 +- bin/bkperf | 2 +- bin/dlog | 2 +- site3/website/docs/api/ledger-api.md | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/bkctl b/bin/bkctl index 26d3b6ececa..1bd635cd610 100755 --- a/bin/bkctl +++ b/bin/bkctl @@ -28,7 +28,7 @@ source ${BK_HOME}/bin/common.sh source ${BK_HOME}/conf/bk_cli_env.sh CLI_MODULE_PATH=tools/all -CLI_MODULE_NAME="(org.apache.bookkeeper-)?bookkeeper-tools" +CLI_MODULE_NAME="(com.datastax.oss-)?bookkeeper-tools" CLI_MODULE_HOME=${BK_HOME}/${CLI_MODULE_PATH} # find the module jar diff --git a/bin/bkperf b/bin/bkperf index 02f8b20da27..d01f4ae143e 100755 --- a/bin/bkperf +++ b/bin/bkperf @@ -27,7 +27,7 @@ source ${BK_HOME}/bin/common.sh source ${BK_HOME}/conf/bk_cli_env.sh CLI_MODULE_PATH=tools/perf -CLI_MODULE_NAME="(org.apache.bookkeeper-)?bookkeeper-perf" +CLI_MODULE_NAME="(com.datastax.oss-)?bookkeeper-perf" CLI_MODULE_HOME=${BK_HOME}/${CLI_MODULE_PATH} # find the module jar diff --git a/bin/dlog b/bin/dlog index 2e8c2a32add..b2b681c1e6b 100755 --- a/bin/dlog +++ b/bin/dlog @@ -25,7 +25,7 @@ BK_HOME=`cd ${BINDIR}/..;pwd` source ${BK_HOME}/bin/common.sh -DLOG_MODULE_NAME="(org.apache.distributedlog-)?distributedlog-core" +DLOG_MODULE_NAME="(com.datastax.oss-)?distributedlog-core" DLOG_MODULE_PATH=stream/distributedlog/core DLOG_MODULE_HOME=${BK_HOME}/${DLOG_MODULE_PATH} diff --git a/site3/website/docs/api/ledger-api.md b/site3/website/docs/api/ledger-api.md index 9b565088231..6364b138d42 100644 --- a/site3/website/docs/api/ledger-api.md +++ b/site3/website/docs/api/ledger-api.md @@ -49,7 +49,7 @@ shaded library, which relocate classes of protobuf and guava into a different na ### Gradle -If you're using [Gradle](https://gradle.org/), add this to your [`build.gradle`](https://spring.io/guides/gs/gradle/) build configuration file: +If you're using [Gradle](https://gradle.org/), add this to your [`build.gradle`] build configuration file: ```groovy dependencies { @@ -105,7 +105,7 @@ There are, however, other ways that you can create a client object: BookKeeper bkClient = new BookKeeper(config); ``` -* By specifying a `ClientConfiguration` and a [`ZooKeeper`](http://zookeeper.apache.org/doc/current/api/org/apache/zookeeper/ZooKeeper.html) client object: +* By specifying a `ClientConfiguration` and a [`ZooKeeper`] client object: ```java ClientConfiguration config = new ClientConfiguration();