Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
186 changes: 92 additions & 94 deletions bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt

Large diffs are not rendered by default.

133 changes: 67 additions & 66 deletions bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt

Large diffs are not rendered by default.

186 changes: 92 additions & 94 deletions bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt

Large diffs are not rendered by default.

46 changes: 23 additions & 23 deletions bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt
Original file line number Diff line number Diff line change
Expand Up @@ -23,31 +23,31 @@ LongAdder), which was released with the following comments:
http://creativecommons.org/publicdomain/zero/1.0/

------------------------------------------------------------------------------------
- lib/io.netty-netty-buffer-4.1.121.Final.jar
- lib/io.netty-netty-codec-4.1.121.Final.jar
- lib/io.netty-netty-codec-dns-4.1.121.Final.jar
- lib/io.netty-netty-codec-http-4.1.121.Final.jar
- lib/io.netty-netty-codec-http2-4.1.121.Final.jar
- lib/io.netty-netty-codec-socks-4.1.121.Final.jar
- lib/io.netty-netty-common-4.1.121.Final.jar
- lib/io.netty-netty-handler-4.1.121.Final.jar
- lib/io.netty-netty-handler-proxy-4.1.121.Final.jar
- lib/io.netty-netty-resolver-4.1.121.Final.jar
- lib/io.netty-netty-resolver-dns-4.1.121.Final.jar
- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final.jar
- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-aarch_64.jar [11]
- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-x86_64.jar [11]
- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-aarch_64.jar [11]
- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-x86_64.jar [11]
- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-windows-x86_64.jar [11]
- lib/io.netty-netty-tcnative-classes-2.0.70.Final.jar
- lib/io.netty-netty-transport-4.1.121.Final.jar
- lib/io.netty-netty-transport-classes-epoll-4.1.121.Final.jar
- lib/io.netty-netty-transport-native-epoll-4.1.121.Final-linux-aarch_64.jar
- lib/io.netty-netty-transport-native-epoll-4.1.121.Final-linux-x86_64.jar
- lib/io.netty-netty-buffer-4.1.130.Final.jar
- lib/io.netty-netty-codec-4.1.130.Final.jar
- lib/io.netty-netty-codec-dns-4.1.130.Final.jar
- lib/io.netty-netty-codec-http-4.1.130.Final.jar
- lib/io.netty-netty-codec-http2-4.1.130.Final.jar
- lib/io.netty-netty-codec-socks-4.1.130.Final.jar
- lib/io.netty-netty-common-4.1.130.Final.jar
- lib/io.netty-netty-handler-4.1.130.Final.jar
- lib/io.netty-netty-handler-proxy-4.1.130.Final.jar
- lib/io.netty-netty-resolver-4.1.130.Final.jar
- lib/io.netty-netty-resolver-dns-4.1.130.Final.jar
- lib/io.netty-netty-tcnative-boringssl-static-2.0.74.Final.jar
- lib/io.netty-netty-tcnative-boringssl-static-2.0.74.Final-linux-aarch_64.jar [11]
- lib/io.netty-netty-tcnative-boringssl-static-2.0.74.Final-linux-x86_64.jar [11]
- lib/io.netty-netty-tcnative-boringssl-static-2.0.74.Final-osx-aarch_64.jar [11]
- lib/io.netty-netty-tcnative-boringssl-static-2.0.74.Final-osx-x86_64.jar [11]
- lib/io.netty-netty-tcnative-boringssl-static-2.0.74.Final-windows-x86_64.jar [11]
- lib/io.netty-netty-tcnative-classes-2.0.74.Final.jar
- lib/io.netty-netty-transport-4.1.130.Final.jar
- lib/io.netty-netty-transport-classes-epoll-4.1.130.Final.jar
- lib/io.netty-netty-transport-native-epoll-4.1.130.Final-linux-aarch_64.jar
- lib/io.netty-netty-transport-native-epoll-4.1.130.Final-linux-x86_64.jar
- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar
- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar
- lib/io.netty-netty-transport-native-unix-common-4.1.121.Final.jar
- lib/io.netty-netty-transport-native-unix-common-4.1.130.Final.jar


The Netty Project
Expand Down
34 changes: 17 additions & 17 deletions bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt
Original file line number Diff line number Diff line change
Expand Up @@ -5,25 +5,25 @@ This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).

------------------------------------------------------------------------------------
- lib/io.netty-netty-buffer-4.1.121.Final.jar
- lib/io.netty-netty-codec-4.1.121.Final.jar
- lib/io.netty-netty-common-4.1.121.Final.jar
- lib/io.netty-netty-handler-4.1.121.Final.jar
- lib/io.netty-netty-resolver-4.1.121.Final.jar
- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final.jar
- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-aarch_64.jar [11]
- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-x86_64.jar [11]
- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-aarch_64.jar [11]
- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-x86_64.jar [11]
- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-windows-x86_64.jar [11]
- lib/io.netty-netty-tcnative-classes-2.0.70.Final.jar
- lib/io.netty-netty-transport-4.1.121.Final.jar
- lib/io.netty-netty-transport-classes-epoll-4.1.121.Final.jar
- lib/io.netty-netty-transport-native-epoll-4.1.121.Final-linux-aarch_64.jar
- lib/io.netty-netty-transport-native-epoll-4.1.121.Final-linux-x86_64.jar
- lib/io.netty-netty-buffer-4.1.130.Final.jar
- lib/io.netty-netty-codec-4.1.130.Final.jar
- lib/io.netty-netty-common-4.1.130.Final.jar
- lib/io.netty-netty-handler-4.1.130.Final.jar
- lib/io.netty-netty-resolver-4.1.130.Final.jar
- lib/io.netty-netty-tcnative-boringssl-static-2.0.74.Final.jar
- lib/io.netty-netty-tcnative-boringssl-static-2.0.74.Final-linux-aarch_64.jar [11]
- lib/io.netty-netty-tcnative-boringssl-static-2.0.74.Final-linux-x86_64.jar [11]
- lib/io.netty-netty-tcnative-boringssl-static-2.0.74.Final-osx-aarch_64.jar [11]
- lib/io.netty-netty-tcnative-boringssl-static-2.0.74.Final-osx-x86_64.jar [11]
- lib/io.netty-netty-tcnative-boringssl-static-2.0.74.Final-windows-x86_64.jar [11]
- lib/io.netty-netty-tcnative-classes-2.0.74.Final.jar
- lib/io.netty-netty-transport-4.1.130.Final.jar
- lib/io.netty-netty-transport-classes-epoll-4.1.130.Final.jar
- lib/io.netty-netty-transport-native-epoll-4.1.130.Final-linux-aarch_64.jar
- lib/io.netty-netty-transport-native-epoll-4.1.130.Final-linux-x86_64.jar
- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar
- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar
- lib/io.netty-netty-transport-native-unix-common-4.1.121.Final.jar
- lib/io.netty-netty-transport-native-unix-common-4.1.130.Final.jar


The Netty Project
Expand Down
46 changes: 23 additions & 23 deletions bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt
Original file line number Diff line number Diff line change
Expand Up @@ -5,31 +5,31 @@ This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).

------------------------------------------------------------------------------------
- lib/io.netty-netty-buffer-4.1.121.Final.jar
- lib/io.netty-netty-codec-4.1.121.Final.jar
- lib/io.netty-netty-codec-dns-4.1.121.Final.jar
- lib/io.netty-netty-codec-http-4.1.121.Final.jar
- lib/io.netty-netty-codec-http2-4.1.121.Final.jar
- lib/io.netty-netty-codec-socks-4.1.121.Final.jar
- lib/io.netty-netty-common-4.1.121.Final.jar
- lib/io.netty-netty-handler-4.1.121.Final.jar
- lib/io.netty-netty-handler-proxy-4.1.121.Final.jar
- lib/io.netty-netty-resolver-4.1.121.Final.jar
- lib/io.netty-netty-resolver-dns-4.1.121.Final.jar
- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final.jar
- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-aarch_64.jar [11]
- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-x86_64.jar [11]
- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-aarch_64.jar [11]
- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-x86_64.jar [11]
- lib/io.netty-netty-tcnative-boringssl-static-2.0.70.Final-windows-x86_64.jar [11]
- lib/io.netty-netty-tcnative-classes-2.0.70.Final.jar
- lib/io.netty-netty-transport-4.1.121.Final.jar
- lib/io.netty-netty-transport-classes-epoll-4.1.121.Final.jar
- lib/io.netty-netty-transport-native-epoll-4.1.121.Final-linux-aarch_64.jar
- lib/io.netty-netty-transport-native-epoll-4.1.121.Final-linux-x86_64.jar
- lib/io.netty-netty-buffer-4.1.130.Final.jar
- lib/io.netty-netty-codec-4.1.130.Final.jar
- lib/io.netty-netty-codec-dns-4.1.130.Final.jar
- lib/io.netty-netty-codec-http-4.1.130.Final.jar
- lib/io.netty-netty-codec-http2-4.1.130.Final.jar
- lib/io.netty-netty-codec-socks-4.1.130.Final.jar
- lib/io.netty-netty-common-4.1.130.Final.jar
- lib/io.netty-netty-handler-4.1.130.Final.jar
- lib/io.netty-netty-handler-proxy-4.1.130.Final.jar
- lib/io.netty-netty-resolver-4.1.130.Final.jar
- lib/io.netty-netty-resolver-dns-4.1.130.Final.jar
- lib/io.netty-netty-tcnative-boringssl-static-2.0.74.Final.jar
- lib/io.netty-netty-tcnative-boringssl-static-2.0.74.Final-linux-aarch_64.jar [11]
- lib/io.netty-netty-tcnative-boringssl-static-2.0.74.Final-linux-x86_64.jar [11]
- lib/io.netty-netty-tcnative-boringssl-static-2.0.74.Final-osx-aarch_64.jar [11]
- lib/io.netty-netty-tcnative-boringssl-static-2.0.74.Final-osx-x86_64.jar [11]
- lib/io.netty-netty-tcnative-boringssl-static-2.0.74.Final-windows-x86_64.jar [11]
- lib/io.netty-netty-tcnative-classes-2.0.74.Final.jar
- lib/io.netty-netty-transport-4.1.130.Final.jar
- lib/io.netty-netty-transport-classes-epoll-4.1.130.Final.jar
- lib/io.netty-netty-transport-native-epoll-4.1.130.Final-linux-aarch_64.jar
- lib/io.netty-netty-transport-native-epoll-4.1.130.Final-linux-x86_64.jar
- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar
- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar
- lib/io.netty-netty-transport-native-unix-common-4.1.121.Final.jar
- lib/io.netty-netty-transport-native-unix-common-4.1.130.Final.jar


The Netty Project
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,11 @@ public LedgerDescriptor getHandle(final long ledgerId, final byte[] masterKey, b
throw BookieException.create(BookieException.Code.LedgerFencedAndDeletedException);
}
handle = LedgerDescriptor.create(masterKey, ledgerId, ledgerStorage);
ledgers.putIfAbsent(ledgerId, handle);
LedgerDescriptor storedValue = ledgers.putIfAbsent(ledgerId, handle);
// If it has been modified by other thread, use the previous one.
if (storedValue != null && storedValue != handle) {
return getHandle(ledgerId, masterKey, journalReplay);
}
}

handle.checkAccess(masterKey);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import static org.apache.bookkeeper.common.concurrent.FutureUtils.result;

import com.google.common.collect.Sets;
import com.google.common.util.concurrent.RateLimiter;
import java.io.IOException;
import java.net.URI;
import java.util.List;
Expand Down Expand Up @@ -84,6 +85,7 @@ public class ScanAndCompareGarbageCollector implements GarbageCollector {
private int activeLedgerCounter;
private StatsLogger statsLogger;
private final int maxConcurrentRequests;
private final RateLimiter gcMetadataOpRateLimiter;

public ScanAndCompareGarbageCollector(LedgerManager ledgerManager, CompactableLedgerStorage ledgerStorage,
ServerConfiguration conf, StatsLogger statsLogger) throws IOException {
Expand All @@ -103,6 +105,7 @@ public ScanAndCompareGarbageCollector(LedgerManager ledgerManager, CompactableLe
enableGcOverReplicatedLedger, gcOverReplicatedLedgerIntervalMillis, maxConcurrentRequests);

verifyMetadataOnGc = conf.getVerifyMetadataOnGC();
this.gcMetadataOpRateLimiter = RateLimiter.create(conf.getGcMetadataOpRateLimit());

this.activeLedgerCounter = 0;
}
Expand Down Expand Up @@ -153,6 +156,7 @@ public void gc(GarbageCleaner garbageCleaner) {
Versioned<LedgerMetadata> metadata = null;
while (!done) {
start = end + 1;
gcMetadataOpRateLimiter.acquire();
if (ledgerRangeIterator.hasNext()) {
LedgerRange lRange = ledgerRangeIterator.next();
ledgersInMetadata = lRange.getLedgers();
Expand All @@ -175,6 +179,7 @@ public void gc(GarbageCleaner garbageCleaner) {
metadata = null;
int rc = BKException.Code.OK;
try {
gcMetadataOpRateLimiter.acquire();
metadata = result(ledgerManager.readLedgerMetadata(bkLid), zkOpTimeoutMs,
TimeUnit.MILLISECONDS);
} catch (BKException | TimeoutException e) {
Expand Down Expand Up @@ -236,6 +241,7 @@ private Set<Long> removeOverReplicatedledgers(Set<Long> bkActiveledgers, final G
// check ledger ensembles before creating lock nodes.
// this is to reduce the number of lock node creations and deletions in ZK.
// the ensemble check is done again after the lock node is created.
gcMetadataOpRateLimiter.acquire();
Versioned<LedgerMetadata> preCheckMetadata = ledgerManager.readLedgerMetadata(ledgerId).get();
if (!isNotBookieIncludedInLedgerEnsembles(preCheckMetadata)) {
latch.countDown();
Expand All @@ -261,6 +267,7 @@ private Set<Long> removeOverReplicatedledgers(Set<Long> bkActiveledgers, final G
// current bookie again and, in that case, we cannot remove the ledger from local storage
lum.acquireUnderreplicatedLedger(ledgerId);
semaphore.acquire();
gcMetadataOpRateLimiter.acquire();
ledgerManager.readLedgerMetadata(ledgerId)
.whenComplete((metadata, exception) -> {
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -449,7 +449,7 @@ boolean isFlushRequired() {
}

@VisibleForTesting
List<SingleDirectoryDbLedgerStorage> getLedgerStorageList() {
public List<SingleDirectoryDbLedgerStorage> getLedgerStorageList() {
return ledgerStorageList;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.PrimitiveIterator.OfLong;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ExecutorService;
Expand Down Expand Up @@ -149,6 +150,8 @@ protected Thread newThread(Runnable r, String name) {

private final Counter flushExecutorTime;
private final boolean singleLedgerDirs;
private final String ledgerBaseDir;
private final String indexBaseDir;

public SingleDirectoryDbLedgerStorage(ServerConfiguration conf, LedgerManager ledgerManager,
LedgerDirsManager ledgerDirsManager, LedgerDirsManager indexDirsManager,
Expand All @@ -158,8 +161,7 @@ public SingleDirectoryDbLedgerStorage(ServerConfiguration conf, LedgerManager le
throws IOException {
checkArgument(ledgerDirsManager.getAllLedgerDirs().size() == 1,
"Db implementation only allows for one storage dir");

String ledgerBaseDir = ledgerDirsManager.getAllLedgerDirs().get(0).getPath();
ledgerBaseDir = ledgerDirsManager.getAllLedgerDirs().get(0).getPath();
// indexBaseDir default use ledgerBaseDir
String indexBaseDir = ledgerBaseDir;
if (CollectionUtils.isEmpty(indexDirsManager.getAllLedgerDirs())
Expand All @@ -172,6 +174,7 @@ public SingleDirectoryDbLedgerStorage(ServerConfiguration conf, LedgerManager le
log.info("indexDir is specified a separate dir, creating single directory db ledger storage on {}",
indexBaseDir);
}
this.indexBaseDir = indexBaseDir;

StatsLogger ledgerIndexDirStatsLogger = statsLogger
.scopeLabel("ledgerDir", ledgerBaseDir)
Expand Down Expand Up @@ -228,9 +231,9 @@ public SingleDirectoryDbLedgerStorage(ServerConfiguration conf, LedgerManager le
flushExecutorTime.addLatency(0, TimeUnit.NANOSECONDS);
});

ledgerDirsManager.addLedgerDirsListener(getLedgerDirsListener());
ledgerDirsManager.addLedgerDirsListener(getLedgerDirsListener(ledgerBaseDir));
if (!ledgerBaseDir.equals(indexBaseDir)) {
indexDirsManager.addLedgerDirsListener(getLedgerDirsListener());
indexDirsManager.addLedgerDirsListener(getLedgerDirsListener(indexBaseDir));
}
}

Expand Down Expand Up @@ -1148,11 +1151,19 @@ public OfLong getListOfEntriesOfLedger(long ledgerId) throws IOException {
"getListOfEntriesOfLedger method is currently unsupported for SingleDirectoryDbLedgerStorage");
}

private LedgerDirsManager.LedgerDirsListener getLedgerDirsListener() {
private LedgerDirsManager.LedgerDirsListener getLedgerDirsListener(String diskPath) {
return new LedgerDirsListener() {
private final String currentFilePath = diskPath;

private boolean isCurrentFile(File disk) {
return Objects.equals(disk.getPath(), currentFilePath);
}

@Override
public void diskAlmostFull(File disk) {
if (!isCurrentFile(disk)) {
return;
}
if (gcThread.isForceGCAllowWhenNoSpace()) {
gcThread.enableForceGC();
} else {
Expand All @@ -1162,6 +1173,9 @@ public void diskAlmostFull(File disk) {

@Override
public void diskFull(File disk) {
if (!isCurrentFile(disk)) {
return;
}
if (gcThread.isForceGCAllowWhenNoSpace()) {
gcThread.enableForceGC();
} else {
Expand All @@ -1182,6 +1196,9 @@ public void allDisksFull(boolean highPriorityWritesAllowed) {

@Override
public void diskWritable(File disk) {
if (!isCurrentFile(disk)) {
return;
}
// we have enough space now
if (gcThread.isForceGCAllowWhenNoSpace()) {
// disable force gc.
Expand All @@ -1195,6 +1212,9 @@ public void diskWritable(File disk) {

@Override
public void diskJustWritable(File disk) {
if (!isCurrentFile(disk)) {
return;
}
if (gcThread.isForceGCAllowWhenNoSpace()) {
// if a disk is just writable, we still need force gc.
gcThread.enableForceGC();
Expand Down Expand Up @@ -1297,4 +1317,14 @@ public void clearStorageStateFlag(StorageState flag) throws IOException {
DbLedgerStorageStats getDbLedgerStorageStats() {
return dbLedgerStorageStats;
}

@VisibleForTesting
public String getLedgerBaseDir() {
return ledgerBaseDir;
}

@VisibleForTesting
public String getIndexBaseDir() {
return indexBaseDir;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ public class ServerConfiguration extends AbstractConfiguration<ServerConfigurati
protected static final String GC_OVERREPLICATED_LEDGER_WAIT_TIME = "gcOverreplicatedLedgerWaitTime";
protected static final String GC_OVERREPLICATED_LEDGER_MAX_CONCURRENT_REQUESTS =
"gcOverreplicatedLedgerMaxConcurrentRequests";
protected static final String GC_METADATA_OP_RATE_LIMIT = "gcMetadataOpRateLimit";
protected static final String USE_TRANSACTIONAL_COMPACTION = "useTransactionalCompaction";
protected static final String VERIFY_METADATA_ON_GC = "verifyMetadataOnGC";
protected static final String GC_ENTRYLOGMETADATA_CACHE_ENABLED = "gcEntryLogMetadataCacheEnabled";
Expand Down Expand Up @@ -481,6 +482,24 @@ public ServerConfiguration setGcOverreplicatedLedgerMaxConcurrentRequests(
return this;
}

/**
* Get the rate limit of metadata operations in garbage collection.
* @return rate limit of metadata operations in garbage collection
*/
public int getGcMetadataOpRateLimit() {
return this.getInt(GC_METADATA_OP_RATE_LIMIT, 1000);
}

/**
* Set the rate limit of metadata operations in garbage collection.
* @param gcRateLimit
* @return server configuration
*/
public ServerConfiguration setGcMetadataOpRateLimit(int gcRateLimit) {
this.setProperty(GC_METADATA_OP_RATE_LIMIT, Integer.toString(gcRateLimit));
return this;
}

/**
* Get whether to use transactional compaction and using a separate log for compaction or not.
*
Expand Down
Loading
Loading