From 459a7a57c1b67cfe161cdc40c007a1c2e403b7cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 9 Feb 2023 02:08:19 +0100 Subject: [PATCH 001/174] [improve][txn] Allow superusers to abort transactions (#19467) ### Motivation Super users must be always allowed to abort a transaction even if they're not the original owner. ### Modifications * Check that only owner or superusers are allowed to perform txn operations (end, add partition and add subscription) --- .../TransactionMetadataStoreService.java | 20 +- .../broker/admin/impl/TransactionsBase.java | 1 + .../pulsar/broker/service/ServerCnx.java | 140 ++++++-- .../pulsar/broker/service/ServerCnxTest.java | 21 ++ .../TransactionMetadataStoreServiceTest.java | 36 +- .../broker/stats/TransactionMetricsTest.java | 8 +- ...icatedTransactionProducerConsumerTest.java | 330 ++++++++++++++++++ .../transaction/TransactionTestBase.java | 21 +- .../client/impl/TransactionEndToEndTest.java | 2 +- .../policies/data/TransactionMetadata.java | 3 + .../coordinator/TransactionMetadataStore.java | 3 +- .../transaction/coordinator/TxnMeta.java | 7 + .../impl/InMemTransactionMetadataStore.java | 10 +- .../impl/MLTransactionMetadataStore.java | 104 +++--- .../coordinator/impl/TxnMetaImpl.java | 8 +- .../proto/PulsarTransactionMetadata.proto | 1 + .../MLTransactionMetadataStoreTest.java | 33 +- .../TransactionMetadataStoreProviderTest.java | 12 +- 18 files changed, 630 insertions(+), 130 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/AuthenticatedTransactionProducerConsumerTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java index d0cf22a86533a..3e3b044ec51b8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java @@ -256,12 +256,13 @@ public CompletableFuture removeTransactionMetadataStore(TransactionCoordin } } - public CompletableFuture newTransaction(TransactionCoordinatorID tcId, long timeoutInMills) { + public CompletableFuture newTransaction(TransactionCoordinatorID tcId, long timeoutInMills, + String owner) { TransactionMetadataStore store = stores.get(tcId); if (store == null) { return FutureUtil.failedFuture(new CoordinatorNotFoundException(tcId)); } - return store.newTransaction(timeoutInMills); + return store.newTransaction(timeoutInMills, owner); } public CompletableFuture addProducedPartitionToTxn(TxnID txnId, List partitions) { @@ -483,6 +484,21 @@ public Map getStores() { return Collections.unmodifiableMap(stores); } + public CompletableFuture verifyTxnOwnership(TxnID txnID, String checkOwner) { + return getTxnMeta(txnID) + .thenCompose(meta -> { + // owner was null in the old versions or no auth enabled + if (meta.getOwner() == null) { + return CompletableFuture.completedFuture(true); + } + if (meta.getOwner().equals(checkOwner)) { + return CompletableFuture.completedFuture(true); + } + return CompletableFuture.completedFuture(false); + }); + } + + public void close () { this.internalPinnedExecutor.shutdown(); stores.forEach((tcId, metadataStore) -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TransactionsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TransactionsBase.java index f537f0ecdb9eb..d596cbdd39db9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TransactionsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/TransactionsBase.java @@ -215,6 +215,7 @@ private void getTransactionMetadata(TxnMeta txnMeta, transactionMetadata.status = txnMeta.status().name(); transactionMetadata.openTimestamp = txnMeta.getOpenTimestamp(); transactionMetadata.timeoutAt = txnMeta.getTimeoutAt(); + transactionMetadata.owner = txnMeta.getOwner(); List> ackedPartitionsFutures = new ArrayList<>(); Map>> ackFutures = new HashMap<>(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index d398dbba9b996..9cbdbcf1ae4d5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -2399,7 +2399,8 @@ protected void handleNewTxn(CommandNewTxn command) { TransactionMetadataStoreService transactionMetadataStoreService = service.pulsar().getTransactionMetadataStoreService(); - transactionMetadataStoreService.newTransaction(tcId, command.getTxnTtlSeconds()) + final String owner = getPrincipal(); + transactionMetadataStoreService.newTransaction(tcId, command.getTxnTtlSeconds(), owner) .whenComplete(((txnID, ex) -> { if (ex == null) { if (log.isDebugEnabled()) { @@ -2443,9 +2444,15 @@ protected void handleAddPartitionToTxn(CommandAddPartitionToTxn command) { TransactionMetadataStoreService transactionMetadataStoreService = service.pulsar().getTransactionMetadataStoreService(); - service.pulsar().getTransactionMetadataStoreService().addProducedPartitionToTxn(txnID, - command.getPartitionsList()) - .whenComplete(((v, ex) -> { + verifyTxnOwnership(txnID) + .thenCompose(isOwner -> { + if (!isOwner) { + return failedFutureTxnNotOwned(txnID); + } + return transactionMetadataStoreService + .addProducedPartitionToTxn(txnID, command.getPartitionsList()); + }) + .whenComplete((v, ex) -> { if (ex == null) { if (log.isDebugEnabled()) { log.debug("Send response success for add published partition to txn request {}", requestId); @@ -2462,7 +2469,25 @@ protected void handleAddPartitionToTxn(CommandAddPartitionToTxn command) { ex.getMessage())); transactionMetadataStoreService.handleOpFail(ex, tcId); } - })); + }); + } + + private CompletableFuture failedFutureTxnNotOwned(TxnID txnID) { + String msg = String.format( + "Client (%s) is neither the owner of the transaction %s nor a super user", + getPrincipal(), txnID + ); + log.warn("[{}] {}", remoteAddress, msg); + return CompletableFuture.failedFuture(new CoordinatorException.TransactionNotFoundException(msg)); + } + + private CompletableFuture failedFutureTxnTcNotAllowed(TxnID txnID) { + String msg = String.format( + "TC client (%s) is not a super user, and is not allowed to operate on transaction %s", + getPrincipal(), txnID + ); + log.warn("[{}] {}", remoteAddress, msg); + return CompletableFuture.failedFuture(new CoordinatorException.TransactionNotFoundException(msg)); } @Override @@ -2480,12 +2505,16 @@ protected void handleEndTxn(CommandEndTxn command) { TransactionMetadataStoreService transactionMetadataStoreService = service.pulsar().getTransactionMetadataStoreService(); - transactionMetadataStoreService - .endTransaction(txnID, txnAction, false) + verifyTxnOwnership(txnID) + .thenCompose(isOwner -> { + if (!isOwner) { + return failedFutureTxnNotOwned(txnID); + } + return transactionMetadataStoreService.endTransaction(txnID, txnAction, false); + }) .whenComplete((v, ex) -> { if (ex == null) { - commandSender.sendEndTxnResponse(requestId, - txnID, txnAction); + commandSender.sendEndTxnResponse(requestId, txnID, txnAction); } else { ex = handleTxnException(ex, BaseCommand.Type.END_TXN.name(), requestId); commandSender.sendEndTxnErrorResponse(requestId, txnID, @@ -2496,6 +2525,34 @@ protected void handleEndTxn(CommandEndTxn command) { }); } + private CompletableFuture verifyTxnOwnershipForTCToBrokerCommands() { + if (service.isAuthenticationEnabled() && service.isAuthorizationEnabled()) { + return getBrokerService() + .getAuthorizationService() + .isSuperUser(getPrincipal(), getAuthenticationData()); + } else { + return CompletableFuture.completedFuture(true); + } + } + + private CompletableFuture verifyTxnOwnership(TxnID txnID) { + final String checkOwner = getPrincipal(); + return service.pulsar().getTransactionMetadataStoreService() + .verifyTxnOwnership(txnID, checkOwner) + .thenCompose(isOwner -> { + if (isOwner) { + return CompletableFuture.completedFuture(true); + } + if (service.isAuthenticationEnabled() && service.isAuthorizationEnabled()) { + return getBrokerService() + .getAuthorizationService() + .isSuperUser(checkOwner, getAuthenticationData()); + } else { + return CompletableFuture.completedFuture(false); + } + }); + } + @Override protected void handleEndTxnOnPartition(CommandEndTxnOnPartition command) { checkArgument(state == State.Connected); @@ -2512,9 +2569,17 @@ protected void handleEndTxnOnPartition(CommandEndTxnOnPartition command) { CompletableFuture> topicFuture = service.getTopicIfExists(TopicName.get(topic).toString()); topicFuture.thenAccept(optionalTopic -> { if (optionalTopic.isPresent()) { - optionalTopic.get().endTxn(txnID, txnAction, lowWaterMark) + // we only accept super user becase this endpoint is reserved for tc to broker communication + verifyTxnOwnershipForTCToBrokerCommands() + .thenCompose(isOwner -> { + if (!isOwner) { + return failedFutureTxnTcNotAllowed(txnID); + } + return optionalTopic.get().endTxn(txnID, txnAction, lowWaterMark); + }) .whenComplete((ignored, throwable) -> { if (throwable != null) { + throwable = FutureUtil.unwrapCompletionException(throwable); log.error("handleEndTxnOnPartition fail!, topic {}, txnId: [{}], " + "txnAction: [{}]", topic, txnID, TxnAction.valueOf(txnAction), throwable); writeAndFlush(Commands.newEndTxnOnPartitionResponse( @@ -2526,7 +2591,6 @@ protected void handleEndTxnOnPartition(CommandEndTxnOnPartition command) { writeAndFlush(Commands.newEndTxnOnPartitionResponse(requestId, txnID.getLeastSigBits(), txnID.getMostSigBits())); }); - } else { getBrokerService().getManagedLedgerFactory() .asyncExists(TopicName.get(topic).getPersistenceNamingEncoding()) @@ -2596,23 +2660,28 @@ protected void handleEndTxnOnSubscription(CommandEndTxnOnSubscription command) { Commands.newEndTxnOnSubscriptionResponse(requestId, txnidLeastBits, txnidMostBits)); return; } - - CompletableFuture completableFuture = - subscription.endTxn(txnidMostBits, txnidLeastBits, txnAction, lowWaterMark); - completableFuture.whenComplete((ignored, e) -> { - if (e != null) { - log.error("handleEndTxnOnSubscription fail ! topic: {}, subscription: {}" - + "txnId: [{}], txnAction: [{}]", topic, subName, - txnID, TxnAction.valueOf(txnAction), e.getCause()); - writeAndFlush(Commands.newEndTxnOnSubscriptionResponse( - requestId, txnidLeastBits, txnidMostBits, - BrokerServiceException.getClientErrorCode(e), - "Handle end txn on subscription failed: " + e.getMessage())); - return; - } - writeAndFlush( - Commands.newEndTxnOnSubscriptionResponse(requestId, txnidLeastBits, txnidMostBits)); - }); + // we only accept super user becase this endpoint is reserved for tc to broker communication + verifyTxnOwnershipForTCToBrokerCommands() + .thenCompose(isOwner -> { + if (!isOwner) { + return failedFutureTxnTcNotAllowed(txnID); + } + return subscription.endTxn(txnidMostBits, txnidLeastBits, txnAction, lowWaterMark); + }).whenComplete((ignored, e) -> { + if (e != null) { + e = FutureUtil.unwrapCompletionException(e); + log.error("handleEndTxnOnSubscription fail ! topic: {}, subscription: {}" + + "txnId: [{}], txnAction: [{}]", topic, subName, + txnID, TxnAction.valueOf(txnAction), e.getCause()); + writeAndFlush(Commands.newEndTxnOnSubscriptionResponse( + requestId, txnidLeastBits, txnidMostBits, + BrokerServiceException.getClientErrorCode(e), + "Handle end txn on subscription failed: " + e.getMessage())); + return; + } + writeAndFlush( + Commands.newEndTxnOnSubscriptionResponse(requestId, txnidLeastBits, txnidMostBits)); + }); } else { getBrokerService().getManagedLedgerFactory() .asyncExists(TopicName.get(topic).getPersistenceNamingEncoding()) @@ -2679,6 +2748,7 @@ protected void handleAddSubscriptionToTxn(CommandAddSubscriptionToTxn command) { checkArgument(state == State.Connected); final TxnID txnID = new TxnID(command.getTxnidMostBits(), command.getTxnidLeastBits()); final long requestId = command.getRequestId(); + final List subscriptionsList = command.getSubscriptionsList(); if (log.isDebugEnabled()) { log.debug("Receive add published partition to txn request {} from {} with txnId {}", requestId, remoteAddress, txnID); @@ -2693,9 +2763,15 @@ protected void handleAddSubscriptionToTxn(CommandAddSubscriptionToTxn command) { TransactionMetadataStoreService transactionMetadataStoreService = service.pulsar().getTransactionMetadataStoreService(); - transactionMetadataStoreService.addAckedPartitionToTxn(txnID, - MLTransactionMetadataStore.subscriptionToTxnSubscription(command.getSubscriptionsList())) - .whenComplete(((v, ex) -> { + verifyTxnOwnership(txnID) + .thenCompose(isOwner -> { + if (!isOwner) { + return failedFutureTxnNotOwned(txnID); + } + return transactionMetadataStoreService.addAckedPartitionToTxn(txnID, + MLTransactionMetadataStore.subscriptionToTxnSubscription(subscriptionsList)); + }) + .whenComplete((v, ex) -> { if (ex == null) { if (log.isDebugEnabled()) { log.debug("Send response success for add published partition to txn request {}", @@ -2711,7 +2787,7 @@ protected void handleAddSubscriptionToTxn(CommandAddSubscriptionToTxn command) { ex.getMessage())); transactionMetadataStoreService.handleOpFail(ex, tcId); } - })); + }); } @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java index b8032f9be3ad7..b7e2839832c5d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java @@ -134,6 +134,7 @@ import org.apache.pulsar.common.topics.TopicList; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.ConcurrentLongHashMap; +import org.apache.pulsar.transaction.coordinator.TxnMeta; import org.awaitility.Awaitility; import org.mockito.ArgumentCaptor; import org.mockito.MockedStatic; @@ -3033,6 +3034,8 @@ public void handlePartitionMetadataRequestWithServiceNotReady() throws Exception @Test(timeOut = 30000) public void sendAddPartitionToTxnResponse() throws Exception { final TransactionMetadataStoreService txnStore = mock(TransactionMetadataStoreService.class); + when(txnStore.getTxnMeta(any())).thenReturn(CompletableFuture.completedFuture(mock(TxnMeta.class))); + when(txnStore.verifyTxnOwnership(any(), any())).thenReturn(CompletableFuture.completedFuture(true)); when(txnStore.addProducedPartitionToTxn(any(TxnID.class), any())) .thenReturn(CompletableFuture.completedFuture(null)); when(pulsar.getTransactionMetadataStoreService()).thenReturn(txnStore); @@ -3056,6 +3059,8 @@ public void sendAddPartitionToTxnResponse() throws Exception { @Test(timeOut = 30000) public void sendAddPartitionToTxnResponseFailed() throws Exception { final TransactionMetadataStoreService txnStore = mock(TransactionMetadataStoreService.class); + when(txnStore.getTxnMeta(any())).thenReturn(CompletableFuture.completedFuture(mock(TxnMeta.class))); + when(txnStore.verifyTxnOwnership(any(), any())).thenReturn(CompletableFuture.completedFuture(true)); when(txnStore.addProducedPartitionToTxn(any(TxnID.class), any())) .thenReturn(CompletableFuture.failedFuture(new RuntimeException("server error"))); when(pulsar.getTransactionMetadataStoreService()).thenReturn(txnStore); @@ -3079,6 +3084,8 @@ public void sendAddPartitionToTxnResponseFailed() throws Exception { @Test(timeOut = 30000) public void sendAddSubscriptionToTxnResponse() throws Exception { final TransactionMetadataStoreService txnStore = mock(TransactionMetadataStoreService.class); + when(txnStore.getTxnMeta(any())).thenReturn(CompletableFuture.completedFuture(mock(TxnMeta.class))); + when(txnStore.verifyTxnOwnership(any(), any())).thenReturn(CompletableFuture.completedFuture(true)); when(txnStore.addAckedPartitionToTxn(any(TxnID.class), any())) .thenReturn(CompletableFuture.completedFuture(null)); when(pulsar.getTransactionMetadataStoreService()).thenReturn(txnStore); @@ -3105,6 +3112,8 @@ public void sendAddSubscriptionToTxnResponse() throws Exception { @Test(timeOut = 30000) public void sendAddSubscriptionToTxnResponseFailed() throws Exception { final TransactionMetadataStoreService txnStore = mock(TransactionMetadataStoreService.class); + when(txnStore.getTxnMeta(any())).thenReturn(CompletableFuture.completedFuture(mock(TxnMeta.class))); + when(txnStore.verifyTxnOwnership(any(), any())).thenReturn(CompletableFuture.completedFuture(true)); when(txnStore.addAckedPartitionToTxn(any(TxnID.class), any())) .thenReturn(CompletableFuture.failedFuture(new RuntimeException("server error"))); when(pulsar.getTransactionMetadataStoreService()).thenReturn(txnStore); @@ -3132,6 +3141,8 @@ public void sendAddSubscriptionToTxnResponseFailed() throws Exception { @Test(timeOut = 30000) public void sendEndTxnResponse() throws Exception { final TransactionMetadataStoreService txnStore = mock(TransactionMetadataStoreService.class); + when(txnStore.getTxnMeta(any())).thenReturn(CompletableFuture.completedFuture(mock(TxnMeta.class))); + when(txnStore.verifyTxnOwnership(any(), any())).thenReturn(CompletableFuture.completedFuture(true)); when(txnStore.endTransaction(any(TxnID.class), anyInt(), anyBoolean())) .thenReturn(CompletableFuture.completedFuture(null)); when(pulsar.getTransactionMetadataStoreService()).thenReturn(txnStore); @@ -3155,6 +3166,8 @@ public void sendEndTxnResponse() throws Exception { @Test(timeOut = 30000) public void sendEndTxnResponseFailed() throws Exception { final TransactionMetadataStoreService txnStore = mock(TransactionMetadataStoreService.class); + when(txnStore.getTxnMeta(any())).thenReturn(CompletableFuture.completedFuture(mock(TxnMeta.class))); + when(txnStore.verifyTxnOwnership(any(), any())).thenReturn(CompletableFuture.completedFuture(true)); when(txnStore.endTransaction(any(TxnID.class), anyInt(), anyBoolean())) .thenReturn(CompletableFuture.failedFuture(new RuntimeException("server error"))); when(pulsar.getTransactionMetadataStoreService()).thenReturn(txnStore); @@ -3178,6 +3191,8 @@ public void sendEndTxnResponseFailed() throws Exception { @Test(timeOut = 30000) public void sendEndTxnOnPartitionResponse() throws Exception { final TransactionMetadataStoreService txnStore = mock(TransactionMetadataStoreService.class); + when(txnStore.getTxnMeta(any())).thenReturn(CompletableFuture.completedFuture(mock(TxnMeta.class))); + when(txnStore.verifyTxnOwnership(any(), any())).thenReturn(CompletableFuture.completedFuture(true)); when(txnStore.endTransaction(any(TxnID.class), anyInt(), anyBoolean())) .thenReturn(CompletableFuture.completedFuture(null)); when(pulsar.getTransactionMetadataStoreService()).thenReturn(txnStore); @@ -3206,6 +3221,8 @@ public void sendEndTxnOnPartitionResponse() throws Exception { @Test(timeOut = 30000) public void sendEndTxnOnPartitionResponseFailed() throws Exception { final TransactionMetadataStoreService txnStore = mock(TransactionMetadataStoreService.class); + when(txnStore.getTxnMeta(any())).thenReturn(CompletableFuture.completedFuture(mock(TxnMeta.class))); + when(txnStore.verifyTxnOwnership(any(), any())).thenReturn(CompletableFuture.completedFuture(true)); when(txnStore.endTransaction(any(TxnID.class), anyInt(), anyBoolean())) .thenReturn(CompletableFuture.completedFuture(null)); when(pulsar.getTransactionMetadataStoreService()).thenReturn(txnStore); @@ -3235,6 +3252,8 @@ public void sendEndTxnOnPartitionResponseFailed() throws Exception { @Test(timeOut = 30000) public void sendEndTxnOnSubscription() throws Exception { final TransactionMetadataStoreService txnStore = mock(TransactionMetadataStoreService.class); + when(txnStore.getTxnMeta(any())).thenReturn(CompletableFuture.completedFuture(mock(TxnMeta.class))); + when(txnStore.verifyTxnOwnership(any(), any())).thenReturn(CompletableFuture.completedFuture(true)); when(txnStore.endTransaction(any(TxnID.class), anyInt(), anyBoolean())) .thenReturn(CompletableFuture.completedFuture(null)); when(pulsar.getTransactionMetadataStoreService()).thenReturn(txnStore); @@ -3269,6 +3288,8 @@ public void sendEndTxnOnSubscription() throws Exception { @Test(timeOut = 30000) public void sendEndTxnOnSubscriptionFailed() throws Exception { final TransactionMetadataStoreService txnStore = mock(TransactionMetadataStoreService.class); + when(txnStore.getTxnMeta(any())).thenReturn(CompletableFuture.completedFuture(mock(TxnMeta.class))); + when(txnStore.verifyTxnOwnership(any(), any())).thenReturn(CompletableFuture.completedFuture(true)); when(txnStore.endTransaction(any(TxnID.class), anyInt(), anyBoolean())) .thenReturn(CompletableFuture.completedFuture(null)); when(pulsar.getTransactionMetadataStoreService()).thenReturn(txnStore); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionMetadataStoreServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionMetadataStoreServiceTest.java index 06fdc13b98c94..5cd3ed9f90454 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionMetadataStoreServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionMetadataStoreServiceTest.java @@ -104,9 +104,9 @@ public void testNewTransaction() throws Exception { .getStores().get(TransactionCoordinatorID.get(1))); checkTransactionMetadataStoreReady((MLTransactionMetadataStore) pulsar.getTransactionMetadataStoreService() .getStores().get(TransactionCoordinatorID.get(2))); - TxnID txnID0 = transactionMetadataStoreService.newTransaction(TransactionCoordinatorID.get(0), 5).get(); - TxnID txnID1 = transactionMetadataStoreService.newTransaction(TransactionCoordinatorID.get(1), 5).get(); - TxnID txnID2 = transactionMetadataStoreService.newTransaction(TransactionCoordinatorID.get(2), 5).get(); + TxnID txnID0 = transactionMetadataStoreService.newTransaction(TransactionCoordinatorID.get(0), 5, null).get(); + TxnID txnID1 = transactionMetadataStoreService.newTransaction(TransactionCoordinatorID.get(1), 5, null).get(); + TxnID txnID2 = transactionMetadataStoreService.newTransaction(TransactionCoordinatorID.get(2), 5, null).get(); Assert.assertEquals(txnID0.getMostSigBits(), 0); Assert.assertEquals(txnID1.getMostSigBits(), 1); Assert.assertEquals(txnID2.getMostSigBits(), 2); @@ -128,7 +128,7 @@ public void testAddProducedPartitionToTxn() throws Exception { .getStores().get(TransactionCoordinatorID.get(0)); checkTransactionMetadataStoreReady(transactionMetadataStore); - TxnID txnID = transactionMetadataStoreService.newTransaction(TransactionCoordinatorID.get(0), 5000).get(); + TxnID txnID = transactionMetadataStoreService.newTransaction(TransactionCoordinatorID.get(0), 5000, null).get(); List partitions = new ArrayList<>(); partitions.add("ptn-0"); partitions.add("ptn-1"); @@ -151,7 +151,7 @@ public void testAddAckedPartitionToTxn() throws Exception { (MLTransactionMetadataStore) pulsar.getTransactionMetadataStoreService() .getStores().get(TransactionCoordinatorID.get(0)); checkTransactionMetadataStoreReady(transactionMetadataStore); - TxnID txnID = transactionMetadataStoreService.newTransaction(TransactionCoordinatorID.get(0), 5000).get(); + TxnID txnID = transactionMetadataStoreService.newTransaction(TransactionCoordinatorID.get(0), 5000, null).get(); List partitions = new ArrayList<>(); partitions.add(TransactionSubscription.builder().topic("ptn-1").subscription("sub-1").build()); partitions.add(TransactionSubscription.builder().topic("ptn-2").subscription("sub-1").build()); @@ -180,7 +180,7 @@ public void testTimeoutTracker() throws Exception { int i = -1; while (++i < 1000) { try { - transactionMetadataStore.newTransaction(2000).get(); + newTransactionWithTimeoutOf(2000); } catch (Exception e) { //no operation } @@ -192,6 +192,14 @@ public void testTimeoutTracker() throws Exception { .until(() -> txnMap.size() == 0); } + private TxnID newTransactionWithTimeoutOf(long timeout) + throws InterruptedException, ExecutionException { + MLTransactionMetadataStore transactionMetadataStore = + (MLTransactionMetadataStore) pulsar.getTransactionMetadataStoreService() + .getStores().get(TransactionCoordinatorID.get(0)); + return transactionMetadataStore.newTransaction(timeout, null).get(); + } + @Test public void testTimeoutTrackerExpired() throws Exception { pulsar.getTransactionMetadataStoreService().handleTcClientConnect(TransactionCoordinatorID.get(0)); @@ -206,7 +214,7 @@ public void testTimeoutTrackerExpired() throws Exception { ConcurrentSkipListMap>> txnMap = (ConcurrentSkipListMap>>) field.get(transactionMetadataStore); - transactionMetadataStore.newTransaction(2000).get(); + newTransactionWithTimeoutOf(2000); assertEquals(txnMap.size(), 1); @@ -214,7 +222,7 @@ public void testTimeoutTrackerExpired() throws Exception { Assert.assertEquals(txnMetaListPair.getLeft().status(), TxnStatus.OPEN)); Awaitility.await().atLeast(1000, TimeUnit.MICROSECONDS).until(() -> txnMap.size() == 0); - transactionMetadataStore.newTransaction(2000).get(); + newTransactionWithTimeoutOf(2000); assertEquals(txnMap.size(), 1); txnMap.forEach((txnID, txnMetaListPair) -> @@ -241,7 +249,7 @@ public void testTimeoutTrackerMultiThreading() throws Exception { int i = -1; while (++i < 100) { try { - transactionMetadataStore.newTransaction(1000); + newTransactionWithTimeoutOf(1000); } catch (Exception e) { //no operation } @@ -252,7 +260,7 @@ public void testTimeoutTrackerMultiThreading() throws Exception { int i = -1; while (++i < 100) { try { - transactionMetadataStore.newTransaction(2000); + newTransactionWithTimeoutOf(2000); } catch (Exception e) { //no operation } @@ -263,7 +271,7 @@ public void testTimeoutTrackerMultiThreading() throws Exception { int i = -1; while (++i < 100) { try { - transactionMetadataStore.newTransaction(3000); + newTransactionWithTimeoutOf(3000); } catch (Exception e) { //no operation } @@ -274,7 +282,7 @@ public void testTimeoutTrackerMultiThreading() throws Exception { int i = -1; while (++i < 100) { try { - transactionMetadataStore.newTransaction(4000); + newTransactionWithTimeoutOf(4000); } catch (Exception e) { //no operation } @@ -304,7 +312,7 @@ public void transactionTimeoutRecoverTest() throws Exception { .getStores().get(TransactionCoordinatorID.get(0)); checkTransactionMetadataStoreReady(transactionMetadataStore); - transactionMetadataStore.newTransaction(timeout); + newTransactionWithTimeoutOf(2000); pulsar.getTransactionMetadataStoreService() .removeTransactionMetadataStore(TransactionCoordinatorID.get(0)); @@ -345,7 +353,7 @@ public void testEndTransactionOpRetry(TxnStatus txnStatus) throws Exception { checkTransactionMetadataStoreReady(transactionMetadataStore); - TxnID txnID = transactionMetadataStore.newTransaction(timeOut - 2000).get(); + TxnID txnID = newTransactionWithTimeoutOf(timeOut - 2000); TxnMeta txnMeta = transactionMetadataStore.getTxnMeta(txnID).get(); txnMeta.updateTxnStatus(txnStatus, TxnStatus.OPEN); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionMetricsTest.java index 37d1f4b086066..30aedc022534c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionMetricsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionMetricsTest.java @@ -111,11 +111,11 @@ public void testTransactionCoordinatorMetrics() throws Exception { Awaitility.await().until(() -> pulsar.getTransactionMetadataStoreService().getStores().size() == 2); pulsar.getTransactionMetadataStoreService().getStores() - .get(transactionCoordinatorIDOne).newTransaction(timeout).get(); + .get(transactionCoordinatorIDOne).newTransaction(timeout, null).get(); pulsar.getTransactionMetadataStoreService().getStores() - .get(transactionCoordinatorIDTwo).newTransaction(timeout).get(); + .get(transactionCoordinatorIDTwo).newTransaction(timeout, null).get(); pulsar.getTransactionMetadataStoreService().getStores() - .get(transactionCoordinatorIDTwo).newTransaction(timeout).get(); + .get(transactionCoordinatorIDTwo).newTransaction(timeout, null).get(); ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); PrometheusMetricsGenerator.generate(pulsar, true, false, false, statsOut); String metricsStr = statsOut.toString(); @@ -202,7 +202,7 @@ public void testTransactionCoordinatorRateMetrics() throws Exception { metric.forEach(item -> assertEquals(item.value, txnCount / 2)); TxnID txnID = pulsar.getTransactionMetadataStoreService().getStores() - .get(transactionCoordinatorIDOne).newTransaction(1000).get(); + .get(transactionCoordinatorIDOne).newTransaction(1000, null).get(); Awaitility.await().atMost(2000, TimeUnit.MILLISECONDS).until(() -> { try { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/AuthenticatedTransactionProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/AuthenticatedTransactionProducerConsumerTest.java new file mode 100644 index 0000000000000..aa9102189d430 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/AuthenticatedTransactionProducerConsumerTest.java @@ -0,0 +1,330 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction; + +import com.google.common.collect.Sets; +import io.jsonwebtoken.Jwts; +import io.jsonwebtoken.SignatureAlgorithm; + +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.NoSuchAlgorithmException; +import java.security.PrivateKey; +import java.time.Duration; +import java.util.Base64; +import java.util.Date; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.List; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +import lombok.Cleanup; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminBuilder; +import org.apache.pulsar.client.api.AuthenticationFactory; +import org.apache.pulsar.client.api.ClientBuilder; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.transaction.Transaction; +import org.apache.pulsar.client.api.transaction.TransactionCoordinatorClientException; +import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.apache.pulsar.client.impl.auth.AuthenticationToken; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.policies.data.AuthAction; +import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.functions.utils.Exceptions; +import org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID; +import org.apache.pulsar.transaction.coordinator.TxnMeta; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +/** + * Test for consuming transaction messages. + */ +@Slf4j +@Test(groups = "broker") +public class AuthenticatedTransactionProducerConsumerTest extends TransactionTestBase { + + private static final String TOPIC = NAMESPACE1 + "/txn-auth"; + + private final String ADMIN_TOKEN; + private final String TOKEN_PUBLIC_KEY; + private final KeyPair kp; + + AuthenticatedTransactionProducerConsumerTest() throws NoSuchAlgorithmException { + KeyPairGenerator kpg = KeyPairGenerator.getInstance("RSA"); + kp = kpg.generateKeyPair(); + + byte[] encodedPublicKey = kp.getPublic().getEncoded(); + TOKEN_PUBLIC_KEY = "data:;base64," + Base64.getEncoder().encodeToString(encodedPublicKey); + ADMIN_TOKEN = generateToken(kp, "admin"); + } + + + private String generateToken(KeyPair kp, String subject) { + PrivateKey pkey = kp.getPrivate(); + long expMillis = System.currentTimeMillis() + Duration.ofHours(1).toMillis(); + Date exp = new Date(expMillis); + + return Jwts.builder() + .setSubject(subject) + .setExpiration(exp) + .signWith(pkey, SignatureAlgorithm.forSigningKey(pkey)) + .compact(); + } + + @BeforeMethod(alwaysRun = true) + public void setup() throws Exception { + conf.setAuthenticationEnabled(true); + conf.setAuthorizationEnabled(true); + + Set superUserRoles = new HashSet<>(); + superUserRoles.add("admin"); + conf.setSuperUserRoles(superUserRoles); + + Set providers = new HashSet<>(); + providers.add(AuthenticationProviderToken.class.getName()); + conf.setAuthenticationProviders(providers); + + // Set provider domain name + Properties properties = new Properties(); + properties.setProperty("tokenPublicKey", TOKEN_PUBLIC_KEY); + + conf.setProperties(properties); + conf.setBrokerClientAuthenticationPlugin(AuthenticationToken.class.getName()); + conf.setBrokerClientAuthenticationParameters("token:" + ADMIN_TOKEN); + setBrokerCount(1); + internalSetup(); + setUpBase(1, 1, TOPIC, 1); + + grantTxnLookupToRole("client"); + admin.namespaces().grantPermissionOnNamespace(NAMESPACE1, "client", + EnumSet.allOf(AuthAction.class)); + grantTxnLookupToRole("client2"); + } + + @SneakyThrows + private void grantTxnLookupToRole(String role) { + admin.namespaces().grantPermissionOnNamespace( + NamespaceName.SYSTEM_NAMESPACE.toString(), + role, + Sets.newHashSet(AuthAction.consume)); + } + + @Override + protected PulsarClient createNewPulsarClient(ClientBuilder clientBuilder) throws PulsarClientException { + return clientBuilder + .enableTransaction(true) + .authentication(AuthenticationFactory.token(ADMIN_TOKEN)) + .build(); + } + + @Override + protected PulsarAdmin createNewPulsarAdmin(PulsarAdminBuilder builder) throws PulsarClientException { + return builder + .authentication(AuthenticationFactory.token(ADMIN_TOKEN)) + .build(); + } + + @AfterMethod(alwaysRun = true) + protected void cleanup() { + super.internalCleanup(); + } + + @DataProvider(name = "actors") + public Object[][] actors() { + return new Object[][]{ + {"client", true}, + {"client", false}, + {"client2", true}, + {"client2", false}, + {"admin", true}, + {"admin", false} + }; + } + + @Test(dataProvider = "actors") + public void testEndTxn(String actor, boolean afterUnload) throws Exception { + @Cleanup final PulsarClient pulsarClientOwner = PulsarClient.builder() + .serviceUrl(pulsarServiceList.get(0).getBrokerServiceUrl()) + .authentication(AuthenticationFactory.token(generateToken(kp, "client"))) + .enableTransaction(true) + .build(); + + @Cleanup final PulsarClient pulsarClientOther = PulsarClient.builder() + .serviceUrl(pulsarServiceList.get(0).getBrokerServiceUrl()) + .authentication(AuthenticationFactory.token(generateToken(kp, actor))) + .enableTransaction(true) + .build(); + Transaction transaction = pulsarClientOwner.newTransaction() + .withTransactionTimeout(60, TimeUnit.SECONDS).build().get(); + + @Cleanup final Consumer consumer = pulsarClientOwner + .newConsumer(Schema.STRING) + .subscriptionName("test") + .topic(TOPIC) + .subscribe(); + + + @Cleanup final Producer producer = pulsarClientOwner + .newProducer(Schema.STRING) + .sendTimeout(60, TimeUnit.SECONDS) + .topic(TOPIC) + .create(); + + producer.newMessage().value("beforetxn").send(); + consumer.acknowledgeAsync(consumer.receive(5, TimeUnit.SECONDS).getMessageId(), transaction); + producer.newMessage(transaction).value("message").send(); + if (afterUnload) { + pulsarServiceList.get(0) + .getTransactionMetadataStoreService() + .removeTransactionMetadataStore( + TransactionCoordinatorID.get(transaction.getTxnID().getMostSigBits())); + } + + final Throwable ex = syncGetException(( + (PulsarClientImpl) pulsarClientOther).getTcClient().commitAsync(transaction.getTxnID()) + ); + if (actor.equals("client") || actor.equals("admin")) { + Assert.assertNull(ex); + Assert.assertEquals(consumer.receive(5, TimeUnit.SECONDS).getValue(), "message"); + } else { + Assert.assertNotNull(ex); + Assert.assertTrue(ex instanceof TransactionCoordinatorClientException, ex.getClass().getName()); + Assert.assertNull(consumer.receive(5, TimeUnit.SECONDS)); + transaction.commit().get(); + Assert.assertEquals(consumer.receive(5, TimeUnit.SECONDS).getValue(), "message"); + } + } + + @Test(dataProvider = "actors") + public void testAddPartitionToTxn(String actor, boolean afterUnload) throws Exception { + @Cleanup final PulsarClient pulsarClientOwner = PulsarClient.builder() + .serviceUrl(pulsarServiceList.get(0).getBrokerServiceUrl()) + .authentication(AuthenticationFactory.token(generateToken(kp, "client"))) + .enableTransaction(true) + .build(); + + @Cleanup final PulsarClient pulsarClientOther = PulsarClient.builder() + .serviceUrl(pulsarServiceList.get(0).getBrokerServiceUrl()) + .authentication(AuthenticationFactory.token(generateToken(kp, actor))) + .enableTransaction(true) + .build(); + Transaction transaction = pulsarClientOwner.newTransaction() + .withTransactionTimeout(60, TimeUnit.SECONDS).build().get(); + + if (afterUnload) { + pulsarServiceList.get(0) + .getTransactionMetadataStoreService() + .removeTransactionMetadataStore( + TransactionCoordinatorID.get(transaction.getTxnID().getMostSigBits())); + } + + final Throwable ex = syncGetException(((PulsarClientImpl) pulsarClientOther) + .getTcClient().addPublishPartitionToTxnAsync(transaction.getTxnID(), List.of(TOPIC))); + + final TxnMeta txnMeta = pulsarServiceList.get(0).getTransactionMetadataStoreService() + .getTxnMeta(transaction.getTxnID()).get(); + if (actor.equals("client") || actor.equals("admin")) { + Assert.assertNull(ex); + Assert.assertEquals(txnMeta.producedPartitions(), List.of(TOPIC)); + } else { + Assert.assertNotNull(ex); + Assert.assertTrue(ex instanceof TransactionCoordinatorClientException); + Assert.assertTrue(txnMeta.producedPartitions().isEmpty()); + } + } + + @Test(dataProvider = "actors") + public void testAddSubscriptionToTxn(String actor, boolean afterUnload) throws Exception { + @Cleanup final PulsarClient pulsarClientOwner = PulsarClient.builder() + .serviceUrl(pulsarServiceList.get(0).getBrokerServiceUrl()) + .authentication(AuthenticationFactory.token(generateToken(kp, "client"))) + .enableTransaction(true) + .build(); + + @Cleanup final PulsarClient pulsarClientOther = PulsarClient.builder() + .serviceUrl(pulsarServiceList.get(0).getBrokerServiceUrl()) + .authentication(AuthenticationFactory.token(generateToken(kp, actor))) + .enableTransaction(true) + .build(); + Transaction transaction = pulsarClientOwner.newTransaction() + .withTransactionTimeout(60, TimeUnit.SECONDS).build().get(); + + if (afterUnload) { + pulsarServiceList.get(0) + .getTransactionMetadataStoreService() + .removeTransactionMetadataStore( + TransactionCoordinatorID.get(transaction.getTxnID().getMostSigBits())); + } + + + final Throwable ex = syncGetException(((PulsarClientImpl) pulsarClientOther) + .getTcClient().addSubscriptionToTxnAsync(transaction.getTxnID(), TOPIC, "sub")); + + final TxnMeta txnMeta = pulsarServiceList.get(0).getTransactionMetadataStoreService() + .getTxnMeta(transaction.getTxnID()).get(); + if (actor.equals("client") || actor.equals("admin")) { + Assert.assertNull(ex); + Assert.assertEquals(txnMeta.ackedPartitions().size(), 1); + } else { + Assert.assertNotNull(ex); + Assert.assertTrue(ex instanceof TransactionCoordinatorClientException); + Assert.assertTrue(txnMeta.ackedPartitions().isEmpty()); + } + } + + @Test + public void testNoAuth() throws Exception { + try { + @Cleanup final PulsarClient pulsarClient = PulsarClient.builder() + .serviceUrl(pulsarServiceList.get(0).getBrokerServiceUrl()) + .enableTransaction(true) + .build(); + Assert.fail("should have failed"); + } catch (Exception t) { + Assert.assertTrue(Exceptions.areExceptionsPresentInChain(t, + PulsarClientException.AuthenticationException.class)); + } + } + + private static Throwable syncGetException(CompletableFuture future) { + try { + future.get(); + } catch (InterruptedException e) { + return e; + } catch (ExecutionException e) { + return FutureUtil.unwrapCompletionException(e); + } + return null; + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java index 444c68df0556a..fd49354342fa0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java @@ -35,7 +35,10 @@ import org.apache.pulsar.broker.service.BrokerTestBase; import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminBuilder; +import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.partition.PartitionedTopicMetadata; @@ -71,7 +74,9 @@ public void internalSetup() throws Exception { if (admin != null) { admin.close(); } - admin = spy(PulsarAdmin.builder().serviceHttpUrl(pulsarServiceList.get(0).getWebServiceAddress()).build()); + admin = spy( + createNewPulsarAdmin(PulsarAdmin.builder().serviceHttpUrl(pulsarServiceList.get(0).getWebServiceAddress())) + ); if (pulsarClient != null) { pulsarClient.shutdown(); @@ -82,6 +87,15 @@ public void internalSetup() throws Exception { private void init() throws Exception { startBroker(); } + + protected PulsarClient createNewPulsarClient(ClientBuilder clientBuilder) throws PulsarClientException { + return clientBuilder.build(); + } + + protected PulsarAdmin createNewPulsarAdmin(PulsarAdminBuilder builder) throws PulsarClientException { + return builder.build(); + } + protected void setUpBase(int numBroker,int numPartitionsOfTC, String topic, int numPartitions) throws Exception{ setBrokerCount(numBroker); internalSetup(); @@ -108,11 +122,10 @@ protected void setUpBase(int numBroker,int numPartitionsOfTC, String topic, int if (pulsarClient != null) { pulsarClient.shutdown(); } - pulsarClient = PulsarClient.builder() + pulsarClient = createNewPulsarClient(PulsarClient.builder() .serviceUrl(getPulsarServiceList().get(0).getBrokerServiceUrl()) .statsInterval(0, TimeUnit.SECONDS) - .enableTransaction(true) - .build(); + .enableTransaction(true)); } protected void createTransactionCoordinatorAssign(int numPartitionsOfTC) throws MetadataStoreException { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java index 663c1c50ce79c..696a0a7957c47 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java @@ -896,7 +896,7 @@ public void testTxnTimeoutAtTransactionMetadataStore() throws Exception{ long timeoutCountOriginal = transactionMetadataStores.stream() .mapToLong(store -> store.getMetadataStoreStats().timeoutCount).sum(); TxnID txnID = pulsarServiceList.get(0).getTransactionMetadataStoreService() - .newTransaction(new TransactionCoordinatorID(0), 1).get(); + .newTransaction(new TransactionCoordinatorID(0), 1, null).get(); Awaitility.await().until(() -> { try { getPulsarServiceList().get(0).getTransactionMetadataStoreService().getTxnMeta(txnID).get(); diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/TransactionMetadata.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/TransactionMetadata.java index 73fb43e45da68..d22b956e8b0d5 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/TransactionMetadata.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/TransactionMetadata.java @@ -41,4 +41,7 @@ public class TransactionMetadata { /** The ackedPartitions of this transaction. */ public Map> ackedPartitions; + + /** The owner of this transaction. */ + public String owner; } diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/TransactionMetadataStore.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/TransactionMetadataStore.java index 7f8280f5226b3..ff5adb4d409c7 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/TransactionMetadataStore.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/TransactionMetadataStore.java @@ -56,11 +56,12 @@ default CompletableFuture getTxnStatus(TxnID txnid) { * Create a new transaction in the transaction metadata store. * * @param timeoutInMills the timeout duration of the transaction in mills +* @param owner the role which is the owner of the transaction * @return a future represents the result of creating a new transaction. * it returns {@link TxnID} as the identifier for identifying the * transaction. */ - CompletableFuture newTransaction(long timeoutInMills); + CompletableFuture newTransaction(long timeoutInMills, String owner); /** * Add the produced partitions to transaction identified by txnid. diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/TxnMeta.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/TxnMeta.java index 104ae0ff1df0d..44f225b9448fd 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/TxnMeta.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/TxnMeta.java @@ -107,4 +107,11 @@ TxnMeta updateTxnStatus(TxnStatus newStatus, * @return transaction timeout at. */ long getTimeoutAt(); + + /** + * Return the transaction's owner. + * + * @return transaction's owner. + */ + String getOwner(); } diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/InMemTransactionMetadataStore.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/InMemTransactionMetadataStore.java index a62df1df8c6aa..0f3c5e42d7a69 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/InMemTransactionMetadataStore.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/InMemTransactionMetadataStore.java @@ -24,6 +24,7 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.LongAdder; +import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.common.policies.data.TransactionCoordinatorStats; import org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID; @@ -73,12 +74,17 @@ public CompletableFuture getTxnMeta(TxnID txnid) { } @Override - public CompletableFuture newTransaction(long timeoutInMills) { + public CompletableFuture newTransaction(long timeoutInMills, String owner) { + if (owner != null) { + if (StringUtils.isBlank(owner)) { + return CompletableFuture.failedFuture(new IllegalArgumentException("Owner can't be blank")); + } + } TxnID txnID = new TxnID( tcID.getId(), localID.getAndIncrement() ); - TxnMetaImpl txn = new TxnMetaImpl(txnID, System.currentTimeMillis(), timeoutInMills); + TxnMetaImpl txn = new TxnMetaImpl(txnID, System.currentTimeMillis(), timeoutInMills, owner); transactions.put(txnID, txn); return CompletableFuture.completedFuture(txnID); } diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java index 903b04a1b5dd1..b6eaad2e3e38f 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionMetadataStore.java @@ -33,6 +33,7 @@ import java.util.concurrent.atomic.LongAdder; import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.Position; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.MutablePair; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.client.api.transaction.TransactionCoordinatorClientException; @@ -149,8 +150,11 @@ public void handleMetadataEntry(Position position, TransactionMetadataEntry tran positions.add(position); long openTimestamp = transactionMetadataEntry.getStartTime(); long timeoutAt = transactionMetadataEntry.getTimeoutMs(); - txnMetaMap.put(transactionId, MutablePair.of(new TxnMetaImpl(txnID, - openTimestamp, timeoutAt), positions)); + final String owner = transactionMetadataEntry.hasOwner() + ? transactionMetadataEntry.getOwner() : null; + final TxnMetaImpl left = new TxnMetaImpl(txnID, + openTimestamp, timeoutAt, owner); + txnMetaMap.put(transactionId, MutablePair.of(left, positions)); recoverTracker.handleOpenStatusTransaction(txnSequenceId, timeoutAt + openTimestamp); } @@ -224,50 +228,58 @@ public CompletableFuture getTxnMeta(TxnID txnID) { } @Override - public CompletableFuture newTransaction(long timeOut) { - if (this.maxActiveTransactionsPerCoordinator != 0 - && this.maxActiveTransactionsPerCoordinator <= txnMetaMap.size()) { + public CompletableFuture newTransaction(long timeOut, String owner) { + if (this.maxActiveTransactionsPerCoordinator == 0 + || this.maxActiveTransactionsPerCoordinator > txnMetaMap.size()) { + CompletableFuture completableFuture = new CompletableFuture<>(); + FutureUtil.safeRunAsync(() -> { + if (!checkIfReady()) { + completableFuture.completeExceptionally(new CoordinatorException + .TransactionMetadataStoreStateException(tcID, State.Ready, getState(), "new Transaction")); + return; + } + + long mostSigBits = tcID.getId(); + long leastSigBits = sequenceIdGenerator.generateSequenceId(); + TxnID txnID = new TxnID(mostSigBits, leastSigBits); + long currentTimeMillis = System.currentTimeMillis(); + TransactionMetadataEntry transactionMetadataEntry = new TransactionMetadataEntry() + .setTxnidMostBits(mostSigBits) + .setTxnidLeastBits(leastSigBits) + .setStartTime(currentTimeMillis) + .setTimeoutMs(timeOut) + .setMetadataOp(TransactionMetadataEntry.TransactionMetadataOp.NEW) + .setLastModificationTime(currentTimeMillis) + .setMaxLocalTxnId(sequenceIdGenerator.getCurrentSequenceId()); + if (owner != null) { + if (StringUtils.isBlank(owner)) { + completableFuture.completeExceptionally(new IllegalArgumentException("Owner can't be blank")); + return; + } + transactionMetadataEntry.setOwner(owner); + } + transactionLog.append(transactionMetadataEntry) + .whenComplete((position, throwable) -> { + if (throwable != null) { + completableFuture.completeExceptionally(throwable); + } else { + appendLogCount.increment(); + TxnMeta txn = new TxnMetaImpl(txnID, currentTimeMillis, timeOut, owner); + List positions = new ArrayList<>(); + positions.add(position); + Pair> pair = MutablePair.of(txn, positions); + txnMetaMap.put(leastSigBits, pair); + this.timeoutTracker.addTransaction(leastSigBits, timeOut); + createdTransactionCount.increment(); + completableFuture.complete(txnID); + } + }); + }, internalPinnedExecutor, completableFuture); + return completableFuture; + } else { return FutureUtil.failedFuture(new CoordinatorException.ReachMaxActiveTxnException("New txn op " + "reach max active txn! tcId : " + getTransactionCoordinatorID().getId())); } - CompletableFuture completableFuture = new CompletableFuture<>(); - FutureUtil.safeRunAsync(() -> { - if (!checkIfReady()) { - completableFuture.completeExceptionally(new CoordinatorException - .TransactionMetadataStoreStateException(tcID, State.Ready, getState(), "new Transaction")); - return; - } - - long mostSigBits = tcID.getId(); - long leastSigBits = sequenceIdGenerator.generateSequenceId(); - TxnID txnID = new TxnID(mostSigBits, leastSigBits); - long currentTimeMillis = System.currentTimeMillis(); - TransactionMetadataEntry transactionMetadataEntry = new TransactionMetadataEntry() - .setTxnidMostBits(mostSigBits) - .setTxnidLeastBits(leastSigBits) - .setStartTime(currentTimeMillis) - .setTimeoutMs(timeOut) - .setMetadataOp(TransactionMetadataEntry.TransactionMetadataOp.NEW) - .setLastModificationTime(currentTimeMillis) - .setMaxLocalTxnId(sequenceIdGenerator.getCurrentSequenceId()); - transactionLog.append(transactionMetadataEntry) - .whenComplete((position, throwable) -> { - if (throwable != null) { - completableFuture.completeExceptionally(throwable); - } else { - appendLogCount.increment(); - TxnMeta txn = new TxnMetaImpl(txnID, currentTimeMillis, timeOut); - List positions = new ArrayList<>(); - positions.add(position); - Pair> pair = MutablePair.of(txn, positions); - txnMetaMap.put(leastSigBits, pair); - this.timeoutTracker.addTransaction(leastSigBits, timeOut); - createdTransactionCount.increment(); - completableFuture.complete(txnID); - } - }); - }, internalPinnedExecutor, completableFuture); - return completableFuture; } @Override @@ -300,9 +312,9 @@ public CompletableFuture addProducedPartitionToTxn(TxnID txnID, List partitions = new ArrayList<>(); @@ -181,7 +181,7 @@ public void testRecoverSequenceId(boolean isUseManagedLedgerProperties) throws E transactionMetadataStore.init(new TransactionRecoverTrackerImpl()).get(); Awaitility.await().until(transactionMetadataStore::checkIfReady); - TxnID txnID = transactionMetadataStore.newTransaction(20000).get(); + TxnID txnID = transactionMetadataStore.newTransaction(20000, null).get(); transactionMetadataStore.updateTxnStatus(txnID, TxnStatus.COMMITTING, TxnStatus.OPEN, false).get(); if (isUseManagedLedgerProperties) { transactionMetadataStore.updateTxnStatus(txnID, TxnStatus.COMMITTED, TxnStatus.COMMITTING, false).get(); @@ -209,7 +209,7 @@ public void testRecoverSequenceId(boolean isUseManagedLedgerProperties) throws E transactionMetadataStore.init(new TransactionRecoverTrackerImpl()).get(); Awaitility.await().until(transactionMetadataStore::checkIfReady); - txnID = transactionMetadataStore.newTransaction(100000).get(); + txnID = transactionMetadataStore.newTransaction(100000, null).get(); assertEquals(txnID.getLeastSigBits(), 1); } @@ -244,10 +244,8 @@ public void testInitTransactionReader(TxnLogBufferedWriterConfig txnLogBufferedW break; } if (transactionMetadataStore.checkIfReady()) { - CompletableFuture txIDFuture1 = transactionMetadataStore.newTransaction(1000); - CompletableFuture txIDFuture2 = transactionMetadataStore.newTransaction(1000); - TxnID txnID1 = txIDFuture1.get(); - TxnID txnID2 = txIDFuture2.get(); + TxnID txnID1 = transactionMetadataStore.newTransaction(1000, "user1").get(); + TxnID txnID2 = transactionMetadataStore.newTransaction(1000, "user2").get(); assertEquals(transactionMetadataStore.getTxnStatus(txnID1).get(), TxnStatus.OPEN); assertEquals(transactionMetadataStore.getTxnStatus(txnID2).get(), TxnStatus.OPEN); @@ -306,6 +304,9 @@ public void testInitTransactionReader(TxnLogBufferedWriterConfig txnLogBufferedW assertEquals(txnMeta2.ackedPartitions().size(), subscriptions.size()); Assert.assertTrue(subscriptions.containsAll(txnMeta1.ackedPartitions())); Assert.assertTrue(subscriptions.containsAll(txnMeta2.ackedPartitions())); + + assertEquals(txnMeta1.getOwner(), "user1"); + assertEquals(txnMeta2.getOwner(), "user2"); assertEquals(txnMeta1.status(), TxnStatus.COMMITTING); assertEquals(txnMeta2.status(), TxnStatus.COMMITTING); transactionMetadataStoreTest @@ -325,7 +326,7 @@ public void testInitTransactionReader(TxnLogBufferedWriterConfig txnLogBufferedW } catch (ExecutionException e) { Assert.assertTrue(e.getCause() instanceof TransactionNotFoundException); } - TxnID txnID = transactionMetadataStoreTest.newTransaction(1000).get(); + TxnID txnID = transactionMetadataStoreTest.newTransaction(1000, null).get(); assertEquals(txnID.getLeastSigBits(), 2L); break; } else { @@ -370,10 +371,8 @@ public void testDeleteLog(TxnLogBufferedWriterConfig txnLogBufferedWriterConfig) break; } if (transactionMetadataStore.checkIfReady()) { - CompletableFuture txIDFuture1 = transactionMetadataStore.newTransaction(1000); - CompletableFuture txIDFuture2 = transactionMetadataStore.newTransaction(1000); - TxnID txnID1 = txIDFuture1.get(); - TxnID txnID2 = txIDFuture2.get(); + TxnID txnID1 = transactionMetadataStore.newTransaction(1000, null).get(); + TxnID txnID2 = transactionMetadataStore.newTransaction(1000, null).get(); assertEquals(transactionMetadataStore.getTxnStatus(txnID1).get(), TxnStatus.OPEN); assertEquals(transactionMetadataStore.getTxnStatus(txnID2).get(), TxnStatus.OPEN); @@ -447,9 +446,9 @@ public void testRecoverWhenDeleteFromCursor(TxnLogBufferedWriterConfig txnLogBuf Awaitility.await().until(transactionMetadataStore::checkIfReady); // txnID1 have not deleted from cursor, we can recover from transaction log - TxnID txnID1 = transactionMetadataStore.newTransaction(1000).get(); + TxnID txnID1 = transactionMetadataStore.newTransaction(1000, null).get(); // txnID2 have deleted from cursor. - TxnID txnID2 = transactionMetadataStore.newTransaction(1000).get(); + TxnID txnID2 = transactionMetadataStore.newTransaction(1000, null).get(); transactionMetadataStore.updateTxnStatus(txnID2, TxnStatus.ABORTING, TxnStatus.OPEN, false).get(); transactionMetadataStore.updateTxnStatus(txnID2, TxnStatus.ABORTED, TxnStatus.ABORTING, false).get(); @@ -485,7 +484,7 @@ public void testManageLedgerWriteFailState(TxnLogBufferedWriterConfig txnLogBuff transactionMetadataStore.init(new TransactionRecoverTrackerImpl()).get(); Awaitility.await().until(transactionMetadataStore::checkIfReady); - transactionMetadataStore.newTransaction(5000).get(); + transactionMetadataStore.newTransaction(5000, null).get(); Field field = MLTransactionLogImpl.class.getDeclaredField("managedLedger"); field.setAccessible(true); ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) field.get(mlTransactionLog); @@ -494,12 +493,12 @@ public void testManageLedgerWriteFailState(TxnLogBufferedWriterConfig txnLogBuff AtomicReferenceFieldUpdater state = (AtomicReferenceFieldUpdater) field.get(managedLedger); state.set(managedLedger, WriteFailed); try { - transactionMetadataStore.newTransaction(5000).get(); + transactionMetadataStore.newTransaction(5000, null).get(); fail(); } catch (ExecutionException e) { assertTrue(e.getCause() instanceof ManagedLedgerException.ManagedLedgerAlreadyClosedException); } - transactionMetadataStore.newTransaction(5000).get(); + transactionMetadataStore.newTransaction(5000, null).get(); } diff --git a/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/TransactionMetadataStoreProviderTest.java b/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/TransactionMetadataStoreProviderTest.java index c7f9cd21ac808..04b2d2fe6505d 100644 --- a/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/TransactionMetadataStoreProviderTest.java +++ b/pulsar-transaction/coordinator/src/test/java/org/apache/pulsar/transaction/coordinator/TransactionMetadataStoreProviderTest.java @@ -92,14 +92,14 @@ public void testGetTxnStatusNotFound() throws Exception { @Test public void testGetTxnStatusSuccess() throws Exception { - TxnID txnID = this.store.newTransaction(0L).get(); + TxnID txnID = this.store.newTransaction(0L, null).get(); TxnStatus txnStatus = this.store.getTxnStatus(txnID).get(); assertEquals(txnStatus, TxnStatus.OPEN); } @Test public void testUpdateTxnStatusSuccess() throws Exception { - TxnID txnID = this.store.newTransaction(0L).get(); + TxnID txnID = this.store.newTransaction(0L, null).get(); TxnStatus txnStatus = this.store.getTxnStatus(txnID).get(); assertEquals(txnStatus, TxnStatus.OPEN); @@ -113,7 +113,7 @@ public void testUpdateTxnStatusSuccess() throws Exception { @Test public void testUpdateTxnStatusNotExpectedStatus() throws Exception { - TxnID txnID = this.store.newTransaction(0L).get(); + TxnID txnID = this.store.newTransaction(0L, null).get(); TxnStatus txnStatus = this.store.getTxnStatus(txnID).get(); assertEquals(txnStatus, TxnStatus.OPEN); @@ -132,7 +132,7 @@ public void testUpdateTxnStatusNotExpectedStatus() throws Exception { @Test public void testUpdateTxnStatusCannotTransition() throws Exception { - TxnID txnID = this.store.newTransaction(0L).get(); + TxnID txnID = this.store.newTransaction(0L, null).get(); TxnStatus txnStatus = this.store.getTxnStatus(txnID).get(); assertEquals(txnStatus, TxnStatus.OPEN); @@ -151,7 +151,7 @@ public void testUpdateTxnStatusCannotTransition() throws Exception { @Test public void testAddProducedPartition() throws Exception { - TxnID txnID = this.store.newTransaction(0L).get(); + TxnID txnID = this.store.newTransaction(0L, null).get(); TxnStatus txnStatus = this.store.getTxnStatus(txnID).get(); assertEquals(txnStatus, TxnStatus.OPEN); @@ -205,7 +205,7 @@ public void testAddProducedPartition() throws Exception { @Test public void testAddAckedPartition() throws Exception { - TxnID txnID = this.store.newTransaction(0L).get(); + TxnID txnID = this.store.newTransaction(0L, null).get(); TxnStatus txnStatus = this.store.getTxnStatus(txnID).get(); assertEquals(txnStatus, TxnStatus.OPEN); From 329c8c02fda225c8711bf6ef15ecbfe9e026a803 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Thu, 9 Feb 2023 23:22:42 +0800 Subject: [PATCH 002/174] [fix] [test] fix flaky test MetadataStoreStatsTest.testMetadataStoreStats (#19433) --- .../org/apache/bookkeeper/test/BookKeeperClusterTestCase.java | 2 ++ .../src/test/java/org/apache/pulsar/metadata/CounterTest.java | 1 + .../pulsar/metadata/impl/MetadataStoreFactoryImplTest.java | 4 +++- .../bookkeeper/bookkeeper/test/MockedBookKeeperTestCase.java | 1 + 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java b/managed-ledger/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java index c889f94b60801..80bb6256591bc 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java @@ -207,6 +207,8 @@ public void tearDown() throws Exception { } // stop zookeeper service try { + // cleanup for metrics. + metadataStore.close(); stopZKCluster(); } catch (Exception e) { LOG.error("Got Exception while trying to stop ZKCluster", e); diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/CounterTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/CounterTest.java index ead80a0287348..c5b4012f0c8f9 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/CounterTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/CounterTest.java @@ -87,6 +87,7 @@ public void testCounterDoesNotAutoReset(String provider, Supplier urlSup // Delete all the empty container nodes zks.checkContainers(); + @Cleanup MetadataStoreExtended store2 = MetadataStoreExtended.create(metadataUrl, MetadataStoreConfig.builder().build()); @Cleanup CoordinationService cs2 = new CoordinationServiceImpl(store2); diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/MetadataStoreFactoryImplTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/MetadataStoreFactoryImplTest.java index ba12ddff06234..c0159be4303bc 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/MetadataStoreFactoryImplTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/MetadataStoreFactoryImplTest.java @@ -20,6 +20,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; +import lombok.Cleanup; import org.apache.pulsar.metadata.api.GetResult; import org.apache.pulsar.metadata.api.MetadataStore; import org.apache.pulsar.metadata.api.MetadataStoreConfig; @@ -55,7 +56,8 @@ public void resetMetadataStoreProperty() { @Test - public void testCreate() throws MetadataStoreException{ + public void testCreate() throws Exception{ + @Cleanup MetadataStore instance = MetadataStoreFactoryImpl.create( "custom://localhost", MetadataStoreConfig.builder().build()); diff --git a/pulsar-package-management/bookkeeper-storage/src/test/java/org/apache/pulsar/packages/management/storage/bookkeeper/bookkeeper/test/MockedBookKeeperTestCase.java b/pulsar-package-management/bookkeeper-storage/src/test/java/org/apache/pulsar/packages/management/storage/bookkeeper/bookkeeper/test/MockedBookKeeperTestCase.java index 052d643ba7192..dabaf10cfe390 100644 --- a/pulsar-package-management/bookkeeper-storage/src/test/java/org/apache/pulsar/packages/management/storage/bookkeeper/bookkeeper/test/MockedBookKeeperTestCase.java +++ b/pulsar-package-management/bookkeeper-storage/src/test/java/org/apache/pulsar/packages/management/storage/bookkeeper/bookkeeper/test/MockedBookKeeperTestCase.java @@ -131,6 +131,7 @@ protected void stopBookKeeper() throws Exception { } protected void stopMetadataStore() throws Exception { + metadataStore.close(); metadataStore.setAlwaysFail(new MetadataStoreException("error")); } From b969fe5e56cc768632e52e9534a1e94b75c29be1 Mon Sep 17 00:00:00 2001 From: tison Date: Fri, 10 Feb 2023 06:34:41 +0800 Subject: [PATCH 003/174] [improve][io] Remove kafka-connect-avro-converter-shaded (#19468) Signed-off-by: tison --- build/run_unit_group.sh | 1 - kafka-connect-avro-converter-shaded/pom.xml | 118 ------------------ pom.xml | 3 - pulsar-io/kafka-connect-adaptor/pom.xml | 20 +-- .../connect/AbstractKafkaConnectSource.java | 6 +- .../io/kafka/connect/KafkaConnectSource.java | 2 +- .../schema/KafkaSchemaWrappedSchema.java | 5 +- .../schema/PulsarSchemaToKafkaSchema.java | 12 +- .../debezium/PulsarDebeziumSourcesTest.java | 3 +- 9 files changed, 16 insertions(+), 154 deletions(-) delete mode 100644 kafka-connect-avro-converter-shaded/pom.xml diff --git a/build/run_unit_group.sh b/build/run_unit_group.sh index af3ce57d27bd2..ba49820ed1d33 100755 --- a/build/run_unit_group.sh +++ b/build/run_unit_group.sh @@ -179,7 +179,6 @@ function test_group_other() { } function test_group_pulsar_io() { - $MVN_TEST_OPTIONS -pl kafka-connect-avro-converter-shaded clean install echo "::group::Running pulsar-io tests" mvn_test --install -Ppulsar-io-tests,-main echo "::endgroup::" diff --git a/kafka-connect-avro-converter-shaded/pom.xml b/kafka-connect-avro-converter-shaded/pom.xml deleted file mode 100644 index a907269086b57..0000000000000 --- a/kafka-connect-avro-converter-shaded/pom.xml +++ /dev/null @@ -1,118 +0,0 @@ - - - - 4.0.0 - - pulsar - org.apache.pulsar - 2.12.0-SNAPSHOT - .. - - - kafka-connect-avro-converter-shaded - Apache Pulsar :: Kafka Connect Avro Converter shaded - - - - - io.confluent - kafka-connect-avro-converter - ${confluent.version} - - - - - - - org.apache.maven.plugins - maven-shade-plugin - - - ${shadePluginPhase} - - shade - - - - - true - true - - - - io.confluent:* - io.confluent:kafka-avro-serializer - io.confluent:kafka-schema-registry-client - io.confluent:common-config - io.confluent:common-utils - org.apache.avro:* - - org.codehaus.jackson:jackson-core-asl - org.codehaus.jackson:jackson-mapper-asl - com.thoughtworks.paranamer:paranamer - org.xerial.snappy:snappy-java - org.apache.commons:commons-compress - org.tukaani:xz - - - - - io.confluent - org.apache.pulsar.kafka.shade.io.confluent - - - org.apache.avro - org.apache.pulsar.kafka.shade.avro - - - org.codehaus.jackson - org.apache.pulsar.kafka.shade.org.codehaus.jackson - - - com.thoughtworks.paranamer - org.apache.pulsar.kafka.shade.com.thoughtworks.paranamer - - - org.xerial.snappy - org.apache.pulsar.kafka.shade.org.xerial.snappy - - - org.apache.commons - org.apache.pulsar.kafka.shade.org.apache.commons - - - org.tukaani - org.apache.pulsar.kafka.shade.org.tukaani - - - - - - - - - - - - diff --git a/pom.xml b/pom.xml index 3d7a2934da406..bf1ad305e3b20 100644 --- a/pom.xml +++ b/pom.xml @@ -2149,9 +2149,6 @@ flexible messaging model and an intuitive client API. pulsar-io - - kafka-connect-avro-converter-shaded - bouncy-castle diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml index 52ea8c03a7181..aee1b800b489c 100644 --- a/pulsar-io/kafka-connect-adaptor/pom.xml +++ b/pulsar-io/kafka-connect-adaptor/pom.xml @@ -104,25 +104,13 @@ commons-lang3 + - ${project.groupId} - kafka-connect-avro-converter-shaded - ${project.version} - - - io.confluent - * - - - org.apache.avro - * - - + io.confluent + kafka-connect-avro-converter + ${confluent.version} - - - ${project.groupId} pulsar-broker diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/AbstractKafkaConnectSource.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/AbstractKafkaConnectSource.java index 8743bb071ea6a..6b4ae9d080257 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/AbstractKafkaConnectSource.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/AbstractKafkaConnectSource.java @@ -18,6 +18,9 @@ */ package org.apache.pulsar.io.kafka.connect; +import io.confluent.connect.avro.AvroConverter; +import io.confluent.kafka.schemaregistry.client.MockSchemaRegistryClient; +import io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; @@ -43,9 +46,6 @@ import org.apache.pulsar.io.core.Source; import org.apache.pulsar.io.core.SourceContext; import org.apache.pulsar.io.kafka.connect.schema.KafkaSchemaWrappedSchema; -import org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroConverter; -import org.apache.pulsar.kafka.shade.io.confluent.kafka.schemaregistry.client.MockSchemaRegistryClient; -import org.apache.pulsar.kafka.shade.io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig; /** * A pulsar source that runs. diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java index ca222fa58a6ba..f5f6efd08bd99 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java @@ -20,6 +20,7 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; +import io.confluent.connect.avro.AvroData; import java.util.Base64; import java.util.Map; import java.util.Optional; @@ -34,7 +35,6 @@ import org.apache.pulsar.functions.api.KVRecord; import org.apache.pulsar.io.core.SourceContext; import org.apache.pulsar.io.kafka.connect.schema.KafkaSchemaWrappedSchema; -import org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroData; /** * A pulsar source that runs. diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaWrappedSchema.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaWrappedSchema.java index 7f85d1027aeb5..7a5192ac719bf 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaWrappedSchema.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaWrappedSchema.java @@ -36,10 +36,9 @@ @Slf4j public class KafkaSchemaWrappedSchema implements Schema, Serializable { - private SchemaInfo schemaInfo = null; + private final SchemaInfo schemaInfo; - public KafkaSchemaWrappedSchema(org.apache.pulsar.kafka.shade.avro.Schema schema, - Converter converter) { + public KafkaSchemaWrappedSchema(org.apache.avro.Schema schema, Converter converter) { Map props = new HashMap<>(); boolean isJsonConverter = converter instanceof JsonConverter; props.put(GenericAvroSchema.OFFSET_PROP, isJsonConverter ? "0" : "5"); diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/PulsarSchemaToKafkaSchema.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/PulsarSchemaToKafkaSchema.java index 181c4c22eb4b0..2eb6573374cdb 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/PulsarSchemaToKafkaSchema.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/PulsarSchemaToKafkaSchema.java @@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.ExecutionError; import com.google.common.util.concurrent.UncheckedExecutionException; +import io.confluent.connect.avro.AvroData; import java.nio.charset.StandardCharsets; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -37,7 +38,6 @@ import org.apache.kafka.connect.errors.DataException; import org.apache.pulsar.client.api.schema.KeyValueSchema; import org.apache.pulsar.common.schema.SchemaType; -import org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroData; @Slf4j public class PulsarSchemaToKafkaSchema { @@ -74,9 +74,8 @@ public static boolean matchesToKafkaLogicalSchema(Schema kafkaSchema) { } // Parse json to shaded schema - private static org.apache.pulsar.kafka.shade.avro.Schema parseAvroSchema(String schemaJson) { - final org.apache.pulsar.kafka.shade.avro.Schema.Parser parser = - new org.apache.pulsar.kafka.shade.avro.Schema.Parser(); + private static org.apache.avro.Schema parseAvroSchema(String schemaJson) { + final org.apache.avro.Schema.Parser parser = new org.apache.avro.Schema.Parser(); parser.setValidateDefaults(false); return parser.parse(schemaJson); } @@ -126,9 +125,8 @@ public static Schema getKafkaConnectSchema(org.apache.pulsar.client.api.Schema p getKafkaConnectSchema(kvSchema.getValueSchema())) .build(); } - org.apache.pulsar.kafka.shade.avro.Schema avroSchema = - parseAvroSchema(new String(pulsarSchema.getSchemaInfo().getSchema(), - StandardCharsets.UTF_8)); + org.apache.avro.Schema avroSchema = parseAvroSchema( + new String(pulsarSchema.getSchemaInfo().getSchema(), StandardCharsets.UTF_8)); return avroData.toConnectSchema(avroSchema); }); } catch (ExecutionException | UncheckedExecutionException | ExecutionError ee) { diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarDebeziumSourcesTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarDebeziumSourcesTest.java index 9da1f10e74afa..5c57c904fc77f 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarDebeziumSourcesTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarDebeziumSourcesTest.java @@ -60,8 +60,7 @@ public void testDebeziumMySqlSourceJsonWithClientBuilder() throws Exception { @Test(groups = "source") public void testDebeziumMySqlSourceAvro() throws Exception { - testDebeziumMySqlConnect( - "org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroConverter", false, false); + testDebeziumMySqlConnect("io.confluent.connect.avro.AvroConverter", false, false); } @Test(groups = "source") From 0205148ca84af194c42535c16d103a6f7851607f Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Fri, 10 Feb 2023 16:26:01 -0600 Subject: [PATCH 004/174] [fix][fn] Fix k8s merge runtime opts bug (#19481) Fixes: https://github.com/apache/pulsar/issues/19478 ### Motivation See issue for additional context. Essentially, we are doing a shallow clone when we needed a deep clone. The consequence is leaked labels, annotations, and tolerations. ### Modifications * Add a `deepClone` method to the `BasicKubernetesManifestCustomizer.RuntimeOpts` method. Note that this method is not technically a deep clone for the k8s objects. However, based on the way we "merge" these objects, it is sufficient to copy references to the objects. ### Verifying this change Added a test that fails before the change and passes afterwards. ### Documentation - [x] `doc-not-needed` This is an internal bug fix. No docs needed. ### Matching PR in forked repository PR in forked repository: https://github.com/michaeljmarshall/pulsar/pull/27 --- .../BasicKubernetesManifestCustomizer.java | 21 ++++++-- ...BasicKubernetesManifestCustomizerTest.java | 53 +++++++++++++++++++ 2 files changed, 71 insertions(+), 3 deletions(-) diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/BasicKubernetesManifestCustomizer.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/BasicKubernetesManifestCustomizer.java index d82a8f2d1d671..3f02ff4849563 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/BasicKubernetesManifestCustomizer.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/BasicKubernetesManifestCustomizer.java @@ -62,7 +62,7 @@ public class BasicKubernetesManifestCustomizer implements KubernetesManifestCust @Setter @NoArgsConstructor @AllArgsConstructor - @Builder(toBuilder = true) + @Builder() public static class RuntimeOpts { private String jobNamespace; private String jobName; @@ -71,6 +71,21 @@ public static class RuntimeOpts { private Map nodeSelectorLabels; private V1ResourceRequirements resourceRequirements; private List tolerations; + + /** + * A clone where the maps and lists are properly cloned. The k8s resources themselves are shallow clones. + */ + public RuntimeOpts partialDeepClone() { + return new RuntimeOpts( + jobNamespace, + jobName, + extraLabels != null ? new HashMap<>(extraLabels) : null, + extraAnnotations != null ? new HashMap<>(extraAnnotations) : null, + nodeSelectorLabels != null ? new HashMap<>(nodeSelectorLabels) : null, + resourceRequirements, + tolerations != null ? new ArrayList<>(tolerations) : null + ); + } } @Getter @@ -82,7 +97,7 @@ public void initialize(Map config) { RuntimeOpts opts = ObjectMapperFactory.getMapper().getObjectMapper().convertValue(config, RuntimeOpts.class); if (opts != null) { - runtimeOpts = opts.toBuilder().build(); + runtimeOpts = opts; } } else { log.warn("initialize with null config"); @@ -176,7 +191,7 @@ private V1ObjectMeta updateMeta(RuntimeOpts opts, V1ObjectMeta meta) { } public static RuntimeOpts mergeRuntimeOpts(RuntimeOpts oriOpts, RuntimeOpts newOpts) { - RuntimeOpts mergedOpts = oriOpts.toBuilder().build(); + RuntimeOpts mergedOpts = oriOpts.partialDeepClone(); if (mergedOpts.getExtraLabels() == null) { mergedOpts.setExtraLabels(new HashMap<>()); } diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/BasicKubernetesManifestCustomizerTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/BasicKubernetesManifestCustomizerTest.java index 0f42755694288..d70345cdfbdd3 100644 --- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/BasicKubernetesManifestCustomizerTest.java +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/BasicKubernetesManifestCustomizerTest.java @@ -24,8 +24,10 @@ import io.kubernetes.client.openapi.models.V1Toleration; import org.testng.annotations.Test; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import static org.testng.Assert.assertEquals; @@ -93,4 +95,55 @@ public void TestMergeRuntimeOpts() { assertEquals(mergedOpts.getResourceRequirements().getLimits().get("cpu").getNumber().intValue(), 20); assertEquals(mergedOpts.getResourceRequirements().getLimits().get("memory").getNumber().intValue(), 10240); } + + // Note: this test creates many new objects to ensure that the tests guarantees objects are not mutated + // unexpectedly. + @Test + public void testMergeRuntimeOptsDoesNotModifyArguments() { + BasicKubernetesManifestCustomizer.RuntimeOpts opts1 = new BasicKubernetesManifestCustomizer.RuntimeOpts( + "namespace1", "job1", new HashMap<>(), new HashMap<>(), new HashMap<>(), new V1ResourceRequirements(), + new ArrayList<>()); + + HashMap testMap = new HashMap<>(); + testMap.put("testKey", "testValue"); + + List testList = new ArrayList<>(); + testList.add(new V1Toleration()); + + V1ResourceRequirements requirements = new V1ResourceRequirements(); + requirements.setLimits(new HashMap<>()); + BasicKubernetesManifestCustomizer.RuntimeOpts opts2 = new BasicKubernetesManifestCustomizer.RuntimeOpts( + "namespace2", "job2", testMap, testMap, testMap,requirements, testList); + + // Merge the runtime opts + BasicKubernetesManifestCustomizer.RuntimeOpts result = + BasicKubernetesManifestCustomizer.mergeRuntimeOpts(opts1, opts2); + + // Assert opts1 is same + assertEquals("namespace1", opts1.getJobNamespace()); + assertEquals("job1", opts1.getJobName()); + assertEquals(new HashMap<>(), opts1.getNodeSelectorLabels()); + assertEquals(new HashMap<>(), opts1.getExtraAnnotations()); + assertEquals(new HashMap<>(), opts1.getExtraLabels()); + assertEquals(new ArrayList<>(), opts1.getTolerations()); + assertEquals(new V1ResourceRequirements(), opts1.getResourceRequirements()); + + // Assert opts2 is same + HashMap expectedTestMap = new HashMap<>(); + expectedTestMap.put("testKey", "testValue"); + + List expectedTestList = new ArrayList<>(); + expectedTestList.add(new V1Toleration()); + + V1ResourceRequirements expectedRequirements = new V1ResourceRequirements(); + expectedRequirements.setLimits(new HashMap<>()); + + assertEquals("namespace2", opts2.getJobNamespace()); + assertEquals("job2", opts2.getJobName()); + assertEquals(expectedTestMap, opts2.getNodeSelectorLabels()); + assertEquals(expectedTestMap, opts2.getExtraAnnotations()); + assertEquals(expectedTestMap, opts2.getExtraLabels()); + assertEquals(expectedTestList, opts2.getTolerations()); + assertEquals(expectedRequirements, opts2.getResourceRequirements()); + } } From 5c8f92965637fbcd2b7e9e0e429884ce527e73ae Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Sat, 11 Feb 2023 03:37:10 -0800 Subject: [PATCH 005/174] [improve][broker] Implemented ExtensibleLoadManagerWrapper.getLoadBalancingMetrics() (#19440) --- .../extensions/ExtensibleLoadManagerImpl.java | 57 +++++ .../ExtensibleLoadManagerWrapper.java | 3 +- .../channel/ServiceUnitStateChannel.java | 8 + .../channel/ServiceUnitStateChannelImpl.java | 116 +++++++++- .../extensions/data/BrokerLoadData.java | 35 +++ .../extensions/models/AssignCounter.java | 83 +++++++ .../extensions/models/SplitCounter.java | 98 +++++++++ .../extensions/models/SplitDecision.java | 81 +++++++ .../extensions/models/UnloadCounter.java | 128 +++++++++++ .../extensions/models/UnloadDecision.java | 1 - .../ExtensibleLoadManagerImplTest.java | 207 ++++++++++++++++++ 11 files changed, 812 insertions(+), 5 deletions(-) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/AssignCounter.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/SplitCounter.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/SplitDecision.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadCounter.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index 66c271ab22eac..171651e50c2f8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -26,6 +26,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicReference; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.PulsarServerException; @@ -39,6 +40,11 @@ import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData; import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerFilter; import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerVersionFilter; +import org.apache.pulsar.broker.loadbalance.extensions.models.AssignCounter; +import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter; +import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision; +import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter; +import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision; import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore; import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStoreException; import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStoreFactory; @@ -48,6 +54,7 @@ import org.apache.pulsar.common.naming.ServiceUnitId; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.stats.Metrics; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; @Slf4j @@ -86,6 +93,17 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager { private boolean started = false; + private final AssignCounter assignCounter = new AssignCounter(); + private final UnloadCounter unloadCounter = new UnloadCounter(); + private final SplitCounter splitCounter = new SplitCounter(); + + // record load metrics + private final AtomicReference> brokerLoadMetrics = new AtomicReference<>(); + // record unload metrics + private final AtomicReference> unloadMetrics = new AtomicReference(); + // record split metrics + private final AtomicReference> splitMetrics = new AtomicReference<>(); + private final ConcurrentOpenHashMap>> lookupRequests = ConcurrentOpenHashMap.>>newBuilder() @@ -158,15 +176,18 @@ public CompletableFuture> assign(Optional { if (brokerOpt.isPresent()) { + assignCounter.incrementSuccess(); log.info("Selected new owner broker: {} for bundle: {}.", brokerOpt.get(), bundle); return serviceUnitStateChannel.publishAssignEventAsync(bundle, brokerOpt.get()) .thenApply(Optional::of); } else { + assignCounter.incrementEmpty(); throw new IllegalStateException( "Failed to select the new owner broker for bundle: " + bundle); } }); } + assignCounter.incrementSkip(); // Already assigned, return it. return CompletableFuture.completedFuture(broker); }); @@ -265,4 +286,40 @@ private boolean isInternalTopic(String topic) { || topic.startsWith(BROKER_LOAD_DATA_STORE_TOPIC) || topic.startsWith(TOP_BUNDLES_LOAD_DATA_STORE_TOPIC); } + + void updateBrokerLoadMetrics(BrokerLoadData loadData) { + this.brokerLoadMetrics.set(loadData.toMetrics(pulsar.getAdvertisedAddress())); + } + + private void updateUnloadMetrics(UnloadDecision decision) { + unloadCounter.update(decision); + this.unloadMetrics.set(unloadCounter.toMetrics(pulsar.getAdvertisedAddress())); + } + + private void updateSplitMetrics(List decisions) { + for (var decision : decisions) { + splitCounter.update(decision); + } + this.splitMetrics.set(splitCounter.toMetrics(pulsar.getAdvertisedAddress())); + } + + public List getMetrics() { + List metricsCollection = new ArrayList<>(); + + if (this.brokerLoadMetrics.get() != null) { + metricsCollection.addAll(this.brokerLoadMetrics.get()); + } + if (this.unloadMetrics.get() != null) { + metricsCollection.addAll(this.unloadMetrics.get()); + } + if (this.splitMetrics.get() != null) { + metricsCollection.addAll(this.splitMetrics.get()); + } + + metricsCollection.addAll(this.assignCounter.toMetrics(pulsar.getAdvertisedAddress())); + + metricsCollection.addAll(this.serviceUnitStateChannel.getMetrics()); + + return metricsCollection; + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java index 6d5797eed6663..48fc4bb7ff4f0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java @@ -129,8 +129,7 @@ public void writeResourceQuotasToZooKeeper() throws Exception { @Override public List getLoadBalancingMetrics() { - // TODO: Add metrics. - return null; + return loadManager.getMetrics(); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java index fece425e75fd9..44950a21ffd20 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java @@ -19,11 +19,13 @@ package org.apache.pulsar.broker.loadbalance.extensions.channel; import java.io.Closeable; +import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.loadbalance.extensions.models.Split; import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; +import org.apache.pulsar.common.stats.Metrics; import org.apache.pulsar.metadata.api.NotificationType; import org.apache.pulsar.metadata.api.extended.SessionEvent; @@ -148,4 +150,10 @@ public interface ServiceUnitStateChannel extends Closeable { */ CompletableFuture publishSplitEventAsync(Split split); + /** + * Generates the metrics to monitor. + * @return a list of the metrics + */ + List getMetrics(); + } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index 3e0931b2d1049..d5bcd3e1436cb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -35,6 +35,8 @@ import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.MetadataState.Unstable; import static org.apache.pulsar.metadata.api.extended.SessionEvent.SessionLost; import static org.apache.pulsar.metadata.api.extended.SessionEvent.SessionReestablished; +import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -68,6 +70,7 @@ import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.stats.Metrics; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.metadata.api.NotificationType; import org.apache.pulsar.metadata.api.coordination.LeaderElectionState; @@ -112,15 +115,16 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel { private long totalCleanupCancelledCnt = 0; private volatile ChannelState channelState; - enum EventType { + public enum EventType { Assign, Split, Unload + } @Getter @AllArgsConstructor - static class Counters { + public static class Counters { private AtomicLong total; private AtomicLong failure; } @@ -863,4 +867,112 @@ private String printCleanupMetrics() { ); } + + @Override + public List getMetrics() { + var metrics = new ArrayList(); + var dimensions = new HashMap(); + dimensions.put("metric", "sunitStateChn"); + dimensions.put("broker", pulsar.getAdvertisedAddress()); + + for (var etr : ownerLookUpCounters.entrySet()) { + var dim = new HashMap<>(dimensions); + dim.put("state", etr.getKey().toString()); + var metric = Metrics.create(dim); + metric.put("brk_sunit_state_chn_owner_lookup_total", etr.getValue()); + metrics.add(metric); + } + + for (var etr : eventCounters.entrySet()) { + { + var dim = new HashMap<>(dimensions); + dim.put("event", etr.getKey().toString()); + dim.put("result", "Total"); + var metric = Metrics.create(dim); + metric.put("brk_sunit_state_chn_event_publish_ops_total", + etr.getValue().getTotal().get()); + metrics.add(metric); + } + + { + var dim = new HashMap<>(dimensions); + dim.put("event", etr.getKey().toString()); + dim.put("result", "Failure"); + var metric = Metrics.create(dim); + metric.put("brk_sunit_state_chn_event_publish_ops_total", + etr.getValue().getFailure().get()); + metrics.add(metric); + } + } + + for (var etr : handlerCounters.entrySet()) { + { + var dim = new HashMap<>(dimensions); + dim.put("event", etr.getKey().toString()); + dim.put("result", "Total"); + var metric = Metrics.create(dim); + metric.put("brk_sunit_state_chn_subscribe_ops_total", + etr.getValue().getTotal().get()); + metrics.add(metric); + } + + { + var dim = new HashMap<>(dimensions); + dim.put("event", etr.getKey().toString()); + dim.put("result", "Failure"); + var metric = Metrics.create(dim); + metric.put("brk_sunit_state_chn_subscribe_ops_total", + etr.getValue().getFailure().get()); + metrics.add(metric); + } + } + + + { + var dim = new HashMap<>(dimensions); + dim.put("result", "Total"); + var metric = Metrics.create(dim); + metric.put("brk_sunit_state_chn_cleanup_ops_total", totalCleanupCnt); + metrics.add(metric); + } + + { + var dim = new HashMap<>(dimensions); + dim.put("result", "Failure"); + var metric = Metrics.create(dim); + metric.put("brk_sunit_state_chn_cleanup_ops_total", totalCleanupErrorCnt.get()); + metrics.add(metric); + } + + { + var dim = new HashMap<>(dimensions); + dim.put("result", "Skip"); + var metric = Metrics.create(dim); + metric.put("brk_sunit_state_chn_cleanup_ops_total", totalCleanupIgnoredCnt); + metrics.add(metric); + } + + { + var dim = new HashMap<>(dimensions); + dim.put("result", "Cancel"); + var metric = Metrics.create(dim); + metric.put("brk_sunit_state_chn_cleanup_ops_total", totalCleanupCancelledCnt); + metrics.add(metric); + } + + { + var dim = new HashMap<>(dimensions); + dim.put("result", "Schedule"); + var metric = Metrics.create(dim); + metric.put("brk_sunit_state_chn_cleanup_ops_total", totalCleanupScheduledCnt); + metrics.add(metric); + } + + var metric = Metrics.create(dimensions); + metric.put("brk_sunit_state_chn_broker_cleanup_ops_total", totalBrokerCleanupTombstoneCnt); + metric.put("brk_sunit_state_chn_su_cleanup_ops_total", totalServiceUnitCleanupTombstoneCnt); + metrics.add(metric); + + return metrics; + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadData.java index 39419946992c6..c3309987bec59 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadData.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadData.java @@ -18,9 +18,13 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.data; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import lombok.EqualsAndHashCode; import lombok.Getter; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.common.stats.Metrics; import org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData; import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage; @@ -187,4 +191,35 @@ public String toString(ServiceConfiguration conf) { ); } + public List toMetrics(String advertisedBrokerAddress) { + var metrics = new ArrayList(); + var dimensions = new HashMap(); + dimensions.put("metric", "loadBalancing"); + dimensions.put("broker", advertisedBrokerAddress); + { + var metric = Metrics.create(dimensions); + metric.put("brk_lb_cpu_usage", getCpu().percentUsage()); + metric.put("brk_lb_memory_usage", getMemory().percentUsage()); + metric.put("brk_lb_directMemory_usage", getDirectMemory().percentUsage()); + metric.put("brk_lb_bandwidth_in_usage", getBandwidthIn().percentUsage()); + metric.put("brk_lb_bandwidth_out_usage", getBandwidthOut().percentUsage()); + metrics.add(metric); + } + { + var dim = new HashMap<>(dimensions); + dim.put("feature", "max_ema"); + var metric = Metrics.create(dim); + metric.put("brk_lb_resource_usage", weightedMaxEMA); + metrics.add(metric); + } + { + var dim = new HashMap<>(dimensions); + dim.put("feature", "max"); + var metric = Metrics.create(dim); + metric.put("brk_lb_resource_usage", maxResourceUsage); + metrics.add(metric); + } + return metrics; + } + } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/AssignCounter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/AssignCounter.java new file mode 100644 index 0000000000000..26ff1f5f401d1 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/AssignCounter.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.models; + +import static org.apache.pulsar.broker.loadbalance.extensions.models.AssignCounter.Label.Empty; +import static org.apache.pulsar.broker.loadbalance.extensions.models.AssignCounter.Label.Skip; +import static org.apache.pulsar.broker.loadbalance.extensions.models.AssignCounter.Label.Success; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.pulsar.common.stats.Metrics; + +/** + * Defines Unload Metrics. + */ +public class AssignCounter { + + enum Label { + Success, + Empty, + Skip, + } + + final Map breakdownCounters; + + public AssignCounter() { + breakdownCounters = Map.of( + Success, new AtomicLong(), + Empty, new AtomicLong(), + Skip, new AtomicLong() + ); + } + + + public void incrementSuccess() { + breakdownCounters.get(Success).incrementAndGet(); + } + + public void incrementEmpty() { + breakdownCounters.get(Empty).incrementAndGet(); + } + + public void incrementSkip() { + breakdownCounters.get(Skip).incrementAndGet(); + } + + public List toMetrics(String advertisedBrokerAddress) { + var metrics = new ArrayList(); + var dimensions = new HashMap(); + dimensions.put("metric", "assign"); + dimensions.put("broker", advertisedBrokerAddress); + + for (var etr : breakdownCounters.entrySet()) { + var label = etr.getKey(); + var count = etr.getValue().get(); + var breakdownDims = new HashMap<>(dimensions); + breakdownDims.put("result", label.toString()); + var breakdownMetric = Metrics.create(breakdownDims); + breakdownMetric.put("brk_lb_assign_broker_breakdown_total", count); + metrics.add(breakdownMetric); + } + + return metrics; + } +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/SplitCounter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/SplitCounter.java new file mode 100644 index 0000000000000..99406412cee2b --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/SplitCounter.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.models; + +import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Label.Failure; +import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Label.Skip; +import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Label.Success; +import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Admin; +import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Balanced; +import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Bandwidth; +import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.MsgRate; +import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Sessions; +import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Topics; +import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Unknown; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang3.mutable.MutableLong; +import org.apache.pulsar.common.stats.Metrics; + +/** + * Defines the information required for a service unit split(e.g. bundle split). + */ +public class SplitCounter { + + long splitCount = 0; + + final Map> breakdownCounters; + + public SplitCounter() { + breakdownCounters = Map.of( + Success, Map.of( + Topics, new MutableLong(), + Sessions, new MutableLong(), + MsgRate, new MutableLong(), + Bandwidth, new MutableLong(), + Admin, new MutableLong()), + Skip, Map.of( + Balanced, new MutableLong() + ), + Failure, Map.of( + Unknown, new MutableLong()) + ); + } + + public void update(SplitDecision decision) { + if (decision.label == Success) { + splitCount++; + } + breakdownCounters.get(decision.getLabel()).get(decision.getReason()).increment(); + } + + public List toMetrics(String advertisedBrokerAddress) { + List metrics = new ArrayList<>(); + Map dimensions = new HashMap<>(); + + dimensions.put("metric", "bundlesSplit"); + dimensions.put("broker", advertisedBrokerAddress); + Metrics m = Metrics.create(dimensions); + m.put("brk_lb_bundles_split_total", splitCount); + metrics.add(m); + + for (Map.Entry> etr + : breakdownCounters.entrySet()) { + var result = etr.getKey(); + for (Map.Entry counter : etr.getValue().entrySet()) { + var reason = counter.getKey(); + var count = counter.getValue(); + Map breakdownDims = new HashMap<>(dimensions); + breakdownDims.put("result", result.toString()); + breakdownDims.put("reason", reason.toString()); + Metrics breakdownMetric = Metrics.create(breakdownDims); + breakdownMetric.put("brk_lb_bundles_split_breakdown_total", count); + metrics.add(breakdownMetric); + } + } + + return metrics; + } + +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/SplitDecision.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/SplitDecision.java new file mode 100644 index 0000000000000..a3dede50c1cd8 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/SplitDecision.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.models; + +import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Label.Failure; +import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Label.Skip; +import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Label.Success; +import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Balanced; +import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Unknown; +import lombok.Data; + +/** + * Defines the information required for a service unit split(e.g. bundle split). + */ +@Data +public class SplitDecision { + Split split; + Label label; + Reason reason; + + public enum Label { + Success, + Skip, + Failure + } + + public enum Reason { + Topics, + Sessions, + MsgRate, + Bandwidth, + Admin, + Balanced, + Unknown + } + + public SplitDecision() { + split = null; + label = null; + reason = null; + } + + public void clear() { + split = null; + label = null; + reason = null; + } + + public void skip() { + label = Skip; + reason = Balanced; + } + + public void succeed(Reason reason) { + label = Success; + this.reason = reason; + } + + + public void fail() { + label = Failure; + reason = Unknown; + } + +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadCounter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadCounter.java new file mode 100644 index 0000000000000..e2a51b1248967 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadCounter.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.models; + +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Failure; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Skip; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Success; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Balanced; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.CoolDown; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBrokers; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBundles; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoLoadData; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.OutDatedData; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Overloaded; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Underloaded; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Unknown; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang3.mutable.MutableLong; +import org.apache.pulsar.common.stats.Metrics; + +/** + * Defines Unload Metrics. + */ +public class UnloadCounter { + + long unloadBrokerCount = 0; + long unloadBundleCount = 0; + + final Map> breakdownCounters; + + double loadAvg; + double loadStd; + + public UnloadCounter() { + breakdownCounters = Map.of( + Success, Map.of( + Overloaded, new MutableLong(), + Underloaded, new MutableLong()), + Skip, Map.of( + Balanced, new MutableLong(), + NoBundles, new MutableLong(), + CoolDown, new MutableLong(), + OutDatedData, new MutableLong(), + NoLoadData, new MutableLong(), + NoBrokers, new MutableLong(), + Unknown, new MutableLong()), + Failure, Map.of( + Unknown, new MutableLong()) + ); + } + + public void update(UnloadDecision decision) { + var unloads = decision.getUnloads(); + unloadBrokerCount += unloads.keySet().size(); + unloadBundleCount += unloads.values().size(); + breakdownCounters.get(decision.getLabel()).get(decision.getReason()).increment(); + loadAvg = decision.loadAvg; + loadStd = decision.loadStd; + } + + public List toMetrics(String advertisedBrokerAddress) { + + var metrics = new ArrayList(); + var dimensions = new HashMap(); + + dimensions.put("metric", "bundleUnloading"); + dimensions.put("broker", advertisedBrokerAddress); + var m = Metrics.create(dimensions); + m.put("brk_lb_unload_broker_total", unloadBrokerCount); + m.put("brk_lb_unload_bundle_total", unloadBundleCount); + metrics.add(m); + + for (var etr : breakdownCounters.entrySet()) { + var result = etr.getKey(); + for (var counter : etr.getValue().entrySet()) { + var reason = counter.getKey(); + var count = counter.getValue().longValue(); + var dim = new HashMap<>(dimensions); + dim.put("result", result.toString()); + dim.put("reason", reason.toString()); + var metric = Metrics.create(dim); + metric.put("brk_lb_unload_broker_breakdown_total", count); + metrics.add(metric); + } + } + + + if (loadAvg > 0 && loadStd > 0) { + { + var dim = new HashMap<>(dimensions); + dim.put("feature", "max_ema"); + dim.put("stat", "avg"); + var metric = Metrics.create(dim); + metric.put("brk_lb_resource_usage_stats", loadAvg); + metrics.add(metric); + } + { + var dim = new HashMap<>(dimensions); + dim.put("feature", "max_ema"); + dim.put("stat", "std"); + var metric = Metrics.create(dim); + metric.put("brk_lb_resource_usage_stats", loadStd); + metrics.add(metric); + } + } + + return metrics; + } +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadDecision.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadDecision.java index 7d6651e3ff91c..67503db34eee7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadDecision.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadDecision.java @@ -113,5 +113,4 @@ public void fail() { reason = Unknown; } - } \ No newline at end of file diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index d650567be8b3d..717846184794a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -18,6 +18,31 @@ */ package org.apache.pulsar.broker.loadbalance.extensions; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Assigned; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Free; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Owned; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Released; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Splitting; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.EventType.Assign; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.EventType.Split; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.EventType.Unload; +import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Admin; +import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Bandwidth; +import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.MsgRate; +import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Sessions; +import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Topics; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Failure; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Skip; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Success; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Balanced; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.CoolDown; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBrokers; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBundles; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoLoadData; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.OutDatedData; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Overloaded; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Underloaded; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Unknown; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.spy; @@ -28,6 +53,10 @@ import static org.testng.Assert.assertTrue; import com.google.common.collect.Sets; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import java.net.URL; import java.util.Collections; @@ -36,6 +65,7 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentMap; +import org.apache.commons.lang3.mutable.MutableLong; import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; @@ -46,8 +76,13 @@ import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerFilter; +import org.apache.pulsar.broker.loadbalance.extensions.models.AssignCounter; +import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter; +import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision; +import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter; import org.apache.pulsar.broker.lookup.LookupResult; import org.apache.pulsar.broker.namespace.LookupOptions; import org.apache.pulsar.broker.resources.NamespaceResources; @@ -60,7 +95,10 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.TenantInfo; +import org.apache.pulsar.common.stats.Metrics; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; +import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; +import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -272,6 +310,175 @@ public Map filter(Map broker assertTrue(brokerLookupData.isPresent()); } + @Test + public void testGetMetrics() throws Exception { + { + var brokerLoadMetrics = (AtomicReference>) + FieldUtils.readDeclaredField(primaryLoadManager, "brokerLoadMetrics", true); + BrokerLoadData loadData = new BrokerLoadData(); + SystemResourceUsage usage = new SystemResourceUsage(); + var cpu = new ResourceUsage(1.0, 100.0); + var memory = new ResourceUsage(800.0, 200.0); + var directMemory = new ResourceUsage(2.0, 100.0); + var bandwidthIn = new ResourceUsage(3.0, 100.0); + var bandwidthOut = new ResourceUsage(4.0, 100.0); + usage.setCpu(cpu); + usage.setMemory(memory); + usage.setDirectMemory(directMemory); + usage.setBandwidthIn(bandwidthIn); + usage.setBandwidthOut(bandwidthOut); + loadData.update(usage, 1, 2, 3, 4, conf); + brokerLoadMetrics.set(loadData.toMetrics(pulsar.getAdvertisedAddress())); + } + { + var unloadMetrics = (AtomicReference>) + FieldUtils.readDeclaredField(primaryLoadManager, "unloadMetrics", true); + UnloadCounter unloadCounter = new UnloadCounter(); + FieldUtils.writeDeclaredField(unloadCounter, "unloadBrokerCount", 2l, true); + FieldUtils.writeDeclaredField(unloadCounter, "unloadBundleCount", 3l, true); + FieldUtils.writeDeclaredField(unloadCounter, "loadAvg", 1.5, true); + FieldUtils.writeDeclaredField(unloadCounter, "loadStd", 0.3, true); + FieldUtils.writeDeclaredField(unloadCounter, "breakdownCounters", Map.of( + Success, Map.of( + Overloaded, new MutableLong(1), + Underloaded, new MutableLong(2)), + Skip, Map.of( + Balanced, new MutableLong(3), + NoBundles, new MutableLong(4), + CoolDown, new MutableLong(5), + OutDatedData, new MutableLong(6), + NoLoadData, new MutableLong(7), + NoBrokers, new MutableLong(8), + Unknown, new MutableLong(9)), + Failure, Map.of( + Unknown, new MutableLong(10)) + ), true); + unloadMetrics.set(unloadCounter.toMetrics(pulsar.getAdvertisedAddress())); + } + { + var splitMetrics = (AtomicReference>) + FieldUtils.readDeclaredField(primaryLoadManager, "splitMetrics", true); + SplitCounter splitCounter = new SplitCounter(); + FieldUtils.writeDeclaredField(splitCounter, "splitCount", 35l, true); + FieldUtils.writeDeclaredField(splitCounter, "breakdownCounters", Map.of( + SplitDecision.Label.Success, Map.of( + Topics, new MutableLong(1), + Sessions, new MutableLong(2), + MsgRate, new MutableLong(3), + Bandwidth, new MutableLong(4), + Admin, new MutableLong(5)), + SplitDecision.Label.Skip, Map.of( + SplitDecision.Reason.Balanced, new MutableLong(6) + ), + SplitDecision.Label.Failure, Map.of( + SplitDecision.Reason.Unknown, new MutableLong(7)) + ), true); + splitMetrics.set(splitCounter.toMetrics(pulsar.getAdvertisedAddress())); + } + + { + AssignCounter assignCounter = new AssignCounter(); + assignCounter.incrementSuccess(); + assignCounter.incrementEmpty(); + assignCounter.incrementEmpty(); + assignCounter.incrementSkip(); + assignCounter.incrementSkip(); + assignCounter.incrementSkip(); + FieldUtils.writeDeclaredField(primaryLoadManager, "assignCounter", assignCounter, true); + } + + { + FieldUtils.writeDeclaredField(channel1, "totalCleanupCnt", 1, true); + FieldUtils.writeDeclaredField(channel1, "totalBrokerCleanupTombstoneCnt", 2, true); + FieldUtils.writeDeclaredField(channel1, "totalServiceUnitCleanupTombstoneCnt", 3, true); + FieldUtils.writeDeclaredField(channel1, "totalCleanupErrorCnt", new AtomicLong(4), true); + FieldUtils.writeDeclaredField(channel1, "totalCleanupScheduledCnt", 5, true); + FieldUtils.writeDeclaredField(channel1, "totalCleanupIgnoredCnt", 6, true); + FieldUtils.writeDeclaredField(channel1, "totalCleanupCancelledCnt", 7, true); + FieldUtils.writeDeclaredField(channel1, "ownerLookUpCounters", Map.of( + Owned, new AtomicLong(1), + Assigned, new AtomicLong(2), + Released, new AtomicLong(3), + Splitting, new AtomicLong(4), + Free, new AtomicLong(5) + ), true); + FieldUtils.writeDeclaredField(channel1, "eventCounters", Map.of( + Assign, new ServiceUnitStateChannelImpl.Counters(new AtomicLong(1), new AtomicLong(2)), + Split, new ServiceUnitStateChannelImpl.Counters(new AtomicLong(3), new AtomicLong(4)), + Unload, new ServiceUnitStateChannelImpl.Counters(new AtomicLong(5), new AtomicLong(6)) + ), true); + + FieldUtils.writeDeclaredField(channel1, "handlerCounters", Map.of( + Owned, new ServiceUnitStateChannelImpl.Counters(new AtomicLong(1), new AtomicLong(2)), + Assigned, new ServiceUnitStateChannelImpl.Counters(new AtomicLong(3), new AtomicLong(4)), + Released, new ServiceUnitStateChannelImpl.Counters(new AtomicLong(5), new AtomicLong(6)), + Splitting, new ServiceUnitStateChannelImpl.Counters(new AtomicLong(7), new AtomicLong(8)), + Free, new ServiceUnitStateChannelImpl.Counters(new AtomicLong(9), new AtomicLong(10)) + ), true); + + } + + var expected = Set.of( + """ + dimensions=[{broker=localhost, metric=loadBalancing}], metrics=[{brk_lb_bandwidth_in_usage=3.0, brk_lb_bandwidth_out_usage=4.0, brk_lb_cpu_usage=1.0, brk_lb_directMemory_usage=2.0, brk_lb_memory_usage=400.0}] + dimensions=[{broker=localhost, feature=max_ema, metric=loadBalancing}], metrics=[{brk_lb_resource_usage=4.0}] + dimensions=[{broker=localhost, feature=max, metric=loadBalancing}], metrics=[{brk_lb_resource_usage=0.04}] + dimensions=[{broker=localhost, metric=bundleUnloading}], metrics=[{brk_lb_unload_broker_total=2, brk_lb_unload_bundle_total=3}] + dimensions=[{broker=localhost, metric=bundleUnloading, reason=Unknown, result=Failure}], metrics=[{brk_lb_unload_broker_breakdown_total=10}] + dimensions=[{broker=localhost, metric=bundleUnloading, reason=CoolDown, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=5}] + dimensions=[{broker=localhost, metric=bundleUnloading, reason=OutDatedData, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=6}] + dimensions=[{broker=localhost, metric=bundleUnloading, reason=NoBundles, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=4}] + dimensions=[{broker=localhost, metric=bundleUnloading, reason=NoLoadData, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=7}] + dimensions=[{broker=localhost, metric=bundleUnloading, reason=NoBrokers, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=8}] + dimensions=[{broker=localhost, metric=bundleUnloading, reason=Unknown, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=9}] + dimensions=[{broker=localhost, metric=bundleUnloading, reason=Balanced, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=3}] + dimensions=[{broker=localhost, metric=bundleUnloading, reason=Underloaded, result=Success}], metrics=[{brk_lb_unload_broker_breakdown_total=2}] + dimensions=[{broker=localhost, metric=bundleUnloading, reason=Overloaded, result=Success}], metrics=[{brk_lb_unload_broker_breakdown_total=1}] + dimensions=[{broker=localhost, feature=max_ema, metric=bundleUnloading, stat=avg}], metrics=[{brk_lb_resource_usage_stats=1.5}] + dimensions=[{broker=localhost, feature=max_ema, metric=bundleUnloading, stat=std}], metrics=[{brk_lb_resource_usage_stats=0.3}] + dimensions=[{broker=localhost, metric=bundlesSplit}], metrics=[{brk_lb_bundles_split_total=35}] + dimensions=[{broker=localhost, metric=bundlesSplit, reason=Bandwidth, result=Success}], metrics=[{brk_lb_bundles_split_breakdown_total=4}] + dimensions=[{broker=localhost, metric=bundlesSplit, reason=Sessions, result=Success}], metrics=[{brk_lb_bundles_split_breakdown_total=2}] + dimensions=[{broker=localhost, metric=bundlesSplit, reason=MsgRate, result=Success}], metrics=[{brk_lb_bundles_split_breakdown_total=3}] + dimensions=[{broker=localhost, metric=bundlesSplit, reason=Admin, result=Success}], metrics=[{brk_lb_bundles_split_breakdown_total=5}] + dimensions=[{broker=localhost, metric=bundlesSplit, reason=Topics, result=Success}], metrics=[{brk_lb_bundles_split_breakdown_total=1}] + dimensions=[{broker=localhost, metric=bundlesSplit, reason=Balanced, result=Skip}], metrics=[{brk_lb_bundles_split_breakdown_total=6}] + dimensions=[{broker=localhost, metric=bundlesSplit, reason=Unknown, result=Failure}], metrics=[{brk_lb_bundles_split_breakdown_total=7}] + dimensions=[{broker=localhost, metric=assign, result=Empty}], metrics=[{brk_lb_assign_broker_breakdown_total=2}] + dimensions=[{broker=localhost, metric=assign, result=Success}], metrics=[{brk_lb_assign_broker_breakdown_total=1}] + dimensions=[{broker=localhost, metric=assign, result=Skip}], metrics=[{brk_lb_assign_broker_breakdown_total=3}] + dimensions=[{broker=localhost, metric=sunitStateChn, state=Splitting}], metrics=[{brk_sunit_state_chn_owner_lookup_total=4}] + dimensions=[{broker=localhost, metric=sunitStateChn, state=Owned}], metrics=[{brk_sunit_state_chn_owner_lookup_total=1}] + dimensions=[{broker=localhost, metric=sunitStateChn, state=Released}], metrics=[{brk_sunit_state_chn_owner_lookup_total=3}] + dimensions=[{broker=localhost, metric=sunitStateChn, state=Free}], metrics=[{brk_sunit_state_chn_owner_lookup_total=5}] + dimensions=[{broker=localhost, metric=sunitStateChn, state=Assigned}], metrics=[{brk_sunit_state_chn_owner_lookup_total=2}] + dimensions=[{broker=localhost, event=Unload, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=5}] + dimensions=[{broker=localhost, event=Unload, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=6}] + dimensions=[{broker=localhost, event=Split, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=3}] + dimensions=[{broker=localhost, event=Split, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=4}] + dimensions=[{broker=localhost, event=Assign, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=1}] + dimensions=[{broker=localhost, event=Assign, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=2}] + dimensions=[{broker=localhost, event=Splitting, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=7}] + dimensions=[{broker=localhost, event=Splitting, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=8}] + dimensions=[{broker=localhost, event=Owned, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=1}] + dimensions=[{broker=localhost, event=Owned, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=2}] + dimensions=[{broker=localhost, event=Released, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=5}] + dimensions=[{broker=localhost, event=Released, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=6}] + dimensions=[{broker=localhost, event=Free, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=9}] + dimensions=[{broker=localhost, event=Free, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=10}] + dimensions=[{broker=localhost, event=Assigned, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=3}] + dimensions=[{broker=localhost, event=Assigned, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=4}] + dimensions=[{broker=localhost, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_cleanup_ops_total=1}] + dimensions=[{broker=localhost, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_cleanup_ops_total=4}] + dimensions=[{broker=localhost, metric=sunitStateChn, result=Skip}], metrics=[{brk_sunit_state_chn_cleanup_ops_total=6}] + dimensions=[{broker=localhost, metric=sunitStateChn, result=Cancel}], metrics=[{brk_sunit_state_chn_cleanup_ops_total=7}] + dimensions=[{broker=localhost, metric=sunitStateChn, result=Schedule}], metrics=[{brk_sunit_state_chn_cleanup_ops_total=5}] + dimensions=[{broker=localhost, metric=sunitStateChn}], metrics=[{brk_sunit_state_chn_broker_cleanup_ops_total=2, brk_sunit_state_chn_su_cleanup_ops_total=3}] + """.split("\n")); + var actual = primaryLoadManager.getMetrics().stream().map(m -> m.toString()).collect(Collectors.toSet()); + assertEquals(actual, expected); + } + private static void cleanTableView(ServiceUnitStateChannel channel) throws IllegalAccessException { var tv = (TableViewImpl) From 93e29161b59a37aa73e3731903666811104ecf2f Mon Sep 17 00:00:00 2001 From: Masahiro Sakamoto Date: Mon, 13 Feb 2023 09:51:29 +0900 Subject: [PATCH 006/174] [feat][cli] Add read command to pulsar-client-tools (#19298) --- .../client/cli/PulsarClientToolTest.java | 51 +++ .../client/cli/PulsarClientToolWsTest.java | 50 ++- .../pulsar/client/cli/AbstractCmdConsume.java | 250 ++++++++++++++ .../apache/pulsar/client/cli/CmdConsume.java | 222 +----------- .../org/apache/pulsar/client/cli/CmdRead.java | 324 ++++++++++++++++++ .../pulsar/client/cli/PulsarClientTool.java | 6 + .../apache/pulsar/client/cli/TestCmdRead.java | 74 ++++ 7 files changed, 760 insertions(+), 217 deletions(-) create mode 100644 pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/AbstractCmdConsume.java create mode 100644 pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdRead.java create mode 100644 pulsar-client-tools/src/test/java/org/apache/pulsar/client/cli/TestCmdRead.java diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolTest.java index 52edde856b7f3..8d416125fd1b3 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolTest.java @@ -19,7 +19,9 @@ package org.apache.pulsar.client.cli; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; import java.time.Duration; import java.util.Properties; import java.util.UUID; @@ -193,6 +195,55 @@ public void testDurableSubscribe() throws Exception { } } + @Test(timeOut = 20000) + public void testRead() throws Exception { + Properties properties = new Properties(); + properties.setProperty("serviceUrl", brokerUrl.toString()); + properties.setProperty("useTls", "false"); + + final String topicName = getTopicWithRandomSuffix("reader"); + + int numberOfMessages = 10; + @Cleanup("shutdownNow") + ExecutorService executor = Executors.newSingleThreadExecutor(); + CompletableFuture future = new CompletableFuture<>(); + executor.execute(() -> { + try { + PulsarClientTool pulsarClientToolReader = new PulsarClientTool(properties); + String[] args = {"read", "-m", "latest", "-n", Integer.toString(numberOfMessages), "--hex", "-r", "30", + topicName}; + assertEquals(pulsarClientToolReader.run(args), 0); + future.complete(null); + } catch (Throwable t) { + future.completeExceptionally(t); + } + }); + + // Make sure subscription has been created + retryStrategically((test) -> { + try { + return admin.topics().getSubscriptions(topicName).size() == 1; + } catch (Exception e) { + return false; + } + }, 10, 500); + + assertEquals(admin.topics().getSubscriptions(topicName).size(), 1); + assertTrue(admin.topics().getSubscriptions(topicName).get(0).startsWith("reader-")); + PulsarClientTool pulsarClientToolProducer = new PulsarClientTool(properties); + + String[] args = {"produce", "--messages", "Have a nice day", "-n", Integer.toString(numberOfMessages), "-r", + "20", "-p", "key1=value1", "-p", "key2=value2", "-k", "partition_key", topicName}; + assertEquals(pulsarClientToolProducer.run(args), 0); + assertFalse(future.isCompletedExceptionally()); + future.get(); + + Awaitility.await() + .ignoreExceptions() + .atMost(Duration.ofMillis(20000)) + .until(()->admin.topics().getSubscriptions(topicName).size() == 0); + } + @Test(timeOut = 20000) public void testEncryption() throws Exception { Properties properties = new Properties(); diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolWsTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolWsTest.java index 5a12e77f99eb1..77c974de80e5c 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolWsTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolWsTest.java @@ -143,4 +143,52 @@ public void testWebSocketDurableSubscriptionMode() throws Exception { Assert.assertNotNull(subscriptions); Assert.assertEquals(subscriptions.size(), 1); } -} \ No newline at end of file + + @Test(timeOut = 30000) + public void testWebSocketReader() throws Exception { + Properties properties = new Properties(); + properties.setProperty("serviceUrl", brokerUrl.toString()); + properties.setProperty("useTls", "false"); + + final String topicName = "persistent://my-property/my-ns/test/topic-" + UUID.randomUUID(); + + int numberOfMessages = 10; + { + @Cleanup("shutdown") + ExecutorService executor = Executors.newSingleThreadExecutor(); + CompletableFuture future = new CompletableFuture<>(); + executor.execute(() -> { + try { + PulsarClientTool pulsarClientToolReader = new PulsarClientTool(properties); + String[] args = {"read", "-m", "latest", "-n", Integer.toString(numberOfMessages), "--hex", "-r", + "30", topicName}; + Assert.assertEquals(pulsarClientToolReader.run(args), 0); + future.complete(null); + } catch (Throwable t) { + future.completeExceptionally(t); + } + }); + + // Make sure subscription has been created + Awaitility.await() + .pollInterval(Duration.ofMillis(200)) + .ignoreExceptions().untilAsserted(() -> { + Assert.assertEquals(admin.topics().getSubscriptions(topicName).size(), 1); + Assert.assertTrue(admin.topics().getSubscriptions(topicName).get(0).startsWith("reader-")); + }); + + PulsarClientTool pulsarClientToolProducer = new PulsarClientTool(properties); + + String[] args = {"produce", "--messages", "Have a nice day", "-n", Integer.toString(numberOfMessages), "-r", + "20", "-p", "key1=value1", "-p", "key2=value2", "-k", "partition_key", topicName}; + Assert.assertEquals(pulsarClientToolProducer.run(args), 0); + future.get(); + Assert.assertFalse(future.isCompletedExceptionally()); + } + + Awaitility.await() + .ignoreExceptions().untilAsserted(() -> { + Assert.assertEquals(admin.topics().getSubscriptions(topicName).size(), 0); + }); + } +} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/AbstractCmdConsume.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/AbstractCmdConsume.java new file mode 100644 index 0000000000000..ef0ffbc297340 --- /dev/null +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/AbstractCmdConsume.java @@ -0,0 +1,250 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.cli; + +import static org.apache.pulsar.client.internal.PulsarClientImplementationBinding.getBytes; +import com.google.common.collect.ImmutableMap; +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.apache.commons.io.HexDump; +import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.ClientBuilder; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.schema.Field; +import org.apache.pulsar.client.api.schema.GenericObject; +import org.apache.pulsar.client.api.schema.GenericRecord; +import org.apache.pulsar.common.schema.KeyValue; +import org.apache.pulsar.common.util.collections.GrowableArrayBlockingQueue; +import org.eclipse.jetty.websocket.api.RemoteEndpoint; +import org.eclipse.jetty.websocket.api.Session; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketClose; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketConnect; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage; +import org.eclipse.jetty.websocket.api.annotations.WebSocket; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * common part of consume command and read command of pulsar-client. + * + */ +public abstract class AbstractCmdConsume { + + protected static final Logger LOG = LoggerFactory.getLogger(PulsarClientTool.class); + protected static final String MESSAGE_BOUNDARY = "----- got message -----"; + + protected ClientBuilder clientBuilder; + protected Authentication authentication; + protected String serviceURL; + + public AbstractCmdConsume() { + // Do nothing + } + + /** + * Set client configuration. + * + */ + public void updateConfig(ClientBuilder clientBuilder, Authentication authentication, String serviceURL) { + this.clientBuilder = clientBuilder; + this.authentication = authentication; + this.serviceURL = serviceURL; + } + + /** + * Interprets the message to create a string representation. + * + * @param message + * The message to interpret + * @param displayHex + * Whether to display BytesMessages in hexdump style, ignored for simple text messages + * @return String representation of the message + */ + protected String interpretMessage(Message message, boolean displayHex) throws IOException { + StringBuilder sb = new StringBuilder(); + + String properties = Arrays.toString(message.getProperties().entrySet().toArray()); + + String data; + Object value = message.getValue(); + if (value == null) { + data = "null"; + } else if (value instanceof byte[]) { + byte[] msgData = (byte[]) value; + data = interpretByteArray(displayHex, msgData); + } else if (value instanceof GenericObject) { + Map asMap = genericObjectToMap((GenericObject) value, displayHex); + data = asMap.toString(); + } else if (value instanceof ByteBuffer) { + data = new String(getBytes((ByteBuffer) value)); + } else { + data = value.toString(); + } + + String key = null; + if (message.hasKey()) { + key = message.getKey(); + } + + sb.append("key:[").append(key).append("], "); + if (!properties.isEmpty()) { + sb.append("properties:").append(properties).append(", "); + } + sb.append("content:").append(data); + + return sb.toString(); + } + + protected static String interpretByteArray(boolean displayHex, byte[] msgData) throws IOException { + String data; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + if (!displayHex) { + return new String(msgData); + } else { + HexDump.dump(msgData, 0, out, 0); + return out.toString(); + } + } + + protected static Map genericObjectToMap(GenericObject value, boolean displayHex) + throws IOException { + switch (value.getSchemaType()) { + case AVRO: + case JSON: + case PROTOBUF_NATIVE: + return genericRecordToMap((GenericRecord) value, displayHex); + case KEY_VALUE: + return keyValueToMap((KeyValue) value.getNativeObject(), displayHex); + default: + return primitiveValueToMap(value.getNativeObject(), displayHex); + } + } + + protected static Map keyValueToMap(KeyValue value, boolean displayHex) throws IOException { + if (value == null) { + return ImmutableMap.of("value", "NULL"); + } + return ImmutableMap.of("key", primitiveValueToMap(value.getKey(), displayHex), + "value", primitiveValueToMap(value.getValue(), displayHex)); + } + + protected static Map primitiveValueToMap(Object value, boolean displayHex) throws IOException { + if (value == null) { + return ImmutableMap.of("value", "NULL"); + } + if (value instanceof GenericObject) { + return genericObjectToMap((GenericObject) value, displayHex); + } + if (value instanceof byte[]) { + value = interpretByteArray(displayHex, (byte[]) value); + } + return ImmutableMap.of("value", value.toString(), "type", value.getClass()); + } + + protected static Map genericRecordToMap(GenericRecord value, boolean displayHex) + throws IOException { + Map res = new HashMap<>(); + for (Field f : value.getFields()) { + Object fieldValue = value.getField(f); + if (fieldValue instanceof GenericRecord) { + fieldValue = genericRecordToMap((GenericRecord) fieldValue, displayHex); + } else if (fieldValue == null) { + fieldValue = "NULL"; + } else if (fieldValue instanceof byte[]) { + fieldValue = interpretByteArray(displayHex, (byte[]) fieldValue); + } + res.put(f.getName(), fieldValue); + } + return res; + } + + @WebSocket(maxTextMessageSize = 64 * 1024) + public static class ConsumerSocket { + private static final String X_PULSAR_MESSAGE_ID = "messageId"; + private final CountDownLatch closeLatch; + private Session session; + private CompletableFuture connected; + final BlockingQueue incomingMessages; + + public ConsumerSocket(CompletableFuture connected) { + this.closeLatch = new CountDownLatch(1); + this.connected = connected; + this.incomingMessages = new GrowableArrayBlockingQueue<>(); + } + + public boolean awaitClose(int duration, TimeUnit unit) throws InterruptedException { + return this.closeLatch.await(duration, unit); + } + + @OnWebSocketClose + public void onClose(int statusCode, String reason) { + log.info("Connection closed: {} - {}", statusCode, reason); + this.session = null; + this.closeLatch.countDown(); + } + + @OnWebSocketConnect + public void onConnect(Session session) throws InterruptedException { + log.info("Got connect: {}", session); + this.session = session; + this.connected.complete(null); + } + + @OnWebSocketMessage + public synchronized void onMessage(String msg) throws Exception { + JsonObject message = new Gson().fromJson(msg, JsonObject.class); + JsonObject ack = new JsonObject(); + String messageId = message.get(X_PULSAR_MESSAGE_ID).getAsString(); + ack.add("messageId", new JsonPrimitive(messageId)); + // Acking the proxy + this.getRemote().sendString(ack.toString()); + this.incomingMessages.put(msg); + } + + public String receive(long timeout, TimeUnit unit) throws Exception { + return incomingMessages.poll(timeout, unit); + } + + public RemoteEndpoint getRemote() { + return this.session.getRemote(); + } + + public Session getSession() { + return this.session; + } + + public void close() { + this.session.close(); + } + + private static final Logger log = LoggerFactory.getLogger(ConsumerSocket.class); + } + +} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdConsume.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdConsume.java index ef3db09a8830c..58ab6360a17bb 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdConsume.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdConsume.java @@ -19,35 +19,21 @@ package org.apache.pulsar.client.cli; import static org.apache.commons.lang3.StringUtils.isNotBlank; -import static org.apache.pulsar.client.internal.PulsarClientImplementationBinding.getBytes; import com.beust.jcommander.Parameter; import com.beust.jcommander.ParameterException; import com.beust.jcommander.Parameters; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.RateLimiter; -import com.google.gson.Gson; -import com.google.gson.JsonObject; -import com.google.gson.JsonPrimitive; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.URI; -import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Arrays; import java.util.Base64; -import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; -import org.apache.commons.io.HexDump; -import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.api.AuthenticationDataProvider; -import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.ConsumerBuilder; import org.apache.pulsar.client.api.Message; @@ -57,33 +43,17 @@ import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.SubscriptionMode; import org.apache.pulsar.client.api.SubscriptionType; -import org.apache.pulsar.client.api.schema.Field; -import org.apache.pulsar.client.api.schema.GenericObject; -import org.apache.pulsar.client.api.schema.GenericRecord; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.schema.KeyValue; -import org.apache.pulsar.common.util.collections.GrowableArrayBlockingQueue; import org.eclipse.jetty.util.ssl.SslContextFactory; -import org.eclipse.jetty.websocket.api.RemoteEndpoint; -import org.eclipse.jetty.websocket.api.Session; -import org.eclipse.jetty.websocket.api.annotations.OnWebSocketClose; -import org.eclipse.jetty.websocket.api.annotations.OnWebSocketConnect; -import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage; -import org.eclipse.jetty.websocket.api.annotations.WebSocket; import org.eclipse.jetty.websocket.client.ClientUpgradeRequest; import org.eclipse.jetty.websocket.client.WebSocketClient; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * pulsar-client consume command implementation. * */ @Parameters(commandDescription = "Consume messages from a specified topic") -public class CmdConsume { - - private static final Logger LOG = LoggerFactory.getLogger(PulsarClientTool.class); - private static final String MESSAGE_BOUNDARY = "----- got message -----"; +public class CmdConsume extends AbstractCmdConsume { @Parameter(description = "TopicName", required = true) private List mainOptions = new ArrayList(); @@ -134,132 +104,14 @@ public class CmdConsume { @Parameter(names = { "-st", "--schema-type"}, description = "Set a schema type on the consumer, it can be 'bytes' or 'auto_consume'") - private String schematype = "bytes"; + private String schemaType = "bytes"; @Parameter(names = { "-pm", "--pool-messages" }, description = "Use the pooled message", arity = 1) private boolean poolMessages = true; - private ClientBuilder clientBuilder; - private Authentication authentication; - private String serviceURL; - public CmdConsume() { // Do nothing - } - - /** - * Set client configuration. - * - */ - public void updateConfig(ClientBuilder clientBuilder, Authentication authentication, String serviceURL) { - this.clientBuilder = clientBuilder; - this.authentication = authentication; - this.serviceURL = serviceURL; - } - - /** - * Interprets the message to create a string representation. - * - * @param message - * The message to interpret - * @param displayHex - * Whether to display BytesMessages in hexdump style, ignored for simple text messages - * @return String representation of the message - */ - private String interpretMessage(Message message, boolean displayHex) throws IOException { - StringBuilder sb = new StringBuilder(); - - String properties = Arrays.toString(message.getProperties().entrySet().toArray()); - - String data; - Object value = message.getValue(); - if (value == null) { - data = "null"; - } else if (value instanceof byte[]) { - byte[] msgData = (byte[]) value; - data = interpretByteArray(displayHex, msgData); - } else if (value instanceof GenericObject) { - Map asMap = genericObjectToMap((GenericObject) value, displayHex); - data = asMap.toString(); - } else if (value instanceof ByteBuffer) { - data = new String(getBytes((ByteBuffer) value)); - } else { - data = value.toString(); - } - - String key = null; - if (message.hasKey()) { - key = message.getKey(); - } - - sb.append("key:[").append(key).append("], "); - if (!properties.isEmpty()) { - sb.append("properties:").append(properties).append(", "); - } - sb.append("content:").append(data); - - return sb.toString(); - } - - private static String interpretByteArray(boolean displayHex, byte[] msgData) throws IOException { - String data; - ByteArrayOutputStream out = new ByteArrayOutputStream(); - if (!displayHex) { - return new String(msgData); - } else { - HexDump.dump(msgData, 0, out, 0); - return out.toString(); - } - } - - private static Map genericObjectToMap(GenericObject value, boolean displayHex) throws IOException { - switch (value.getSchemaType()) { - case AVRO: - case JSON: - case PROTOBUF_NATIVE: - return genericRecordToMap((GenericRecord) value, displayHex); - case KEY_VALUE: - return keyValueToMap((KeyValue) value.getNativeObject(), displayHex); - default: - return primitiveValueToMap(value.getNativeObject(), displayHex); - } - } - - private static Map keyValueToMap(KeyValue value, boolean displayHex) throws IOException { - if (value == null) { - return ImmutableMap.of("value", "NULL"); - } - return ImmutableMap.of("key", primitiveValueToMap(value.getKey(), displayHex), - "value", primitiveValueToMap(value.getValue(), displayHex)); - } - - private static Map primitiveValueToMap(Object value, boolean displayHex) throws IOException { - if (value == null) { - return ImmutableMap.of("value", "NULL"); - } - if (value instanceof GenericObject) { - return genericObjectToMap((GenericObject) value, displayHex); - } - if (value instanceof byte[]) { - value = interpretByteArray(displayHex, (byte[]) value); - } - return ImmutableMap.of("value", value.toString(), "type", value.getClass()); - } - - private static Map genericRecordToMap(GenericRecord value, boolean displayHex) throws IOException { - Map res = new HashMap<>(); - for (Field f : value.getFields()) { - Object fieldValue = value.getField(f); - if (fieldValue instanceof GenericRecord) { - fieldValue = genericRecordToMap((GenericRecord) fieldValue, displayHex); - } else if (fieldValue == null) { - fieldValue = "NULL"; - } else if (fieldValue instanceof byte[]) { - fieldValue = interpretByteArray(displayHex, (byte[]) fieldValue); - } - res.put(f.getName(), fieldValue); - } - return res; + super(); } /** @@ -294,10 +146,10 @@ private int consume(String topic) { try (PulsarClient client = clientBuilder.build()){ ConsumerBuilder builder; Schema schema = poolMessages ? Schema.BYTEBUFFER : Schema.BYTES; - if ("auto_consume".equals(schematype)) { + if ("auto_consume".equals(schemaType)) { schema = Schema.AUTO_CONSUME(); - } else if (!"bytes".equals(schematype)) { - throw new IllegalArgumentException("schema type must be 'bytes' or 'auto_consume"); + } else if (!"bytes".equals(schemaType)) { + throw new IllegalArgumentException("schema type must be 'bytes' or 'auto_consume'"); } builder = client.newConsumer(schema) .subscriptionName(this.subscriptionName) @@ -458,66 +310,4 @@ private int consumeFromWebSocket(String topic) { return returnCode; } - @WebSocket(maxTextMessageSize = 64 * 1024) - public static class ConsumerSocket { - private static final String X_PULSAR_MESSAGE_ID = "messageId"; - private final CountDownLatch closeLatch; - private Session session; - private CompletableFuture connected; - final BlockingQueue incomingMessages; - - public ConsumerSocket(CompletableFuture connected) { - this.closeLatch = new CountDownLatch(1); - this.connected = connected; - this.incomingMessages = new GrowableArrayBlockingQueue<>(); - } - - public boolean awaitClose(int duration, TimeUnit unit) throws InterruptedException { - return this.closeLatch.await(duration, unit); - } - - @OnWebSocketClose - public void onClose(int statusCode, String reason) { - log.info("Connection closed: {} - {}", statusCode, reason); - this.session = null; - this.closeLatch.countDown(); - } - - @OnWebSocketConnect - public void onConnect(Session session) throws InterruptedException { - log.info("Got connect: {}", session); - this.session = session; - this.connected.complete(null); - } - - @OnWebSocketMessage - public synchronized void onMessage(String msg) throws Exception { - JsonObject message = new Gson().fromJson(msg, JsonObject.class); - JsonObject ack = new JsonObject(); - String messageId = message.get(X_PULSAR_MESSAGE_ID).getAsString(); - ack.add("messageId", new JsonPrimitive(messageId)); - // Acking the proxy - this.getRemote().sendString(ack.toString()); - this.incomingMessages.put(msg); - } - - public String receive(long timeout, TimeUnit unit) throws Exception { - return incomingMessages.poll(timeout, unit); - } - - public RemoteEndpoint getRemote() { - return this.session.getRemote(); - } - - public Session getSession() { - return this.session; - } - - public void close() { - this.session.close(); - } - - private static final Logger log = LoggerFactory.getLogger(ConsumerSocket.class); - - } } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdRead.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdRead.java new file mode 100644 index 0000000000000..4ad8a5293f6e1 --- /dev/null +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdRead.java @@ -0,0 +1,324 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.cli; + +import static org.apache.commons.lang3.StringUtils.isNotBlank; +import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParameterException; +import com.beust.jcommander.Parameters; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.util.concurrent.RateLimiter; +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.apache.pulsar.client.api.AuthenticationDataProvider; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Reader; +import org.apache.pulsar.client.api.ReaderBuilder; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.MessageIdImpl; +import org.apache.pulsar.common.naming.TopicName; +import org.eclipse.jetty.util.ssl.SslContextFactory; +import org.eclipse.jetty.websocket.client.ClientUpgradeRequest; +import org.eclipse.jetty.websocket.client.WebSocketClient; + +/** + * pulsar-client read command implementation. + * + */ +@Parameters(commandDescription = "Read messages from a specified topic") +public class CmdRead extends AbstractCmdConsume { + + private static final Pattern MSG_ID_PATTERN = Pattern.compile("^(-?[1-9][0-9]*|0):(-?[1-9][0-9]*|0)$"); + + @Parameter(description = "TopicName", required = true) + private List mainOptions = new ArrayList(); + + @Parameter(names = { "-m", "--start-message-id" }, + description = "Initial reader position, it can be 'latest', 'earliest' or ':'") + private String startMessageId = "latest"; + + @Parameter(names = { "-i", "--start-message-id-inclusive" }, + description = "Whether to include the position specified by -m option.") + private boolean startMessageIdInclusive = false; + + @Parameter(names = { "-n", + "--num-messages" }, description = "Number of messages to read, 0 means to read forever.") + private int numMessagesToRead = 1; + + @Parameter(names = { "--hex" }, description = "Display binary messages in hex.") + private boolean displayHex = false; + + @Parameter(names = { "--hide-content" }, description = "Do not write the message to console.") + private boolean hideContent = false; + + @Parameter(names = { "-r", "--rate" }, description = "Rate (in msg/sec) at which to read, " + + "value 0 means to read messages as fast as possible.") + private double readRate = 0; + + @Parameter(names = {"-q", "--queue-size"}, description = "Reader receiver queue size.") + private int receiverQueueSize = 0; + + @Parameter(names = { "-mc", "--max_chunked_msg" }, description = "Max pending chunk messages") + private int maxPendingChunkedMessage = 0; + + @Parameter(names = { "-ac", + "--auto_ack_chunk_q_full" }, description = "Auto ack for oldest message on queue is full") + private boolean autoAckOldestChunkedMessageOnQueueFull = false; + + @Parameter(names = { "-ekv", + "--encryption-key-value" }, description = "The URI of private key to decrypt payload, for example " + + "file:///path/to/private.key or data:application/x-pem-file;base64,*****") + private String encKeyValue; + + @Parameter(names = { "-st", "--schema-type"}, + description = "Set a schema type on the reader, it can be 'bytes' or 'auto_consume'") + private String schemaType = "bytes"; + + @Parameter(names = { "-pm", "--pool-messages" }, description = "Use the pooled message", arity = 1) + private boolean poolMessages = true; + + public CmdRead() { + // Do nothing + super(); + } + + /** + * Run the read command. + * + * @return 0 for success, < 0 otherwise + */ + public int run() throws PulsarClientException, IOException { + if (mainOptions.size() != 1) { + throw (new ParameterException("Please provide one and only one topic name.")); + } + if (this.numMessagesToRead < 0) { + throw (new ParameterException("Number of messages should be zero or positive.")); + } + + String topic = this.mainOptions.get(0); + + if (this.serviceURL.startsWith("ws")) { + return readFromWebSocket(topic); + } else { + return read(topic); + } + } + + private int read(String topic) { + int numMessagesRead = 0; + int returnCode = 0; + + try (PulsarClient client = clientBuilder.build()){ + ReaderBuilder builder; + + Schema schema = poolMessages ? Schema.BYTEBUFFER : Schema.BYTES; + if ("auto_consume".equals(schemaType)) { + schema = Schema.AUTO_CONSUME(); + } else if (!"bytes".equals(schemaType)) { + throw new IllegalArgumentException("schema type must be 'bytes' or 'auto_consume'"); + } + builder = client.newReader(schema) + .topic(topic) + .startMessageId(parseMessageId(startMessageId)) + .poolMessages(poolMessages); + + if (this.startMessageIdInclusive) { + builder.startMessageIdInclusive(); + } + if (this.maxPendingChunkedMessage > 0) { + builder.maxPendingChunkedMessage(this.maxPendingChunkedMessage); + } + if (this.receiverQueueSize > 0) { + builder.receiverQueueSize(this.receiverQueueSize); + } + + builder.autoAckOldestChunkedMessageOnQueueFull(this.autoAckOldestChunkedMessageOnQueueFull); + + if (isNotBlank(this.encKeyValue)) { + builder.defaultCryptoKeyReader(this.encKeyValue); + } + + try (Reader reader = builder.create()) { + RateLimiter limiter = (this.readRate > 0) ? RateLimiter.create(this.readRate) : null; + while (this.numMessagesToRead == 0 || numMessagesRead < this.numMessagesToRead) { + if (limiter != null) { + limiter.acquire(); + } + + Message msg = reader.readNext(5, TimeUnit.SECONDS); + if (msg == null) { + LOG.debug("No message to read after waiting for 5 seconds."); + } else { + try { + numMessagesRead += 1; + if (!hideContent) { + System.out.println(MESSAGE_BOUNDARY); + String output = this.interpretMessage(msg, displayHex); + System.out.println(output); + } else if (numMessagesRead % 1000 == 0) { + System.out.println("Received " + numMessagesRead + " messages"); + } + } finally { + msg.release(); + } + } + } + } + } catch (Exception e) { + LOG.error("Error while reading messages"); + LOG.error(e.getMessage(), e); + returnCode = -1; + } finally { + LOG.info("{} messages successfully read", numMessagesRead); + } + + return returnCode; + + } + + @SuppressWarnings("deprecation") + @VisibleForTesting + public String getWebSocketReadUri(String topic) { + String serviceURLWithoutTrailingSlash = serviceURL.substring(0, + serviceURL.endsWith("/") ? serviceURL.length() - 1 : serviceURL.length()); + + TopicName topicName = TopicName.get(topic); + String wsTopic; + if (topicName.isV2()) { + wsTopic = String.format("%s/%s/%s/%s", topicName.getDomain(), topicName.getTenant(), + topicName.getNamespacePortion(), topicName.getLocalName()); + } else { + wsTopic = String.format("%s/%s/%s/%s/%s", topicName.getDomain(), topicName.getTenant(), + topicName.getCluster(), topicName.getNamespacePortion(), topicName.getLocalName()); + } + + String msgIdQueryParam; + if ("latest".equals(startMessageId) || "earliest".equals(startMessageId)) { + msgIdQueryParam = startMessageId; + } else { + MessageId msgId = parseMessageId(startMessageId); + msgIdQueryParam = Base64.getEncoder().encodeToString(msgId.toByteArray()); + } + + String uriFormat = "%s/ws" + (topicName.isV2() ? "/v2/" : "/") + "reader/%s?messageId=%s"; + return String.format(uriFormat, serviceURLWithoutTrailingSlash, wsTopic, msgIdQueryParam); + } + + @SuppressWarnings("deprecation") + private int readFromWebSocket(String topic) { + int numMessagesRead = 0; + int returnCode = 0; + + URI readerUri = URI.create(getWebSocketReadUri(topic)); + + WebSocketClient readClient = new WebSocketClient(new SslContextFactory(true)); + ClientUpgradeRequest readRequest = new ClientUpgradeRequest(); + try { + if (authentication != null) { + authentication.start(); + AuthenticationDataProvider authData = authentication.getAuthData(); + if (authData.hasDataForHttp()) { + for (Map.Entry kv : authData.getHttpHeaders()) { + readRequest.setHeader(kv.getKey(), kv.getValue()); + } + } + } + } catch (Exception e) { + LOG.error("Authentication plugin error: " + e.getMessage()); + return -1; + } + CompletableFuture connected = new CompletableFuture<>(); + ConsumerSocket readerSocket = new ConsumerSocket(connected); + try { + readClient.start(); + } catch (Exception e) { + LOG.error("Failed to start websocket-client", e); + return -1; + } + + try { + LOG.info("Trying to create websocket session..{}", readerUri); + readClient.connect(readerSocket, readerUri, readRequest); + connected.get(); + } catch (Exception e) { + LOG.error("Failed to create web-socket session", e); + return -1; + } + + try { + RateLimiter limiter = (this.readRate > 0) ? RateLimiter.create(this.readRate) : null; + while (this.numMessagesToRead == 0 || numMessagesRead < this.numMessagesToRead) { + if (limiter != null) { + limiter.acquire(); + } + String msg = readerSocket.receive(5, TimeUnit.SECONDS); + if (msg == null) { + LOG.debug("No message to read after waiting for 5 seconds."); + } else { + try { + String output = interpretByteArray(displayHex, Base64.getDecoder().decode(msg)); + System.out.println(output); // print decode + } catch (Exception e) { + System.out.println(msg); + } + numMessagesRead += 1; + } + } + readerSocket.awaitClose(2, TimeUnit.SECONDS); + } catch (Exception e) { + LOG.error("Error while reading messages"); + LOG.error(e.getMessage(), e); + returnCode = -1; + } finally { + LOG.info("{} messages successfully read", numMessagesRead); + } + + return returnCode; + } + + @VisibleForTesting + static MessageId parseMessageId(String msgIdStr) { + MessageId msgId; + if ("latest".equals(msgIdStr)) { + msgId = MessageId.latest; + } else if ("earliest".equals(msgIdStr)) { + msgId = MessageId.earliest; + } else { + Matcher matcher = MSG_ID_PATTERN.matcher(msgIdStr); + if (matcher.find()) { + msgId = new MessageIdImpl(Long.parseLong(matcher.group(1)), Long.parseLong(matcher.group(2)), -1); + } else { + throw new IllegalArgumentException("Message ID must be 'latest', 'earliest' or ':'"); + } + } + return msgId; + } + +} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/PulsarClientTool.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/PulsarClientTool.java index 2c3e6935b515e..c64d80f380b9f 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/PulsarClientTool.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/PulsarClientTool.java @@ -99,6 +99,7 @@ public static class RootParams { IUsageFormatter usageFormatter; protected CmdProduce produceCommand; protected CmdConsume consumeCommand; + protected CmdRead readCommand; CmdGenerateDocumentation generateDocumentation; public PulsarClientTool(Properties properties) { @@ -126,6 +127,7 @@ public PulsarClientTool(Properties properties) { protected void initJCommander() { produceCommand = new CmdProduce(); consumeCommand = new CmdConsume(); + readCommand = new CmdRead(); generateDocumentation = new CmdGenerateDocumentation(); this.jcommander = new JCommander(); @@ -134,6 +136,7 @@ protected void initJCommander() { jcommander.addObject(rootParams); jcommander.addCommand("produce", produceCommand); jcommander.addCommand("consume", consumeCommand); + jcommander.addCommand("read", readCommand); jcommander.addCommand("generate_documentation", generateDocumentation); } @@ -196,6 +199,7 @@ private void updateConfig() throws UnsupportedAuthenticationException { } this.produceCommand.updateConfig(clientBuilder, authentication, this.rootParams.serviceURL); this.consumeCommand.updateConfig(clientBuilder, authentication, this.rootParams.serviceURL); + this.readCommand.updateConfig(clientBuilder, authentication, this.rootParams.serviceURL); } public int run(String[] args) { @@ -231,6 +235,8 @@ public int run(String[] args) { return produceCommand.run(); } else if ("consume".equals(chosenCommand)) { return consumeCommand.run(); + } else if ("read".equals(chosenCommand)) { + return readCommand.run(); } else if ("generate_documentation".equals(chosenCommand)) { return generateDocumentation.run(); } else { diff --git a/pulsar-client-tools/src/test/java/org/apache/pulsar/client/cli/TestCmdRead.java b/pulsar-client-tools/src/test/java/org/apache/pulsar/client/cli/TestCmdRead.java new file mode 100644 index 0000000000000..ac5c562b343b8 --- /dev/null +++ b/pulsar-client-tools/src/test/java/org/apache/pulsar/client/cli/TestCmdRead.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.cli; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +import java.lang.reflect.Field; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.impl.MessageIdImpl; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class TestCmdRead { + + @DataProvider(name = "startMessageIds") + public Object[][] startMessageIds() { + return new Object[][] { + { "latest", "latest" }, + { "earliest", "earliest" }, + { "10:0", "CAoQADAA" }, + { "10:1", "CAoQATAA" }, + }; + } + + @Test(dataProvider = "startMessageIds") + public void testGetWebSocketReadUri(String msgId, String msgIdQueryParam) throws Exception { + CmdRead cmdRead = new CmdRead(); + cmdRead.updateConfig(null, null, "ws://localhost:8080/"); + Field startMessageIdField = CmdRead.class.getDeclaredField("startMessageId"); + startMessageIdField.setAccessible(true); + startMessageIdField.set(cmdRead, msgId); + + String topicNameV1 = "persistent://public/cluster/default/t1"; + assertEquals(cmdRead.getWebSocketReadUri(topicNameV1), + "ws://localhost:8080/ws/reader/persistent/public/cluster/default/t1?messageId=" + msgIdQueryParam); + + String topicNameV2 = "persistent://public/default/t2"; + assertEquals(cmdRead.getWebSocketReadUri(topicNameV2), + "ws://localhost:8080/ws/v2/reader/persistent/public/default/t2?messageId=" + msgIdQueryParam); + } + + @Test + public void testParseMessageId() { + assertEquals(CmdRead.parseMessageId("latest"), MessageId.latest); + assertEquals(CmdRead.parseMessageId("earliest"), MessageId.earliest); + assertEquals(CmdRead.parseMessageId("20:-1"), new MessageIdImpl(20, -1, -1)); + assertEquals(CmdRead.parseMessageId("30:0"), new MessageIdImpl(30, 0, -1)); + try { + CmdRead.parseMessageId("invalid"); + fail("Should fail to parse invalid message ID"); + } catch (Throwable t) { + assertTrue(t instanceof IllegalArgumentException); + } + } + +} From f5c532dca17dbdbdec1011495d5174094ccfc7c4 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Mon, 13 Feb 2023 10:06:24 +0800 Subject: [PATCH 007/174] [feat][txn] implement the SnapshotSegmentAbortedTxnProcessor (#18273) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Master Issue: https://github.com/apache/pulsar/issues/16913 ### Motivation Implement an abortedTxnProcessor to handle the storage of the aborted transaction ID. ### Modifications The structure overview: ![image](https://user-images.githubusercontent.com/55571188/197683651-6ccb106d-1e71-4841-9da7-2644275a401a.png) The main idea is to move the logic of the operation of checking and persistent aborted transaction IDs(take snapshots) and the operation of updating maxReadPosition into the AbortedTxnProcessor. And the AbortedTxnProcessor can be implemented in different designs. **Add `persistentWorker` to handle snapshot persistenting** ： image The first four items below are the corresponding four tasks in the figure. The fifth item is not strictly a task, but a part of the first two tasks. * takeSnapshotSegmentAsync -> writeSnapshotSegmentAsync * These two method is used to persist the snapshot segment. * deleteSnapshotSegment * This method is used to delete the snapshot segment. * clearSnapshotSegmentAndIndexes * Delete all segments and then delete the index of this topic. * updateSnapshotIndex * Called by the deleteSnapshotSegment and writeSnapshotSegmentAsync. Do update the index after writing the snapshot segment. * Called to update index snapshot by `takeSnapshotByChangeTimes` and `takeSnapshotByTimeout`. * Called by recovery as a compensation mechanism for updating the index. --- .../SystemTopicTxnBufferSnapshotService.java | 2 +- .../NamespaceEventsSystemTopicFactory.java | 8 +- ...onBufferSnapshotBaseSystemTopicClient.java | 1 + .../buffer/AbortedTxnProcessor.java | 10 +- ...SingleSnapshotAbortedTxnProcessorImpl.java | 18 +- ...napshotSegmentAbortedTxnProcessorImpl.java | 784 ++++++++++++++++++ .../buffer/impl/TopicTransactionBuffer.java | 14 +- .../v2/TransactionBufferSnapshotIndex.java | 21 +- ...nsactionBufferSnapshotIndexesMetadata.java | 4 +- .../v2/TransactionBufferSnapshotSegment.java | 4 +- .../SegmentAbortedTxnProcessorTest.java | 280 +++++++ .../TopicTransactionBufferRecoverTest.java | 124 ++- 12 files changed, 1224 insertions(+), 46 deletions(-) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicTxnBufferSnapshotService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicTxnBufferSnapshotService.java index 7be599c8c2781..a1b78d89a13eb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicTxnBufferSnapshotService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicTxnBufferSnapshotService.java @@ -72,7 +72,7 @@ protected CompletableFuture> getTransactionBufferSystemTopi } return CompletableFuture.completedFuture(clients.computeIfAbsent(systemTopicName, (v) -> namespaceEventsSystemTopicFactory - .createTransactionBufferSystemTopicClient(topicName.getNamespaceObject(), + .createTransactionBufferSystemTopicClient(systemTopicName, this, schemaType))); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/NamespaceEventsSystemTopicFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/NamespaceEventsSystemTopicFactory.java index 8d30d1d140f5a..f5e6c7748d10b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/NamespaceEventsSystemTopicFactory.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/NamespaceEventsSystemTopicFactory.java @@ -44,12 +44,10 @@ public TopicPoliciesSystemTopicClient createTopicPoliciesSystemTopicClient(Names } public TransactionBufferSnapshotBaseSystemTopicClient createTransactionBufferSystemTopicClient( - NamespaceName namespaceName, SystemTopicTxnBufferSnapshotService + TopicName systemTopicName, SystemTopicTxnBufferSnapshotService systemTopicTxnBufferSnapshotService, Class schemaType) { - TopicName topicName = TopicName.get(TopicDomain.persistent.value(), namespaceName, - SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT); - log.info("Create transaction buffer snapshot client, topicName : {}", topicName.toString()); - return new TransactionBufferSnapshotBaseSystemTopicClient(client, topicName, + log.info("Create transaction buffer snapshot client, topicName : {}", systemTopicName.toString()); + return new TransactionBufferSnapshotBaseSystemTopicClient(client, systemTopicName, systemTopicTxnBufferSnapshotService, schemaType); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/TransactionBufferSnapshotBaseSystemTopicClient.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/TransactionBufferSnapshotBaseSystemTopicClient.java index b18bf552c3004..8efa983a64d73 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/TransactionBufferSnapshotBaseSystemTopicClient.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/TransactionBufferSnapshotBaseSystemTopicClient.java @@ -188,6 +188,7 @@ public SystemTopicClient getSystemTopic() { protected CompletableFuture> newWriterAsyncInternal() { return client.newProducer(Schema.AVRO(schemaType)) .topic(topicName.toString()) + .enableBatching(false) .createAsync().thenApply(producer -> { if (log.isDebugEnabled()) { log.debug("[{}] A new {} writer is created", topicName, schemaType.getName()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java index e436e1df24972..8223aa12b75ae 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java @@ -19,7 +19,6 @@ package org.apache.pulsar.broker.transaction.buffer; import java.util.concurrent.CompletableFuture; -import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.client.api.transaction.TxnID; @@ -30,9 +29,9 @@ public interface AbortedTxnProcessor { * After the transaction buffer writes a transaction aborted marker to the topic, * the transaction buffer will put the aborted txnID and the aborted marker position to AbortedTxnProcessor. * @param txnID aborted transaction ID. - * @param position the position of the abort txnID + * @param abortedMarkerPersistentPosition the position of the abort txn marker. */ - void putAbortedTxnAndPosition(TxnID txnID, PositionImpl position); + void putAbortedTxnAndPosition(TxnID txnID, PositionImpl abortedMarkerPersistentPosition); /** * Clean up invalid aborted transactions. @@ -42,10 +41,9 @@ public interface AbortedTxnProcessor { /** * Check whether the transaction ID is an aborted transaction ID. * @param txnID the transaction ID that needs to be checked. - * @param readPosition the read position of the transaction message, can be used to find the segment. * @return a boolean, whether the transaction ID is an aborted transaction ID. */ - boolean checkAbortedTransaction(TxnID txnID, Position readPosition); + boolean checkAbortedTransaction(TxnID txnID); /** * Recover transaction buffer by transaction buffer snapshot. @@ -58,7 +56,7 @@ public interface AbortedTxnProcessor { * Delete the transaction buffer aborted transaction snapshot. * @return a completableFuture. */ - CompletableFuture deleteAbortedTxnSnapshot(); + CompletableFuture clearAbortedTxnSnapshot(); /** * Take aborted transactions snapshot. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java index f8d0d32391233..87161e97512b9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java @@ -24,7 +24,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import lombok.extern.slf4j.Slf4j; -import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.collections4.map.LinkedMap; @@ -57,11 +56,16 @@ public SingleSnapshotAbortedTxnProcessorImpl(PersistentTopic topic) { this.takeSnapshotWriter = this.topic.getBrokerService().getPulsar() .getTransactionBufferSnapshotServiceFactory() .getTxnBufferSnapshotService().createWriter(TopicName.get(topic.getName())); + this.takeSnapshotWriter.exceptionally((ex) -> { + log.error("{} Failed to create snapshot writer", topic.getName()); + topic.close(); + return null; + }); } @Override - public void putAbortedTxnAndPosition(TxnID abortedTxnId, PositionImpl position) { - aborts.put(abortedTxnId, position); + public void putAbortedTxnAndPosition(TxnID abortedTxnId, PositionImpl abortedMarkerPersistentPosition) { + aborts.put(abortedTxnId, abortedMarkerPersistentPosition); } //In this implementation we clear the invalid aborted txn ID one by one. @@ -78,7 +82,7 @@ public void trimExpiredAbortedTxns() { } @Override - public boolean checkAbortedTransaction(TxnID txnID, Position readPosition) { + public boolean checkAbortedTransaction(TxnID txnID) { return aborts.containsKey(txnID); } @@ -127,14 +131,12 @@ public CompletableFuture recoverFromSnapshot() { } @Override - public CompletableFuture deleteAbortedTxnSnapshot() { + public CompletableFuture clearAbortedTxnSnapshot() { return this.takeSnapshotWriter.thenCompose(writer -> { TransactionBufferSnapshot snapshot = new TransactionBufferSnapshot(); snapshot.setTopicName(topic.getName()); return writer.deleteAsync(snapshot.getTopicName(), snapshot); - }).thenRun(() -> { - log.info("[{}] Successes to delete the aborted transaction snapshot", this.topic); - }); + }).thenRun(() -> log.info("[{}] Successes to delete the aborted transaction snapshot", this.topic)); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java new file mode 100644 index 0000000000000..7a9e0e1abedd9 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -0,0 +1,784 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.buffer.impl; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; +import java.util.function.Supplier; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.AsyncCallbacks; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.impl.ReadOnlyManagedLedgerImpl; +import org.apache.commons.collections4.map.LinkedMap; +import org.apache.commons.lang3.tuple.MutablePair; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.broker.service.BrokerServiceException; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.systopic.SystemTopicClient; +import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; +import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndex; +import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndexes; +import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndexesMetadata; +import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotSegment; +import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TxnIDData; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.client.impl.MessageIdImpl; +import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.apache.pulsar.common.naming.SystemTopicNames; +import org.apache.pulsar.common.naming.TopicDomain; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.protocol.Commands; +import org.apache.pulsar.common.util.FutureUtil; + +@Slf4j +public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcessor { + + /** + * Stored the unsealed aborted transaction IDs Whose size is always less than the snapshotSegmentCapacity. + * It will be persistent as a snapshot segment when its size reach the configured capacity. + */ + private LinkedList unsealedTxnIds; + + /** + * The map is used to clear the aborted transaction IDs persistent in the expired ledger. + *

+ * The key PositionImpl {@link PositionImpl} is the persistent position of + * the latest transaction of a segment. + * The value TxnID {@link TxnID} is the latest Transaction ID in a segment. + *

+ * + *

+ * If the position is expired, the processor can get the according latest + * transaction ID in this map. And then the processor can clear all the + * transaction IDs in the aborts {@link SnapshotSegmentAbortedTxnProcessorImpl#aborts} + * that lower than the transaction ID. + * And then the processor can delete the segments persistently according to + * the positions. + *

+ */ + private final LinkedMap segmentIndex = new LinkedMap<>(); + + /** + * This map is used to check whether a transaction is an aborted transaction. + *

+ * The transaction IDs is appended in order, so the processor can delete expired + * transaction IDs according to the latest expired transaction IDs in segmentIndex + * {@link SnapshotSegmentAbortedTxnProcessorImpl#segmentIndex}. + *

+ */ + private final LinkedMap aborts = new LinkedMap<>(); + /** + * This map stores the indexes of the snapshot segment. + *

+ * The key is the persistent position of the marker of the last transaction in the segment. + * The value TransactionBufferSnapshotIndex {@link TransactionBufferSnapshotIndex} is the + * indexes of the snapshot segment. + *

+ */ + private final LinkedMap indexes = new LinkedMap<>(); + + private final PersistentTopic topic; + + private volatile long lastSnapshotTimestamps; + + /** + * The number of the aborted transaction IDs in a segment. + * This is calculated according to the configured memory size. + */ + private final int snapshotSegmentCapacity; + /** + * Responsible for executing the persistent tasks. + *

Including:

+ *

Update segment index.

+ *

Write snapshot segment.

+ *

Delete snapshot segment.

+ *

Clear all snapshot segment.

+ */ + private final PersistentWorker persistentWorker; + + private static final String SNAPSHOT_PREFIX = "multiple-"; + + public SnapshotSegmentAbortedTxnProcessorImpl(PersistentTopic topic) { + this.topic = topic; + this.persistentWorker = new PersistentWorker(topic); + /* + Calculate the segment capital according to its size configuration. +

+ The empty transaction segment size is 5. + Adding an empty linkedList, the size increase to 6. + Add the topic name the size increase to the 7 + topic.getName().length(). + Add the aborted transaction IDs, the size increase to 8 + + topic.getName().length() + 3 * aborted transaction ID size. +

+ */ + this.snapshotSegmentCapacity = (topic.getBrokerService().getPulsar() + .getConfiguration().getTransactionBufferSnapshotSegmentSize() - 8 - topic.getName().length()) / 3; + this.unsealedTxnIds = new LinkedList<>(); + } + + @Override + public void putAbortedTxnAndPosition(TxnID txnID, PositionImpl position) { + unsealedTxnIds.add(txnID); + aborts.put(txnID, txnID); + /* + The size of lastAbortedTxns reaches the configuration of the size of snapshot segment. + Append a task to persistent the segment with the aborted transaction IDs and the latest + transaction mark persistent position passed by param. + */ + if (unsealedTxnIds.size() >= snapshotSegmentCapacity) { + LinkedList abortedSegment = unsealedTxnIds; + segmentIndex.put(position, txnID); + persistentWorker.appendTask(PersistentWorker.OperationType.WriteSegment, + () -> persistentWorker.takeSnapshotSegmentAsync(abortedSegment, position)); + this.unsealedTxnIds = new LinkedList<>(); + } + } + + @Override + public boolean checkAbortedTransaction(TxnID txnID) { + return aborts.containsKey(txnID); + } + + /** + * Check werther the position in segmentIndex {@link SnapshotSegmentAbortedTxnProcessorImpl#segmentIndex} + * is expired. If the position is not exist in the original topic, the according transaction is an invalid + * transaction. And the according segment is invalid, too. The transaction IDs before the transaction ID + * in the aborts are invalid, too. + */ + @Override + public void trimExpiredAbortedTxns() { + //Checking whether there are some segment expired. + List positionsNeedToDelete = new ArrayList<>(); + while (!segmentIndex.isEmpty() && !((ManagedLedgerImpl) topic.getManagedLedger()) + .ledgerExists(segmentIndex.firstKey().getLedgerId())) { + if (log.isDebugEnabled()) { + log.debug("[{}] Topic transaction buffer clear aborted transactions, maxReadPosition : {}", + topic.getName(), segmentIndex.firstKey()); + } + PositionImpl positionNeedToDelete = segmentIndex.firstKey(); + positionsNeedToDelete.add(positionNeedToDelete); + + TxnID theLatestDeletedTxnID = segmentIndex.remove(0); + while (!aborts.firstKey().equals(theLatestDeletedTxnID)) { + aborts.remove(0); + } + aborts.remove(0); + } + //Batch delete the expired segment + if (!positionsNeedToDelete.isEmpty()) { + persistentWorker.appendTask(PersistentWorker.OperationType.DeleteSegment, + () -> persistentWorker.deleteSnapshotSegment(positionsNeedToDelete)); + } + } + + private String buildKey(long sequenceId) { + return SNAPSHOT_PREFIX + sequenceId + "-" + this.topic.getName(); + } + + @Override + public CompletableFuture takeAbortedTxnsSnapshot(PositionImpl maxReadPosition) { + //Store the latest aborted transaction IDs in unsealedTxnIDs and the according the latest max read position. + TransactionBufferSnapshotIndexesMetadata metadata = new TransactionBufferSnapshotIndexesMetadata( + maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(), + convertTypeToTxnIDData(unsealedTxnIds)); + return persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, + () -> persistentWorker.updateSnapshotIndex(metadata)); + } + + @Override + public CompletableFuture recoverFromSnapshot() { + return topic.getBrokerService().getPulsar().getTransactionBufferSnapshotServiceFactory() + .getTxnBufferSnapshotIndexService() + .createReader(TopicName.get(topic.getName())).thenComposeAsync(reader -> { + PositionImpl startReadCursorPosition = null; + TransactionBufferSnapshotIndexes persistentSnapshotIndexes = null; + try { + /* + Read the transaction snapshot segment index. +

+ The processor can get the sequence ID, unsealed transaction IDs, + segment index list and max read position in the snapshot segment index. + Then we can traverse the index list to read all aborted transaction IDs + in segments to aborts. +

+ */ + while (reader.hasMoreEvents()) { + Message message = reader.readNextAsync() + .get(getSystemClientOperationTimeoutMs(), TimeUnit.MILLISECONDS); + if (topic.getName().equals(message.getKey())) { + TransactionBufferSnapshotIndexes transactionBufferSnapshotIndexes = message.getValue(); + if (transactionBufferSnapshotIndexes != null) { + persistentSnapshotIndexes = transactionBufferSnapshotIndexes; + startReadCursorPosition = PositionImpl.get( + transactionBufferSnapshotIndexes.getSnapshot().getMaxReadPositionLedgerId(), + transactionBufferSnapshotIndexes.getSnapshot().getMaxReadPositionEntryId()); + } + } + } + } catch (TimeoutException ex) { + Throwable t = FutureUtil.unwrapCompletionException(ex); + String errorMessage = String.format("[%s] Transaction buffer recover fail by read " + + "transactionBufferSnapshot timeout!", topic.getName()); + log.error(errorMessage, t); + return FutureUtil.failedFuture( + new BrokerServiceException.ServiceUnitNotReadyException(errorMessage, t)); + } catch (Exception ex) { + log.error("[{}] Transaction buffer recover fail when read " + + "transactionBufferSnapshot!", topic.getName(), ex); + return FutureUtil.failedFuture(ex); + } finally { + closeReader(reader); + } + PositionImpl finalStartReadCursorPosition = startReadCursorPosition; + TransactionBufferSnapshotIndexes finalPersistentSnapshotIndexes = persistentSnapshotIndexes; + if (persistentSnapshotIndexes == null) { + return CompletableFuture.completedFuture(null); + } else { + this.unsealedTxnIds = convertTypeToTxnID(persistentSnapshotIndexes + .getSnapshot().getAborts()); + } + //Read snapshot segment to recover aborts. + ArrayList> completableFutures = new ArrayList<>(); + CompletableFuture openManagedLedgerAndHandleSegmentsFuture = new CompletableFuture<>(); + AtomicBoolean hasInvalidIndex = new AtomicBoolean(false); + AsyncCallbacks.OpenReadOnlyManagedLedgerCallback callback = new AsyncCallbacks + .OpenReadOnlyManagedLedgerCallback() { + @Override + public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedgerImpl readOnlyManagedLedger, + Object ctx) { + finalPersistentSnapshotIndexes.getIndexList().forEach(index -> { + CompletableFuture handleSegmentFuture = new CompletableFuture<>(); + completableFutures.add(handleSegmentFuture); + readOnlyManagedLedger.asyncReadEntry( + new PositionImpl(index.getSegmentLedgerID(), + index.getSegmentEntryID()), + new AsyncCallbacks.ReadEntryCallback() { + @Override + public void readEntryComplete(Entry entry, Object ctx) { + handleSnapshotSegmentEntry(entry); + indexes.put(new PositionImpl( + index.abortedMarkLedgerID, + index.abortedMarkEntryID), + index); + entry.release(); + handleSegmentFuture.complete(null); + } + + @Override + public void readEntryFailed(ManagedLedgerException exception, Object ctx) { + /* + The logic flow of deleting expired segment is: +

+ 1. delete segment + 2. update segment index +

+ If the worker delete segment successfully + but failed to update segment index, + the segment can not be read according to the index. + We update index again if there are invalid indexes. + */ + if (((ManagedLedgerImpl) topic.getManagedLedger()) + .ledgerExists(index.getAbortedMarkLedgerID())) { + log.error("[{}] Failed to read snapshot segment [{}:{}]", + topic.getName(), index.segmentLedgerID, + index.segmentEntryID, exception); + handleSegmentFuture.completeExceptionally(exception); + } else { + hasInvalidIndex.set(true); + } + } + }, null); + }); + openManagedLedgerAndHandleSegmentsFuture.complete(null); + } + + @Override + public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Object ctx) { + log.error("[{}] Failed to open readOnly managed ledger", topic, exception); + openManagedLedgerAndHandleSegmentsFuture.completeExceptionally(exception); + } + }; + + TopicName snapshotSegmentTopicName = TopicName.get(TopicDomain.persistent.toString(), + TopicName.get(topic.getName()).getNamespaceObject(), + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_SEGMENTS); + this.topic.getBrokerService().getPulsar().getManagedLedgerFactory() + .asyncOpenReadOnlyManagedLedger(snapshotSegmentTopicName + .getPersistenceNamingEncoding(), callback, + topic.getManagedLedger().getConfig(), + null); + /* + Wait the processor recover completely and then allow TB + to recover the messages after the startReadCursorPosition. + */ + return openManagedLedgerAndHandleSegmentsFuture + .thenCompose((ignore) -> FutureUtil.waitForAll(completableFutures)) + .thenCompose((i) -> { + /* + Update the snapshot segment index if there exist invalid indexes. + */ + if (hasInvalidIndex.get()) { + persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, + () -> persistentWorker.updateSnapshotIndex( + finalPersistentSnapshotIndexes.getSnapshot())); + } + /* + If there is no segment index, the persistent worker will write segment begin from 0. + */ + if (indexes.size() != 0) { + persistentWorker.sequenceID.set(indexes.get(indexes.lastKey()).sequenceID + 1); + } + /* + Append the aborted txn IDs in the index metadata + can keep the order of the aborted txn in the aborts. + So that we can trim the expired snapshot segment in aborts + according to the latest transaction IDs in the segmentIndex. + */ + unsealedTxnIds.forEach(txnID -> aborts.put(txnID, txnID)); + return CompletableFuture.completedFuture(finalStartReadCursorPosition); + }).exceptionally(ex -> { + log.error("[{}] Failed to recover snapshot segment", this.topic.getName(), ex); + return null; + }); + + }, topic.getBrokerService().getPulsar().getTransactionExecutorProvider() + .getExecutor(this)); + } + + @Override + public CompletableFuture clearAbortedTxnSnapshot() { + return persistentWorker.appendTask(PersistentWorker.OperationType.Clear, + persistentWorker::clearSnapshotSegmentAndIndexes); + } + + @Override + public long getLastSnapshotTimestamps() { + return this.lastSnapshotTimestamps; + } + + @Override + public CompletableFuture closeAsync() { + return persistentWorker.closeAsync(); + } + + private void handleSnapshotSegmentEntry(Entry entry) { + //decode snapshot from entry + ByteBuf headersAndPayload = entry.getDataBuffer(); + //skip metadata + Commands.parseMessageMetadata(headersAndPayload); + TransactionBufferSnapshotSegment snapshotSegment = Schema.AVRO(TransactionBufferSnapshotSegment.class) + .decode(Unpooled.wrappedBuffer(headersAndPayload).nioBuffer()); + + TxnIDData lastTxn = snapshotSegment.getAborts().get(snapshotSegment.getAborts().size() - 1); + segmentIndex.put(new PositionImpl(snapshotSegment.getPersistentPositionLedgerId(), + snapshotSegment.getPersistentPositionEntryId()), + new TxnID(lastTxn.getMostSigBits(), lastTxn.getLeastSigBits())); + convertTypeToTxnID(snapshotSegment.getAborts()).forEach(txnID -> aborts.put(txnID, txnID)); + } + + private long getSystemClientOperationTimeoutMs() throws Exception { + PulsarClientImpl pulsarClient = (PulsarClientImpl) topic.getBrokerService().getPulsar().getClient(); + return pulsarClient.getConfiguration().getOperationTimeoutMs(); + } + + private void closeReader(SystemTopicClient.Reader reader) { + reader.closeAsync().exceptionally(e -> { + log.error("[{}]Transaction buffer snapshot reader close error!", topic.getName(), e); + return null; + }); + } + + /** + * The PersistentWorker be responsible for executing the persistent tasks, including: + *

+ * 1. Write snapshot segment --- Encapsulate a sealed snapshot segment and persistent it. + * 2. Delete snapshot segment --- Evict expired snapshot segments. + * 3. Update snapshot indexes --- Update snapshot indexes after writing or deleting snapshot segment + * or update snapshot indexes metadata regularly. + * 4. Clear all snapshot segments and indexes. --- Executed when deleting this topic. + *

+ * * Task 1 and task 2 will be put into a task queue. The tasks in the queue will be executed in order. + * * If the task queue is empty, task 3 will be executed immediately when it is appended to the worker. + * Else, the worker will try to execute the tasks in the task queue. + * * When task 4 was appended into worker, the worker will change the operation state to closed + * and cancel all tasks in the task queue. finally, execute the task 4 (clear task). + * If there are race conditions, throw an Exception to let users try again. + */ + public class PersistentWorker { + protected final AtomicLong sequenceID = new AtomicLong(0); + + private final PersistentTopic topic; + + //Persistent snapshot segment and index at the single thread. + private final CompletableFuture> + snapshotSegmentsWriterFuture; + private final CompletableFuture> + snapshotIndexWriterFuture; + + private enum OperationState { + None, + Operating, + Closed + } + private static final AtomicReferenceFieldUpdater + STATE_UPDATER = AtomicReferenceFieldUpdater.newUpdater(PersistentWorker.class, + PersistentWorker.OperationState.class, "operationState"); + + public enum OperationType { + UpdateIndex, + WriteSegment, + DeleteSegment, + Clear + } + + private volatile OperationState operationState = OperationState.None; + + ConcurrentLinkedDeque, + Supplier>>>> taskQueue = new ConcurrentLinkedDeque<>(); + + public PersistentWorker(PersistentTopic topic) { + this.topic = topic; + this.snapshotSegmentsWriterFuture = this.topic.getBrokerService().getPulsar() + .getTransactionBufferSnapshotServiceFactory() + .getTxnBufferSnapshotSegmentService().createWriter(TopicName.get(topic.getName())); + this.snapshotSegmentsWriterFuture.exceptionally(ex -> { + log.error("{} Failed to create snapshot index writer", topic.getName()); + topic.close(); + return null; + }); + this.snapshotIndexWriterFuture = this.topic.getBrokerService().getPulsar() + .getTransactionBufferSnapshotServiceFactory() + .getTxnBufferSnapshotIndexService().createWriter(TopicName.get(topic.getName())); + this.snapshotIndexWriterFuture.exceptionally((ex) -> { + log.error("{} Failed to create snapshot writer", topic.getName()); + topic.close(); + return null; + }); + } + + public CompletableFuture appendTask(OperationType operationType, + Supplier> task) { + CompletableFuture taskExecutedResult = new CompletableFuture<>(); + switch (operationType) { + case UpdateIndex -> { + /* + The update index operation can be canceled when the task queue is not empty, + so it should be executed immediately instead of appending to the task queue. + If the taskQueue is not empty, the worker will execute the tasks in the queue. + */ + if (!taskQueue.isEmpty()) { + executeTask(); + return cancelUpdateIndexTask(); + } else if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.Operating)) { + return task.get().whenComplete((ignore, throwable) -> { + if (throwable != null && log.isDebugEnabled()) { + log.debug("[{}] Failed to update index snapshot", topic.getName(), throwable); + } + STATE_UPDATER.compareAndSet(this, OperationState.Operating, OperationState.None); + }); + } else { + return cancelUpdateIndexTask(); + } + } + /* + Only the operations of WriteSegment and DeleteSegment will be appended into the taskQueue. + The operation will be canceled when the worker is close which means the topic is deleted. + */ + case WriteSegment, DeleteSegment -> { + if (!STATE_UPDATER.get(this).equals(OperationState.Closed)) { + taskQueue.add(new MutablePair<>(operationType, new MutablePair<>(taskExecutedResult, task))); + executeTask(); + return taskExecutedResult; + } else { + return CompletableFuture.completedFuture(null); + } + } + case Clear -> { + /* + Do not clear the snapshots if the topic is used. + If the users want to delete a topic, they should stop the usage of the topic. + */ + if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.Closed)) { + taskQueue.forEach(pair -> + pair.getRight().getRight().get().completeExceptionally( + new BrokerServiceException.ServiceUnitNotReadyException( + String.format("Cancel the operation [%s] due to the" + + " transaction buffer of the topic[%s] already closed", + pair.getLeft().name(), this.topic.getName())))); + taskQueue.clear(); + /* + The task of clear all snapshot segments and indexes is executed immediately. + */ + return task.get(); + } else { + return FutureUtil.failedFuture( + new BrokerServiceException.NotAllowedException( + String.format("Failed to clear the snapshot of topic [%s] due to " + + "the topic is used. Please stop the using of the topic " + + "and try it again", this.topic.getName()))); + } + } + default -> { + return FutureUtil.failedFuture(new BrokerServiceException + .NotAllowedException(String.format("Th operation [%s] is unsupported", + operationType.name()))); + } + } + } + + private CompletableFuture cancelUpdateIndexTask() { + if (log.isDebugEnabled()) { + log.debug("The operation of updating index is canceled due there is other operation executing"); + } + return FutureUtil.failedFuture(new BrokerServiceException + .ServiceUnitNotReadyException("The operation of updating index is canceled")); + } + + private void executeTask() { + if (taskQueue.isEmpty()) { + return; + } + if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.Operating)) { + //Double-check. Avoid NoSuchElementException due to the first task is completed by other thread. + if (taskQueue.isEmpty()) { + return; + } + Pair, Supplier>>> firstTask = + taskQueue.getFirst(); + firstTask.getValue().getRight().get().whenComplete((ignore, throwable) -> { + if (throwable != null) { + if (log.isDebugEnabled()) { + log.debug("[{}] Failed to do operation do operation of [{}]", + topic.getName(), firstTask.getKey().name(), throwable); + } + //Do not execute the tasks in the task queue until the next task is appended to the task queue. + firstTask.getRight().getKey().completeExceptionally(throwable); + } else { + firstTask.getRight().getKey().complete(null); + taskQueue.removeFirst(); + //Execute the next task in the other thread. + topic.getBrokerService().getPulsar().getTransactionExecutorProvider() + .getExecutor(this).submit(this::executeTask); + } + STATE_UPDATER.compareAndSet(this, OperationState.Operating, + OperationState.None); + }); + } + } + + private CompletableFuture takeSnapshotSegmentAsync(LinkedList sealedAbortedTxnIdSegment, + PositionImpl abortedMarkerPersistentPosition) { + CompletableFuture res = writeSnapshotSegmentAsync(sealedAbortedTxnIdSegment, + abortedMarkerPersistentPosition).thenRun(() -> { + if (log.isDebugEnabled()) { + log.debug("Successes to take snapshot segment [{}] at maxReadPosition [{}] " + + "for the topic [{}], and the size of the segment is [{}]", + this.sequenceID, abortedMarkerPersistentPosition, topic.getName(), + sealedAbortedTxnIdSegment.size()); + } + this.sequenceID.getAndIncrement(); + }); + res.exceptionally(e -> { + //Just log the error, and the processor will try to take snapshot again when the transactionBuffer + //append aborted txn next time. + log.error("Failed to take snapshot segment [{}] at maxReadPosition [{}] " + + "for the topic [{}], and the size of the segment is [{}]", + this.sequenceID, abortedMarkerPersistentPosition, topic.getName(), + sealedAbortedTxnIdSegment.size(), e); + return null; + }); + return res; + } + + private CompletableFuture writeSnapshotSegmentAsync(LinkedList segment, + PositionImpl abortedMarkerPersistentPosition) { + TransactionBufferSnapshotSegment transactionBufferSnapshotSegment = new TransactionBufferSnapshotSegment(); + transactionBufferSnapshotSegment.setAborts(convertTypeToTxnIDData(segment)); + transactionBufferSnapshotSegment.setTopicName(this.topic.getName()); + transactionBufferSnapshotSegment.setPersistentPositionEntryId(abortedMarkerPersistentPosition.getEntryId()); + transactionBufferSnapshotSegment.setPersistentPositionLedgerId( + abortedMarkerPersistentPosition.getLedgerId()); + + return snapshotSegmentsWriterFuture.thenCompose(segmentWriter -> { + transactionBufferSnapshotSegment.setSequenceId(this.sequenceID.get()); + return segmentWriter.writeAsync(buildKey(this.sequenceID.get()), transactionBufferSnapshotSegment); + }).thenCompose((messageId) -> { + //Build index for this segment + TransactionBufferSnapshotIndex index = new TransactionBufferSnapshotIndex(); + index.setSequenceID(transactionBufferSnapshotSegment.getSequenceId()); + index.setAbortedMarkLedgerID(abortedMarkerPersistentPosition.getLedgerId()); + index.setAbortedMarkEntryID(abortedMarkerPersistentPosition.getEntryId()); + index.setSegmentLedgerID(((MessageIdImpl) messageId).getLedgerId()); + index.setSegmentEntryID(((MessageIdImpl) messageId).getEntryId()); + + indexes.put(abortedMarkerPersistentPosition, index); + //update snapshot segment index. + //If the index can not be written successfully, the snapshot segment wil be overwritten + //when the processor writes snapshot segment next time. + //And if the task is not the newest in the queue, it is no need to update the index. + return updateIndexWhenExecuteTheLatestTask(); + }); + } + + private CompletionStage updateIndexWhenExecuteTheLatestTask() { + PositionImpl maxReadPosition = topic.getMaxReadPosition(); + List aborts = convertTypeToTxnIDData(unsealedTxnIds); + if (taskQueue.size() != 1) { + return CompletableFuture.completedFuture(null); + } else { + return updateSnapshotIndex(new TransactionBufferSnapshotIndexesMetadata( + maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(), aborts)); + } + } + + // update index after delete all segment. + private CompletableFuture deleteSnapshotSegment(List positionNeedToDeletes) { + List> results = new ArrayList<>(); + for (PositionImpl positionNeedToDelete : positionNeedToDeletes) { + long sequenceIdNeedToDelete = indexes.get(positionNeedToDelete).getSequenceID(); + CompletableFuture res = snapshotSegmentsWriterFuture + .thenCompose(writer -> writer.deleteAsync(buildKey(sequenceIdNeedToDelete), null)) + .thenCompose(messageId -> { + if (log.isDebugEnabled()) { + log.debug("[{}] Successes to delete the snapshot segment, " + + "whose sequenceId is [{}] and maxReadPosition is [{}]", + this.topic.getName(), this.sequenceID, positionNeedToDelete); + } + //The index may fail to update but the processor will check + //whether the snapshot segment is null, and update the index when recovering. + //And if the task is not the newest in the queue, it is no need to update the index. + indexes.remove(positionNeedToDelete); + return updateIndexWhenExecuteTheLatestTask(); + }); + res.exceptionally(e -> { + log.warn("[{}] Failed to delete the snapshot segment, " + + "whose sequenceId is [{}] and maxReadPosition is [{}]", + this.topic.getName(), this.sequenceID, positionNeedToDelete, e); + return null; + }); + results.add(res); + } + return FutureUtil.waitForAll(results); + } + + private CompletableFuture updateSnapshotIndex(TransactionBufferSnapshotIndexesMetadata snapshotSegment) { + TransactionBufferSnapshotIndexes snapshotIndexes = new TransactionBufferSnapshotIndexes(); + CompletableFuture res = snapshotIndexWriterFuture + .thenCompose((indexesWriter) -> { + snapshotIndexes.setIndexList(indexes.values().stream().toList()); + snapshotIndexes.setSnapshot(snapshotSegment); + snapshotIndexes.setTopicName(topic.getName()); + return indexesWriter.writeAsync(topic.getName(), snapshotIndexes) + .thenCompose(messageId -> CompletableFuture.completedFuture(null)); + }); + res.thenRun(() -> lastSnapshotTimestamps = System.currentTimeMillis()).exceptionally(e -> { + log.error("[{}] Failed to update snapshot segment index", snapshotIndexes.getTopicName(), e); + return null; + }); + return res; + } + + private CompletableFuture clearSnapshotSegmentAndIndexes() { + CompletableFuture res = persistentWorker.clearAllSnapshotSegments() + .thenCompose((ignore) -> snapshotIndexWriterFuture + .thenCompose(indexesWriter -> indexesWriter.writeAsync(topic.getName(), null))) + .thenRun(() -> + log.debug("Successes to clear the snapshot segment and indexes for the topic [{}]", + topic.getName())); + res.exceptionally(e -> { + log.error("Failed to clear the snapshot segment and indexes for the topic [{}]", + topic.getName(), e); + return null; + }); + return res; + } + + /** + * Because the operation of writing segment and index is not atomic, + * we cannot use segment index to clear the snapshot segments. + * If we use segment index to clear snapshot segments, there will case dirty data in the below case: + *

+ * 1. Write snapshot segment 1, 2, 3, update index (1, 2, 3) + * 2. Write snapshot 4, failing to update index + * 3. Trim expired snapshot segment 1, 2, 3, update index (empty) + * 4. Write snapshot segment 1, 2, update index (1, 2) + * 5. Delete topic, clear all snapshot segment (segment1. segment2). + * Segment 3 and segment 4 can not be cleared until this namespace being deleted. + *

+ */ + private CompletableFuture clearAllSnapshotSegments() { + return topic.getBrokerService().getPulsar().getTransactionBufferSnapshotServiceFactory() + .getTxnBufferSnapshotSegmentService() + .createReader(TopicName.get(topic.getName())).thenComposeAsync(reader -> { + try { + while (reader.hasMoreEvents()) { + Message message = reader.readNextAsync() + .get(getSystemClientOperationTimeoutMs(), TimeUnit.MILLISECONDS); + if (topic.getName().equals(message.getValue().getTopicName())) { + snapshotSegmentsWriterFuture.get().write(message.getKey(), null); + } + } + return CompletableFuture.completedFuture(null); + } catch (Exception ex) { + log.error("[{}] Transaction buffer clear snapshot segments fail!", topic.getName(), ex); + return FutureUtil.failedFuture(ex); + } finally { + closeReader(reader); + } + }); + } + + + CompletableFuture closeAsync() { + return CompletableFuture.allOf( + this.snapshotIndexWriterFuture.thenCompose(SystemTopicClient.Writer::closeAsync), + this.snapshotSegmentsWriterFuture.thenCompose(SystemTopicClient.Writer::closeAsync)); + } + } + + private LinkedList convertTypeToTxnID(List snapshotSegment) { + LinkedList abortedTxns = new LinkedList<>(); + snapshotSegment.forEach(txnIDData -> + abortedTxns.add(new TxnID(txnIDData.getMostSigBits(), txnIDData.getLeastSigBits()))); + return abortedTxns; + } + + private List convertTypeToTxnIDData(List abortedTxns) { + List segment = new LinkedList<>(); + abortedTxns.forEach(txnID -> segment.add(new TxnIDData(txnID.getMostSigBits(), txnID.getLeastSigBits()))); + return segment; + } + +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index f3bf4f95923cd..89a8e95afba1f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -110,7 +110,11 @@ public TopicTransactionBuffer(PersistentTopic topic) { this.takeSnapshotIntervalTime = topic.getBrokerService().getPulsar() .getConfiguration().getTransactionBufferSnapshotMinTimeInMillis(); this.maxReadPosition = (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); - this.snapshotAbortedTxnProcessor = new SingleSnapshotAbortedTxnProcessorImpl(topic); + if (topic.getBrokerService().getPulsar().getConfiguration().isTransactionBufferSegmentedSnapshotEnabled()) { + snapshotAbortedTxnProcessor = new SnapshotSegmentAbortedTxnProcessorImpl(topic); + } else { + snapshotAbortedTxnProcessor = new SingleSnapshotAbortedTxnProcessorImpl(topic); + } this.recover(); } @@ -275,7 +279,7 @@ public void addFailed(ManagedLedgerException exception, Object ctx) { private void handleTransactionMessage(TxnID txnId, Position position) { if (!ongoingTxns.containsKey(txnId) && !this.snapshotAbortedTxnProcessor - .checkAbortedTransaction(txnId, position)) { + .checkAbortedTransaction(txnId)) { ongoingTxns.put(txnId, (PositionImpl) position); PositionImpl firstPosition = ongoingTxns.get(ongoingTxns.firstKey()); //max read position is less than first ongoing transaction message position, so entryId -1 @@ -349,8 +353,8 @@ public CompletableFuture abortTxn(TxnID txnID, long lowWaterMark) { @Override public void addComplete(Position position, ByteBuf entryData, Object ctx) { synchronized (TopicTransactionBuffer.this) { + snapshotAbortedTxnProcessor.putAbortedTxnAndPosition(txnID, (PositionImpl) position); updateMaxReadPosition(txnID); - snapshotAbortedTxnProcessor.putAbortedTxnAndPosition(txnID, maxReadPosition); snapshotAbortedTxnProcessor.trimExpiredAbortedTxns(); takeSnapshotByChangeTimes(); } @@ -455,7 +459,7 @@ public CompletableFuture purgeTxns(List dataLedgers) { @Override public CompletableFuture clearSnapshot() { - return snapshotAbortedTxnProcessor.deleteAbortedTxnSnapshot(); + return snapshotAbortedTxnProcessor.clearAbortedTxnSnapshot(); } @Override @@ -466,7 +470,7 @@ public CompletableFuture closeAsync() { @Override public boolean isTxnAborted(TxnID txnID, PositionImpl readPosition) { - return snapshotAbortedTxnProcessor.checkAbortedTransaction(txnID, readPosition); + return snapshotAbortedTxnProcessor.checkAbortedTransaction(txnID); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndex.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndex.java index 118472397adda..b86edc845c1e5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndex.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndex.java @@ -29,8 +29,21 @@ @NoArgsConstructor public class TransactionBufferSnapshotIndex { public long sequenceID; - public long maxReadPositionLedgerID; - public long maxReadPositionEntryID; - public long persistentPositionLedgerID; - public long persistentPositionEntryID; + /** + * Location(ledger id of position) of a transaction marker in the origin topic. + */ + public long abortedMarkLedgerID; + + /** + * Location(entry id of position) of a transaction marker in the origin topic. + */ + public long abortedMarkEntryID; + /** + * Location(ledger id of position) of a segment data in the system topic __transaction_buffer_snapshot_segments. + */ + public long segmentLedgerID; + /** + * Location(entry id of position) of a segment data in the system topic __transaction_buffer_snapshot_segments. + */ + public long segmentEntryID; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndexesMetadata.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndexesMetadata.java index 9a468d250bbbf..f9c28a818f8b8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndexesMetadata.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndexesMetadata.java @@ -18,7 +18,7 @@ */ package org.apache.pulsar.broker.transaction.buffer.metadata.v2; -import java.util.Set; +import java.util.List; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -29,5 +29,5 @@ public class TransactionBufferSnapshotIndexesMetadata { private long maxReadPositionLedgerId; private long maxReadPositionEntryId; - private Set aborts; + private List aborts; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotSegment.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotSegment.java index 77bb546880dbe..7ca828cc3e64f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotSegment.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotSegment.java @@ -29,7 +29,7 @@ public class TransactionBufferSnapshotSegment { private String topicName; private long sequenceId; - private long maxReadPositionLedgerId; - private long maxReadPositionEntryId; + private long persistentPositionLedgerId; + private long persistentPositionEntryId; private List aborts; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java new file mode 100644 index 0000000000000..ffc059de8e656 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java @@ -0,0 +1,280 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction; + +import static org.testng.Assert.assertTrue; +import java.lang.reflect.Field; +import java.util.LinkedList; +import java.util.NavigableMap; +import java.util.Queue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.proto.MLDataFormats; +import org.apache.commons.collections4.map.LinkedMap; +import org.apache.commons.lang3.tuple.MutablePair; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.service.BrokerServiceException; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory; +import org.apache.pulsar.broker.systopic.SystemTopicClient; +import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; +import org.apache.pulsar.broker.transaction.buffer.impl.SnapshotSegmentAbortedTxnProcessorImpl; +import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndexes; +import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotSegment; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.common.events.EventType; +import org.apache.pulsar.common.naming.TopicName; +import org.testcontainers.shaded.org.awaitility.Awaitility; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Slf4j +public class SegmentAbortedTxnProcessorTest extends TransactionTestBase { + + private static final String PROCESSOR_TOPIC = "persistent://" + NAMESPACE1 + "/abortedTxnProcessor"; + private static final int SEGMENT_SIZE = 5; + private PulsarService pulsarService = null; + + @Override + @BeforeClass + protected void setup() throws Exception { + setUpBase(1, 1, PROCESSOR_TOPIC, 0); + this.pulsarService = getPulsarServiceList().get(0); + this.pulsarService.getConfig().setTransactionBufferSegmentedSnapshotEnabled(true); + this.pulsarService.getConfig().setTransactionBufferSnapshotSegmentSize(8 + PROCESSOR_TOPIC.length() + + SEGMENT_SIZE * 3); + } + + @Override + @AfterClass + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + /** + * Test api: + * 1. putAbortedTxnAndPosition + * 2. checkAbortedTransaction + * 3. takeAbortedTxnsSnapshot + * 4. recoverFromSnapshot + * 5. trimExpiredAbortedTxns + * @throws Exception + */ + @Test + public void testPutAbortedTxnIntoProcessor() throws Exception { + PersistentTopic persistentTopic = (PersistentTopic) pulsarService.getBrokerService() + .getTopic(PROCESSOR_TOPIC, false).get().get(); + AbortedTxnProcessor processor = new SnapshotSegmentAbortedTxnProcessorImpl(persistentTopic); + //1. prepare test data. + //1.1 Put 10 aborted txn IDs to persistent two sealed segments. + for (int i = 0; i < 10; i++) { + TxnID txnID = new TxnID(0, i); + PositionImpl position = new PositionImpl(0, i); + processor.putAbortedTxnAndPosition(txnID, position); + } + //1.2 Put 4 aborted txn IDs into the unsealed segment. + for (int i = 10; i < 14; i++) { + TxnID txnID = new TxnID(0, i); + PositionImpl position = new PositionImpl(0, i); + processor.putAbortedTxnAndPosition(txnID, position); + } + //1.3 Verify the common data flow + verifyAbortedTxnIDAndSegmentIndex(processor, 0, 14); + //2. Take the latest snapshot and verify recover from snapshot + AbortedTxnProcessor newProcessor = new SnapshotSegmentAbortedTxnProcessorImpl(persistentTopic); + PositionImpl maxReadPosition = new PositionImpl(0, 14); + //2.1 Avoid update operation being canceled. + waitTaskExecuteCompletely(processor); + //2.2 take the latest snapshot + processor.takeAbortedTxnsSnapshot(maxReadPosition).get(); + newProcessor.recoverFromSnapshot().get(); + //Verify the recovery data flow + verifyAbortedTxnIDAndSegmentIndex(newProcessor, 0, 14); + //3. Delete the ledgers and then verify the date. + Field ledgersField = ManagedLedgerImpl.class.getDeclaredField("ledgers"); + ledgersField.setAccessible(true); + NavigableMap ledgers = + (NavigableMap) + ledgersField.get(persistentTopic.getManagedLedger()); + ledgers.forEach((k, v) -> { + ledgers.remove(k); + }); + newProcessor.trimExpiredAbortedTxns(); + //4. Verify the two sealed segment will be deleted. + Awaitility.await().untilAsserted(() -> verifyAbortedTxnIDAndSegmentIndex(newProcessor, 11, 4)); + } + + private void waitTaskExecuteCompletely(AbortedTxnProcessor processor) throws Exception { + Field workerField = SnapshotSegmentAbortedTxnProcessorImpl.class.getDeclaredField("persistentWorker"); + workerField.setAccessible(true); + SnapshotSegmentAbortedTxnProcessorImpl.PersistentWorker persistentWorker = + (SnapshotSegmentAbortedTxnProcessorImpl.PersistentWorker) workerField.get(processor); + Field taskQueueField = SnapshotSegmentAbortedTxnProcessorImpl.PersistentWorker.class + .getDeclaredField("taskQueue"); + taskQueueField.setAccessible(true); + Queue queue = (Queue) taskQueueField.get(persistentWorker); + Awaitility.await().untilAsserted(() -> Assert.assertEquals(queue.size(), 0)); + } + + private void verifyAbortedTxnIDAndSegmentIndex(AbortedTxnProcessor processor, int begin, int txnIdSize) + throws Exception { + //Verify the checking of the aborted txn IDs + for (int i = begin; i < txnIdSize; i++) { + Assert.assertTrue(processor.checkAbortedTransaction(new TxnID(0, i))); + } + //Verify there are 2 sealed segment and the unsealed segment size is 4. + Field unsealedSegmentField = SnapshotSegmentAbortedTxnProcessorImpl.class + .getDeclaredField("unsealedTxnIds"); + Field indexField = SnapshotSegmentAbortedTxnProcessorImpl.class + .getDeclaredField("segmentIndex"); + unsealedSegmentField.setAccessible(true); + indexField.setAccessible(true); + LinkedList unsealedSegment = (LinkedList) unsealedSegmentField.get(processor); + LinkedMap indexes = (LinkedMap) indexField.get(processor); + Assert.assertEquals(unsealedSegment.size(), txnIdSize % SEGMENT_SIZE); + Assert.assertEquals(indexes.size(), txnIdSize / SEGMENT_SIZE); + } + + // Verify the update index future can be completed when the queue has other tasks. + @Test + public void testFuturesCanCompleteWhenItIsCanceled() throws Exception { + PersistentTopic persistentTopic = (PersistentTopic) pulsarService.getBrokerService() + .getTopic(PROCESSOR_TOPIC, false).get().get(); + AbortedTxnProcessor processor = new SnapshotSegmentAbortedTxnProcessorImpl(persistentTopic); + Field workerField = SnapshotSegmentAbortedTxnProcessorImpl.class.getDeclaredField("persistentWorker"); + workerField.setAccessible(true); + SnapshotSegmentAbortedTxnProcessorImpl.PersistentWorker persistentWorker = + (SnapshotSegmentAbortedTxnProcessorImpl.PersistentWorker) workerField.get(processor); + Field taskQueueField = SnapshotSegmentAbortedTxnProcessorImpl.PersistentWorker.class + .getDeclaredField("taskQueue"); + taskQueueField.setAccessible(true); + Supplier task = CompletableFuture::new; + Queue queue = (Queue) taskQueueField.get(persistentWorker); + queue.add(new MutablePair<>(SnapshotSegmentAbortedTxnProcessorImpl.PersistentWorker.OperationType.WriteSegment, + new MutablePair<>(new CompletableFuture<>(), task))); + try { + processor.takeAbortedTxnsSnapshot(new PositionImpl(1, 10)).get(2, TimeUnit.SECONDS); + } catch (Exception e) { + Assert.assertTrue(e.getCause() instanceof BrokerServiceException.ServiceUnitNotReadyException); + } + } + + @Test + public void testClearSnapshotSegments() throws Exception { + PersistentTopic persistentTopic = (PersistentTopic) pulsarService.getBrokerService() + .getTopic(PROCESSOR_TOPIC, false).get().get(); + AbortedTxnProcessor processor = new SnapshotSegmentAbortedTxnProcessorImpl(persistentTopic); + //1. Write two snapshot segment. + for (int j = 0; j < SEGMENT_SIZE * 2; j++) { + TxnID txnID = new TxnID(0, j); + PositionImpl position = new PositionImpl(0, j); + processor.putAbortedTxnAndPosition(txnID, position); + } + Awaitility.await().untilAsserted(() -> verifySnapshotSegmentsSize(PROCESSOR_TOPIC, 2)); + //2. Close index writer, making the index can not be updated. + Field field = SnapshotSegmentAbortedTxnProcessorImpl.class.getDeclaredField("persistentWorker"); + field.setAccessible(true); + SnapshotSegmentAbortedTxnProcessorImpl.PersistentWorker worker = + (SnapshotSegmentAbortedTxnProcessorImpl.PersistentWorker) field.get(processor); + Field indexWriteFutureField = SnapshotSegmentAbortedTxnProcessorImpl + .PersistentWorker.class.getDeclaredField("snapshotIndexWriterFuture"); + indexWriteFutureField.setAccessible(true); + CompletableFuture> snapshotIndexWriterFuture = + (CompletableFuture>) + indexWriteFutureField.get(worker); + snapshotIndexWriterFuture.get().close(); + //3. Try to write a snapshot segment that will fail to update indexes. + for (int j = 0; j < SEGMENT_SIZE; j++) { + TxnID txnID = new TxnID(0, j); + PositionImpl position = new PositionImpl(0, j); + processor.putAbortedTxnAndPosition(txnID, position); + } + //4. Wait writing segment completed. + Awaitility.await().untilAsserted(() -> verifySnapshotSegmentsSize(PROCESSOR_TOPIC, 3)); + //5. Clear all the snapshot segments and indexes. + try { + processor.clearAbortedTxnSnapshot().get(); + //Failed to clear index due to the index writer is closed. + Assert.fail(); + } catch (Exception ignored) { + } + //6. Do compaction and wait it completed. + TopicName segmentTopicName = NamespaceEventsSystemTopicFactory.getSystemTopicName( + TopicName.get(PROCESSOR_TOPIC).getNamespaceObject(), + EventType.TRANSACTION_BUFFER_SNAPSHOT_SEGMENTS); + TopicName indexTopicName = NamespaceEventsSystemTopicFactory.getSystemTopicName( + TopicName.get(PROCESSOR_TOPIC).getNamespaceObject(), + EventType.TRANSACTION_BUFFER_SNAPSHOT_INDEXES); + doCompaction(segmentTopicName); + doCompaction(indexTopicName); + //7. Verify the snapshot segments and index after clearing. + verifySnapshotSegmentsSize(PROCESSOR_TOPIC, 0); + verifySnapshotSegmentsIndexSize(PROCESSOR_TOPIC, 1); + } + + private void verifySnapshotSegmentsSize(String topic, int size) throws Exception { + SystemTopicClient.Reader reader = + pulsarService.getTransactionBufferSnapshotServiceFactory() + .getTxnBufferSnapshotSegmentService() + .createReader(TopicName.get(topic)).get(); + int segmentCount = 0; + while (reader.hasMoreEvents()) { + Message message = reader.readNextAsync() + .get(5, TimeUnit.SECONDS); + if (topic.equals(message.getValue().getTopicName())) { + segmentCount++; + } + } + Assert.assertEquals(segmentCount, size); + } + + private void verifySnapshotSegmentsIndexSize(String topic, int size) throws Exception { + SystemTopicClient.Reader reader = + pulsarService.getTransactionBufferSnapshotServiceFactory() + .getTxnBufferSnapshotIndexService() + .createReader(TopicName.get(topic)).get(); + int indexCount = 0; + while (reader.hasMoreEvents()) { + Message message = reader.readNextAsync() + .get(5, TimeUnit.SECONDS); + if (topic.equals(message.getValue().getTopicName())) { + indexCount++; + } + System.out.printf("message.getValue().getTopicName() :" + message.getValue().getTopicName()); + } + Assert.assertEquals(indexCount, size); + } + + private void doCompaction(TopicName topic) throws Exception { + PersistentTopic snapshotTopic = (PersistentTopic) pulsarService.getBrokerService() + .getTopic(topic.toString(), false).get().get(); + Field field = PersistentTopic.class.getDeclaredField("currentCompaction"); + field.setAccessible(true); + snapshotTopic.triggerCompaction(); + CompletableFuture compactionFuture = (CompletableFuture) field.get(snapshotTopic); + org.awaitility.Awaitility.await().untilAsserted(() -> assertTrue(compactionFuture.isDone())); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java index a2b72fc458db4..d4ddb26e014ca 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java @@ -79,6 +79,8 @@ import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.client.api.ReaderBuilder; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.api.schema.GenericRecord; import org.apache.pulsar.client.api.transaction.Transaction; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.MessageIdImpl; @@ -91,6 +93,7 @@ import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.awaitility.Awaitility; +import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; @@ -530,9 +533,14 @@ public void clearTransactionBufferSnapshotTest(Boolean enableSnapshotSegment) th (AbortedTxnProcessor) abortedTxnProcessorField.get(topicTransactionBuffer); abortedTxnProcessor.takeAbortedTxnsSnapshot(topicTransactionBuffer.getMaxReadPosition()); - TopicName transactionBufferTopicName = - NamespaceEventsSystemTopicFactory.getSystemTopicName( - TopicName.get(topic).getNamespaceObject(), EventType.TRANSACTION_BUFFER_SNAPSHOT); + TopicName transactionBufferTopicName; + if (!enableSnapshotSegment) { + transactionBufferTopicName = NamespaceEventsSystemTopicFactory.getSystemTopicName( + TopicName.get(topic).getNamespaceObject(), EventType.TRANSACTION_BUFFER_SNAPSHOT); + } else { + transactionBufferTopicName = NamespaceEventsSystemTopicFactory.getSystemTopicName( + TopicName.get(topic).getNamespaceObject(), EventType.TRANSACTION_BUFFER_SNAPSHOT_INDEXES); + } PersistentTopic snapshotTopic = (PersistentTopic) getPulsarServiceList().get(0) .getBrokerService().getTopic(transactionBufferTopicName.toString(), false).get().get(); Field field = PersistentTopic.class.getDeclaredField("currentCompaction"); @@ -550,7 +558,7 @@ private void checkSnapshotCount(TopicName topicName, boolean hasSnapshot, CompletableFuture compactionFuture = (CompletableFuture) field.get(persistentTopic); Awaitility.await().untilAsserted(() -> assertTrue(compactionFuture.isDone())); - Reader reader = pulsarClient.newReader(Schema.AVRO(TransactionBufferSnapshot.class)) + Reader reader = pulsarClient.newReader(Schema.AUTO_CONSUME()) .readCompacted(true) .startMessageId(MessageId.earliest) .startMessageIdInclusive() @@ -559,7 +567,7 @@ private void checkSnapshotCount(TopicName topicName, boolean hasSnapshot, int count = 0; while (true) { - Message snapshotMsg = reader.readNext(2, TimeUnit.SECONDS); + Message snapshotMsg = reader.readNext(2, TimeUnit.SECONDS); if (snapshotMsg != null) { count++; } else { @@ -721,10 +729,10 @@ public void testTransactionBufferIndexSystemTopic() throws Exception { TransactionBufferSnapshotIndex transactionBufferSnapshotIndex = transactionBufferTransactionBufferSnapshotIndexes.getIndexList().get(1); - assertEquals(transactionBufferSnapshotIndex.getMaxReadPositionLedgerID(), 1L); - assertEquals(transactionBufferSnapshotIndex.getMaxReadPositionEntryID(), 1L); - assertEquals(transactionBufferSnapshotIndex.getPersistentPositionLedgerID(), 1L); - assertEquals(transactionBufferSnapshotIndex.getPersistentPositionEntryID(), 1L); + assertEquals(transactionBufferSnapshotIndex.getAbortedMarkLedgerID(), 1L); + assertEquals(transactionBufferSnapshotIndex.getAbortedMarkEntryID(), 1L); + assertEquals(transactionBufferSnapshotIndex.getSegmentLedgerID(), 1L); + assertEquals(transactionBufferSnapshotIndex.getSegmentEntryID(), 1L); assertEquals(transactionBufferSnapshotIndex.getSequenceID(), 1L); } @@ -765,8 +773,8 @@ public void testTransactionBufferSegmentSystemTopic() throws Exception { //build and send snapshot snapshot.setTopicName(snapshotTopic); snapshot.setSequenceId(1L); - snapshot.setMaxReadPositionLedgerId(2L); - snapshot.setMaxReadPositionEntryId(3L); + snapshot.setPersistentPositionLedgerId(2L); + snapshot.setPersistentPositionEntryId(3L); LinkedList txnIDSet = new LinkedList<>(); txnIDSet.add(new TxnIDData(1, 1)); snapshot.setAborts(txnIDSet ); @@ -818,9 +826,99 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob //verify snapshot assertEquals(snapshot.getTopicName(), snapshotTopic); assertEquals(snapshot.getSequenceId(), 2L); - assertEquals(snapshot.getMaxReadPositionLedgerId(), 2L); - assertEquals(snapshot.getMaxReadPositionEntryId(), 3L); + assertEquals(snapshot.getPersistentPositionLedgerId(), 2L); + assertEquals(snapshot.getPersistentPositionEntryId(), 3L); assertEquals(snapshot.getAborts().toArray()[0], new TxnIDData(1, 1)); } + //Verify the snapshotSegmentProcessor end to end + @Test + public void testSnapshotSegment() throws Exception { + String topic ="persistent://" + NAMESPACE1 + "/testSnapshotSegment"; + String subName = "testSnapshotSegment"; + + LinkedMap ongoingTxns = new LinkedMap<>(); + LinkedList abortedTxns = new LinkedList<>(); + // 0. Modify the configurations, enabling the segment snapshot and set the size of the snapshot segment. + int theSizeOfSegment = 10; + int theCountOfSnapshotMaxTxnCount = 3; + this.getPulsarServiceList().get(0).getConfig().setTransactionBufferSegmentedSnapshotEnabled(true); + this.getPulsarServiceList().get(0).getConfig() + .setTransactionBufferSnapshotSegmentSize(8 + topic.length() + theSizeOfSegment * 3); + this.getPulsarServiceList().get(0).getConfig() + .setTransactionBufferSnapshotMaxTransactionCount(theCountOfSnapshotMaxTxnCount); + // 1. Build producer and consumer + Producer producer = pulsarClient.newProducer(Schema.INT32) + .topic(topic) + .enableBatching(false) + .create(); + + Consumer consumer = pulsarClient.newConsumer(Schema.INT32) + .topic(topic) + .subscriptionName(subName) + .subscriptionType(SubscriptionType.Exclusive) + .subscribe(); + + // 2. Check the AbortedTxnProcessor workflow 10 times + int messageSize = theSizeOfSegment * 4; + for (int i = 0; i < 10; i++) { + MessageId maxReadMessage = null; + int abortedTxnSize = 0; + for (int j = 0; j < messageSize; j++) { + Transaction transaction = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.MINUTES).build().get(); + //Half common message and half transaction message. + if (j % 2 == 0) { + MessageId messageId = producer.newMessage(transaction).value(i * 10 + j).send(); + //And the transaction message have a half which are aborted. + if (RandomUtils.nextInt() % 2 == 0) { + transaction.abort().get(); + abortedTxns.add(messageId); + abortedTxnSize++; + } else { + ongoingTxns.put(transaction, messageId); + if (maxReadMessage == null) { + //The except number of the messages that can be read + maxReadMessage = messageId; + } + } + } else { + producer.newMessage().value(i * 10 + j).send(); + transaction.commit().get(); + } + } + // 2.1 Receive all message before the maxReadPosition to verify the correctness of the max read position. + int hasReceived = 0; + while (true) { + Message message = consumer.receive(2, TimeUnit.SECONDS); + if (message != null) { + Assert.assertTrue(message.getMessageId().compareTo(maxReadMessage) < 0); + hasReceived ++; + } else { + break; + } + } + //2.2 Commit all ongoing transaction and verify that the consumer can receive all rest message + // expect for aborted txn message. + for (Transaction ongoingTxn: ongoingTxns.keySet()) { + ongoingTxn.commit().get(); + } + ongoingTxns.clear(); + for (int k = hasReceived; k < messageSize - abortedTxnSize; k++) { + Message message = consumer.receive(2, TimeUnit.SECONDS); + assertNotNull(message); + assertFalse(abortedTxns.contains(message.getMessageId())); + } + } + // 3. After the topic unload, the consumer can receive all the messages in the 10 tests + // expect for the aborted transaction messages. + admin.topics().unload(topic); + for (int i = 0; i < messageSize * 10 - abortedTxns.size(); i++) { + Message message = consumer.receive(2, TimeUnit.SECONDS); + assertNotNull(message); + assertFalse(abortedTxns.contains(message.getMessageId())); + } + assertNull(consumer.receive(2, TimeUnit.SECONDS)); + } + } From de4f62003939d16fe635ab4204b8444bc10eb112 Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Sun, 12 Feb 2023 21:26:39 -0800 Subject: [PATCH 008/174] [improve][broker] PIP-192 Added broker and top-bundles load reporters (#19471) --- .../pulsar/broker/ServiceConfiguration.java | 11 ++ .../extensions/ExtensibleLoadManagerImpl.java | 55 ++++++- .../extensions/data/BrokerLoadData.java | 21 ++- .../extensions/data/TopBundlesLoadData.java | 27 +-- .../extensions/models/TopKBundles.java | 112 +++++++++++++ .../reporter/BrokerLoadDataReporter.java | 155 ++++++++++++++++++ .../reporter/TopBundleLoadDataReporter.java | 86 ++++++++++ .../extensions/scheduler/TransferShedder.java | 65 +++----- .../pulsar/broker/service/PulsarStats.java | 26 +++ .../pulsar/broker/stats/BrokerStats.java | 33 ++++ .../ExtensibleLoadManagerImplTest.java | 2 +- .../extensions/data/BrokerLoadDataTest.java | 14 +- .../data/TopBundlesLoadDataTest.java | 59 ------- .../extensions/models/TopKBundlesTest.java | 118 +++++++++++++ .../reporter/BrokerLoadDataReporterTest.java | 126 ++++++++++++++ .../TopBundleLoadDataReporterTest.java | 125 ++++++++++++++ .../scheduler/TransferShedderTest.java | 38 ++--- .../LeastResourceUsageWithWeightTest.java | 4 +- 18 files changed, 930 insertions(+), 147 deletions(-) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporter.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerStats.java delete mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/TopBundlesLoadDataTest.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundlesTest.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 557b30245af38..106410d855e22 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -2514,6 +2514,17 @@ The delayed message index bucket time step(in seconds) in per bucket snapshot se ) private long loadBalancerBrokerLoadDataTTLInSeconds = 1800; + @FieldContext( + dynamic = true, + category = CATEGORY_LOAD_BALANCER, + doc = "Percentage of bundles to compute topK bundle load data from each broker. " + + "The load balancer distributes bundles across brokers, " + + "based on topK bundle load data and other broker load data." + + "The bigger value will increase the overhead of reporting many bundles in load data. " + + "(only used in load balancer extension logics)" + ) + private double loadBalancerBundleLoadReportPercentage = 10; + /**** --- Replication. --- ****/ @FieldContext( category = CATEGORY_REPLICATION, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index 171651e50c2f8..fe28d67227e8a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -26,6 +26,8 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -45,6 +47,8 @@ import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision; import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter; import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision; +import org.apache.pulsar.broker.loadbalance.extensions.reporter.BrokerLoadDataReporter; +import org.apache.pulsar.broker.loadbalance.extensions.reporter.TopBundleLoadDataReporter; import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore; import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStoreException; import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStoreFactory; @@ -91,6 +95,16 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager { @Getter private final List brokerFilterPipeline; + /** + * The load data reporter. + */ + private BrokerLoadDataReporter brokerLoadDataReporter; + + private TopBundleLoadDataReporter topBundleLoadDataReporter; + + private ScheduledFuture brokerLoadDataReportTask; + private ScheduledFuture topBundlesLoadDataReportTask; + private boolean started = false; private final AssignCounter assignCounter = new AssignCounter(); @@ -147,7 +161,38 @@ public void start() throws PulsarServerException { .brokerRegistry(brokerRegistry) .brokerLoadDataStore(brokerLoadDataStore) .topBundleLoadDataStore(topBundlesLoadDataStore).build(); - // TODO: Start load data reporter. + + + this.brokerLoadDataReporter = + new BrokerLoadDataReporter(pulsar, brokerRegistry.getBrokerId(), brokerLoadDataStore); + + this.topBundleLoadDataReporter = + new TopBundleLoadDataReporter(pulsar, brokerRegistry.getBrokerId(), topBundlesLoadDataStore); + + var interval = conf.getLoadBalancerReportUpdateMinIntervalMillis(); + this.brokerLoadDataReportTask = this.pulsar.getLoadManagerExecutor() + .scheduleAtFixedRate(() -> { + try { + brokerLoadDataReporter.reportAsync(false); + // TODO: update broker load metrics using getLocalData + } catch (Throwable e) { + log.error("Failed to run the broker load manager executor job.", e); + } + }, + interval, + interval, TimeUnit.MILLISECONDS); + + this.topBundlesLoadDataReportTask = this.pulsar.getLoadManagerExecutor() + .scheduleAtFixedRate(() -> { + try { + // TODO: consider excluding the bundles that are in the process of split. + topBundleLoadDataReporter.reportAsync(false); + } catch (Throwable e) { + log.error("Failed to run the top bundles load manager executor job.", e); + } + }, + interval, + interval, TimeUnit.MILLISECONDS); // TODO: Start unload scheduler and bundle split scheduler this.started = true; @@ -264,6 +309,14 @@ public void close() throws PulsarServerException { return; } try { + if (brokerLoadDataReportTask != null) { + brokerLoadDataReportTask.cancel(true); + } + + if (topBundlesLoadDataReportTask != null) { + topBundlesLoadDataReportTask.cancel(true); + } + this.brokerLoadDataStore.close(); this.topBundlesLoadDataStore.close(); } catch (IOException ex) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadData.java index c3309987bec59..dbd17152d26e4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadData.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadData.java @@ -23,6 +23,8 @@ import java.util.List; import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.Setter; +import lombok.ToString; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.common.stats.Metrics; import org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData; @@ -37,6 +39,7 @@ */ @Getter @EqualsAndHashCode +@ToString public class BrokerLoadData { private static final double DEFAULT_RESOURCE_USAGE = 1.0d; @@ -53,6 +56,7 @@ public class BrokerLoadData { private double msgThroughputOut; // bytes/sec private double msgRateIn; // messages/sec private double msgRateOut; // messages/sec + private int bundleCount; // Load data features computed from the above resources. private double maxResourceUsage; // max of resource usages @@ -72,6 +76,9 @@ public class BrokerLoadData { private double weightedMaxEMA; private long updatedAt; + @Setter + private long reportedAt; + public BrokerLoadData() { cpu = new ResourceUsage(); memory = new ResourceUsage(); @@ -95,6 +102,8 @@ public BrokerLoadData() { * broker-level message input rate in messages/s. * @param msgRateOut * broker-level message output rate in messages/s. + * @param bundleCount + * broker-level bundle counts. * @param conf * Service configuration to compute load data features. */ @@ -103,12 +112,14 @@ public void update(final SystemResourceUsage usage, double msgThroughputOut, double msgRateIn, double msgRateOut, + int bundleCount, ServiceConfiguration conf) { updateSystemResourceUsage(usage.cpu, usage.memory, usage.directMemory, usage.bandwidthIn, usage.bandwidthOut); this.msgThroughputIn = msgThroughputIn; this.msgThroughputOut = msgThroughputOut; this.msgRateIn = msgRateIn; this.msgRateOut = msgRateOut; + this.bundleCount = bundleCount; updateFeatures(conf); updatedAt = System.currentTimeMillis(); } @@ -125,9 +136,11 @@ public void update(final BrokerLoadData other) { msgThroughputOut = other.msgThroughputOut; msgRateIn = other.msgRateIn; msgRateOut = other.msgRateOut; + bundleCount = other.bundleCount; weightedMaxEMA = other.weightedMaxEMA; maxResourceUsage = other.maxResourceUsage; updatedAt = other.updatedAt; + reportedAt = other.reportedAt; } // Update resource usage given each individual usage. @@ -177,7 +190,9 @@ public String toString(ServiceConfiguration conf) { + "cpuWeight= %f, memoryWeight= %f, directMemoryWeight= %f, " + "bandwithInResourceWeight= %f, bandwithOutResourceWeight= %f, " + "msgThroughputIn= %.2f, msgThroughputOut= %.2f, msgRateIn= %.2f, msgRateOut= %.2f, " - + "maxResourceUsage= %.2f%%, weightedMaxEMA= %.2f%%, updatedAt= %d", + + "bundleCount= %d, " + + "maxResourceUsage= %.2f%%, weightedMaxEMA= %.2f%%, " + + "updatedAt= %d, reportedAt= %d", cpu.percentUsage(), memory.percentUsage(), directMemory.percentUsage(), bandwidthIn.percentUsage(), bandwidthOut.percentUsage(), @@ -187,7 +202,9 @@ public String toString(ServiceConfiguration conf) { conf.getLoadBalancerBandwithInResourceWeight(), conf.getLoadBalancerBandwithOutResourceWeight(), msgThroughputIn, msgThroughputOut, msgRateIn, msgRateOut, - maxResourceUsage * 100, weightedMaxEMA * 100, updatedAt + bundleCount, + maxResourceUsage * 100, weightedMaxEMA * 100, + updatedAt, reportedAt ); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/TopBundlesLoadData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/TopBundlesLoadData.java index 9c34b40ff0416..a9562a32a3542 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/TopBundlesLoadData.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/TopBundlesLoadData.java @@ -18,19 +18,25 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.data; +import java.util.ArrayList; import java.util.List; import java.util.Objects; -import java.util.stream.Collectors; +import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.ToString; import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; /** * Defines the information of top bundles load data. */ @Getter +@ToString +@EqualsAndHashCode +@NoArgsConstructor public class TopBundlesLoadData { - private final List topBundlesLoadData; + private final List topBundlesLoadData = new ArrayList<>(); public record BundleLoadData(String bundleName, NamespaceBundleStats stats) { public BundleLoadData { @@ -38,21 +44,4 @@ public record BundleLoadData(String bundleName, NamespaceBundleStats stats) { } } - private TopBundlesLoadData(List bundleStats, int topK) { - topBundlesLoadData = bundleStats - .stream() - .sorted((o1, o2) -> o2.stats().compareTo(o1.stats())) - .limit(topK) - .collect(Collectors.toList()); - } - - /** - * Give full bundle stats, and return the top K bundle stats. - * - * @param bundleStats full bundle stats. - * @param topK Top K bundles. - */ - public static TopBundlesLoadData of(List bundleStats, int topK) { - return new TopBundlesLoadData(bundleStats, topK); - } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java new file mode 100644 index 0000000000000..c189005b9539c --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.models; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.ToString; +import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; + +/** + * Defines the information of top k highest-loaded bundles. + */ +@Getter +@ToString +@EqualsAndHashCode +@NoArgsConstructor +public class TopKBundles { + + // temp array for sorting + private final List> arr = new ArrayList<>(); + + private final TopBundlesLoadData loadData = new TopBundlesLoadData(); + + /** + * Update the topK bundles from the input bundleStats. + * + * @param bundleStats bundle stats. + * @param topk top k bundle stats to select. + */ + public void update(Map bundleStats, int topk) { + arr.clear(); + for (var etr : bundleStats.entrySet()) { + if (etr.getKey().startsWith(NamespaceName.SYSTEM_NAMESPACE.toString())) { + continue; + } + arr.add(etr); + } + var topKBundlesLoadData = loadData.getTopBundlesLoadData(); + topKBundlesLoadData.clear(); + if (arr.isEmpty()) { + return; + } + topk = Math.min(topk, arr.size()); + partitionSort(arr, topk); + + for (int i = 0; i < topk; i++) { + var etr = arr.get(i); + topKBundlesLoadData.add( + new TopBundlesLoadData.BundleLoadData(etr.getKey(), (NamespaceBundleStats) etr.getValue())); + } + arr.clear(); + } + + static void partitionSort(List> arr, int k) { + int start = 0; + int end = arr.size() - 1; + int target = k - 1; + while (start < end) { + int lo = start; + int hi = end; + int mid = lo; + var pivot = arr.get(hi).getValue(); + while (mid <= hi) { + int cmp = pivot.compareTo(arr.get(mid).getValue()); + if (cmp < 0) { + var tmp = arr.get(lo); + arr.set(lo++, arr.get(mid)); + arr.set(mid++, tmp); + } else if (cmp > 0) { + var tmp = arr.get(mid); + arr.set(mid, arr.get(hi)); + arr.set(hi--, tmp); + } else { + mid++; + } + } + if (lo <= target && target < mid) { + end = lo; + break; + } + if (target < lo) { + end = lo - 1; + } else { + start = mid; + } + } + Collections.sort(arr.subList(0, end), (a, b) -> b.getValue().compareTo(a.getValue())); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporter.java new file mode 100644 index 0000000000000..cf50f942e11b4 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporter.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.reporter; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.SystemUtils; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.BrokerHostUsage; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; +import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore; +import org.apache.pulsar.broker.loadbalance.impl.GenericBrokerHostUsageImpl; +import org.apache.pulsar.broker.loadbalance.impl.LinuxBrokerHostUsageImpl; +import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; +import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage; + +/** + * The broker load data reporter. + */ +@Slf4j +public class BrokerLoadDataReporter implements LoadDataReporter { + + private final PulsarService pulsar; + + private final ServiceConfiguration conf; + + private final LoadDataStore brokerLoadDataStore; + + private final BrokerHostUsage brokerHostUsage; + + private final String lookupServiceAddress; + + @Getter + private final BrokerLoadData localData; + + private final BrokerLoadData lastData; + + public BrokerLoadDataReporter(PulsarService pulsar, + String lookupServiceAddress, + LoadDataStore brokerLoadDataStore) { + this.brokerLoadDataStore = brokerLoadDataStore; + this.lookupServiceAddress = lookupServiceAddress; + this.pulsar = pulsar; + this.conf = this.pulsar.getConfiguration(); + if (SystemUtils.IS_OS_LINUX) { + brokerHostUsage = new LinuxBrokerHostUsageImpl(pulsar); + } else { + brokerHostUsage = new GenericBrokerHostUsageImpl(pulsar); + } + this.localData = new BrokerLoadData(); + this.lastData = new BrokerLoadData(); + + } + + @Override + public BrokerLoadData generateLoadData() { + final SystemResourceUsage systemResourceUsage = LoadManagerShared.getSystemResourceUsage(brokerHostUsage); + final var pulsarStats = pulsar.getBrokerService().getPulsarStats(); + synchronized (pulsarStats) { + var brokerStats = pulsarStats.getBrokerStats(); + localData.update(systemResourceUsage, + brokerStats.msgThroughputIn, + brokerStats.msgThroughputOut, + brokerStats.msgRateIn, + brokerStats.msgRateOut, + brokerStats.bundleCount, + pulsar.getConfiguration()); + + } + return this.localData; + } + + @Override + public CompletableFuture reportAsync(boolean force) { + BrokerLoadData newLoadData = this.generateLoadData(); + if (force || needBrokerDataUpdate()) { + log.info("publishing load report:{}", localData.toString(conf)); + CompletableFuture future = + this.brokerLoadDataStore.pushAsync(this.lookupServiceAddress, newLoadData); + future.whenComplete((__, ex) -> { + if (ex == null) { + localData.setReportedAt(System.currentTimeMillis()); + lastData.update(localData); + } else { + log.error("Failed to report the broker load data.", ex); + } + }); + return future; + } else { + log.info("skipping load report:{}", localData.toString(conf)); + } + return CompletableFuture.completedFuture(null); + } + + private boolean needBrokerDataUpdate() { + int loadBalancerReportUpdateMaxIntervalMinutes = conf.getLoadBalancerReportUpdateMaxIntervalMinutes(); + int loadBalancerReportUpdateThresholdPercentage = conf.getLoadBalancerReportUpdateThresholdPercentage(); + final long updateMaxIntervalMillis = TimeUnit.MINUTES + .toMillis(loadBalancerReportUpdateMaxIntervalMinutes); + long timeSinceLastReportWrittenToStore = System.currentTimeMillis() - localData.getReportedAt(); + if (timeSinceLastReportWrittenToStore > updateMaxIntervalMillis) { + log.info("Writing local data to metadata store because time since last" + + " update exceeded threshold of {} minutes", + loadBalancerReportUpdateMaxIntervalMinutes); + // Always update after surpassing the maximum interval. + return true; + } + final double maxChange = Math + .max(100.0 * (Math.abs(lastData.getMaxResourceUsage() - localData.getMaxResourceUsage())), + Math.max(percentChange(lastData.getMsgRateIn() + lastData.getMsgRateOut(), + localData.getMsgRateIn() + localData.getMsgRateOut()), + Math.max( + percentChange(lastData.getMsgThroughputIn() + lastData.getMsgThroughputOut(), + localData.getMsgThroughputIn() + localData.getMsgThroughputOut()), + percentChange(lastData.getBundleCount(), localData.getBundleCount())))); + if (maxChange > loadBalancerReportUpdateThresholdPercentage) { + log.info("Writing local data to metadata store because maximum change {}% exceeded threshold {}%; " + + "time since last report written is {} seconds", maxChange, + loadBalancerReportUpdateThresholdPercentage, + timeSinceLastReportWrittenToStore / 1000.0); + return true; + } + return false; + } + + protected double percentChange(final double oldValue, final double newValue) { + if (oldValue == 0) { + if (newValue == 0) { + // Avoid NaN + return 0; + } + return Double.POSITIVE_INFINITY; + } + return 100 * Math.abs((oldValue - newValue) / oldValue); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java new file mode 100644 index 0000000000000..59e328fc2be80 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.reporter; + +import java.util.concurrent.CompletableFuture; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData; +import org.apache.pulsar.broker.loadbalance.extensions.models.TopKBundles; +import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore; + +/** + * The top k highest-loaded bundles' load data reporter. + */ +@Slf4j +public class TopBundleLoadDataReporter implements LoadDataReporter { + + private final PulsarService pulsar; + + private final String lookupServiceAddress; + + private final LoadDataStore bundleLoadDataStore; + + private final TopKBundles topKBundles; + + private long lastBundleStatsUpdatedAt; + + public TopBundleLoadDataReporter(PulsarService pulsar, + String lookupServiceAddress, + LoadDataStore bundleLoadDataStore) { + this.pulsar = pulsar; + this.lookupServiceAddress = lookupServiceAddress; + this.bundleLoadDataStore = bundleLoadDataStore; + this.lastBundleStatsUpdatedAt = 0; + this.topKBundles = new TopKBundles(); + } + + @Override + public TopBundlesLoadData generateLoadData() { + + var pulsarStats = pulsar.getBrokerService().getPulsarStats(); + TopBundlesLoadData result = null; + synchronized (pulsarStats) { + var pulsarStatsUpdatedAt = pulsarStats.getUpdatedAt(); + if (pulsarStatsUpdatedAt > lastBundleStatsUpdatedAt) { + var bundleStats = pulsar.getBrokerService().getBundleStats(); + double percentage = pulsar.getConfiguration().getLoadBalancerBundleLoadReportPercentage(); + int topk = Math.max(1, (int) (bundleStats.size() * percentage / 100.0)); + topKBundles.update(bundleStats, topk); + lastBundleStatsUpdatedAt = pulsarStatsUpdatedAt; + result = topKBundles.getLoadData(); + } + } + return result; + } + + @Override + public CompletableFuture reportAsync(boolean force) { + var topBundlesLoadData = generateLoadData(); + if (topBundlesLoadData != null || force) { + return this.bundleLoadDataStore.pushAsync(lookupServiceAddress, topKBundles.getLoadData()) + .exceptionally(e -> { + log.error("Failed to report top-bundles load data.", e); + return null; + }); + } else { + return CompletableFuture.completedFuture(null); + } + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java index a1047edc06fc9..7f9128de81754 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java @@ -28,10 +28,6 @@ import lombok.Getter; import lombok.experimental.Accessors; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.mutable.MutableBoolean; -import org.apache.commons.lang3.mutable.MutableDouble; -import org.apache.commons.lang3.mutable.MutableInt; -import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; @@ -323,8 +319,8 @@ public UnloadDecision findBundlesForUnloading(LoadManagerContext context, offload * 100, offloadThroughput / KB, (brokerThroughput - offloadThroughput) / KB); } - MutableDouble trafficMarkedToOffload = new MutableDouble(0); - MutableBoolean atLeastOneBundleSelected = new MutableBoolean(false); + double trafficMarkedToOffload = 0; + boolean atLeastOneBundleSelected = false; Optional bundlesLoadData = context.topBundleLoadDataStore().get(maxBroker); if (bundlesLoadData.isEmpty() || bundlesLoadData.get().getTopBundlesLoadData().isEmpty()) { @@ -335,37 +331,30 @@ public UnloadDecision findBundlesForUnloading(LoadManagerContext context, var topBundlesLoadData = bundlesLoadData.get().getTopBundlesLoadData(); if (topBundlesLoadData.size() > 1) { - MutableInt remainingTopBundles = new MutableInt(); - topBundlesLoadData.stream() - .filter(e -> - !recentlyUnloadedBundles.containsKey(e.bundleName()) && isTransferable( - e.bundleName()) - ).map((e) -> { - String bundle = e.bundleName(); - var bundleData = e.stats(); - double throughput = bundleData.msgThroughputIn + bundleData.msgThroughputOut; - remainingTopBundles.increment(); - return Pair.of(bundle, throughput); - }).sorted((e1, e2) -> - Double.compare(e2.getRight(), e1.getRight()) - ).forEach(e -> { - if (remainingTopBundles.getValue() > 1 - && (trafficMarkedToOffload.doubleValue() < offloadThroughput - || atLeastOneBundleSelected.isFalse())) { - if (transfer) { - selectedBundlesCache.put(maxBroker, - new Unload(maxBroker, e.getLeft(), - Optional.of(minBroker))); - } else { - selectedBundlesCache.put(maxBroker, - new Unload(maxBroker, e.getLeft())); - } - trafficMarkedToOffload.add(e.getRight()); - atLeastOneBundleSelected.setTrue(); - remainingTopBundles.decrement(); + int remainingTopBundles = topBundlesLoadData.size(); + for (var e : topBundlesLoadData) { + String bundle = e.bundleName(); + if (!recentlyUnloadedBundles.containsKey(bundle) && isTransferable(bundle)) { + var bundleData = e.stats(); + double throughput = bundleData.msgThroughputIn + bundleData.msgThroughputOut; + if (remainingTopBundles > 1 + && (trafficMarkedToOffload < offloadThroughput + || !atLeastOneBundleSelected)) { + if (transfer) { + selectedBundlesCache.put(maxBroker, + new Unload(maxBroker, bundle, + Optional.of(minBroker))); + } else { + selectedBundlesCache.put(maxBroker, + new Unload(maxBroker, bundle)); } - }); - if (atLeastOneBundleSelected.isFalse()) { + trafficMarkedToOffload += throughput; + atLeastOneBundleSelected = true; + remainingTopBundles--; + } + } + } + if (!atLeastOneBundleSelected) { numOfBrokersWithFewBundles++; } } else if (topBundlesLoadData.size() == 1) { @@ -379,9 +368,7 @@ public UnloadDecision findBundlesForUnloading(LoadManagerContext context, log.warn("Broker {} is overloaded despite having no bundles", maxBroker); } - - - if (trafficMarkedToOffload.getValue() > 0) { + if (trafficMarkedToOffload > 0) { stats.offload(max, min, offload); if (debugMode) { log.info( diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java index aa734ce73dbc0..045bb336d62e2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java @@ -28,10 +28,12 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Consumer; +import lombok.Getter; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.stats.BrokerOperabilityMetrics; +import org.apache.pulsar.broker.stats.BrokerStats; import org.apache.pulsar.broker.stats.ClusterReplicationMetrics; import org.apache.pulsar.broker.stats.NamespaceStats; import org.apache.pulsar.common.naming.NamespaceBundle; @@ -47,7 +49,11 @@ public class PulsarStats implements Closeable { private volatile ByteBuf topicStatsBuf; private volatile ByteBuf tempTopicStatsBuf; + + @Getter + private BrokerStats brokerStats; private NamespaceStats nsStats; + private final ClusterReplicationMetrics clusterReplicationMetrics; private Map bundleStats; private List tempMetricsCollection; @@ -58,11 +64,15 @@ public class PulsarStats implements Closeable { private final ReentrantReadWriteLock bufferLock = new ReentrantReadWriteLock(); + @Getter + private long updatedAt; + public PulsarStats(PulsarService pulsar) { this.topicStatsBuf = Unpooled.buffer(16 * 1024); this.tempTopicStatsBuf = Unpooled.buffer(16 * 1024); this.nsStats = new NamespaceStats(pulsar.getConfig().getStatsUpdateFrequencyInSecs()); + this.brokerStats = new BrokerStats(pulsar.getConfig().getStatsUpdateFrequencyInSecs()); this.clusterReplicationMetrics = new ClusterReplicationMetrics(pulsar.getConfiguration().getClusterName(), pulsar.getConfiguration().isReplicationMetricsEnabled()); this.bundleStats = new ConcurrentHashMap<>(); @@ -73,6 +83,8 @@ public PulsarStats(PulsarService pulsar) { this.tempNonPersistentTopics = new ArrayList<>(); this.exposePublisherStats = pulsar.getConfiguration().isExposePublisherStats(); + this.updatedAt = 0; + } @Override @@ -100,6 +112,7 @@ public synchronized void updateStats( tempMetricsCollection.clear(); bundleStats.clear(); brokerOperabilityMetrics.reset(); + brokerStats.reset(); // Json begin topicStatsStream.startObject(); @@ -169,6 +182,18 @@ public synchronized void updateStats( topicStatsStream.endObject(); }); + brokerStats.bundleCount += bundles.size(); + brokerStats.producerCount += nsStats.producerCount; + brokerStats.replicatorCount += nsStats.replicatorCount; + brokerStats.subsCount += nsStats.subsCount; + brokerStats.consumerCount += nsStats.consumerCount; + brokerStats.msgBacklog += nsStats.msgBacklog; + brokerStats.msgRateIn += nsStats.msgRateIn; + brokerStats.msgRateOut += nsStats.msgRateOut; + brokerStats.msgThroughputIn += nsStats.msgThroughputIn; + brokerStats.msgThroughputOut += nsStats.msgThroughputOut; + NamespaceStats.add(nsStats.addLatencyBucket, brokerStats.addLatencyBucket); + topicStatsStream.endObject(); // Update metricsCollection with namespace stats tempMetricsCollection.add(nsStats.add(namespaceName)); @@ -204,6 +229,7 @@ public synchronized void updateStats( } finally { bufferLock.writeLock().unlock(); } + updatedAt = System.currentTimeMillis(); } public NamespaceBundleStats invalidBundleStats(String bundleName) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerStats.java new file mode 100644 index 0000000000000..d0be71167002d --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerStats.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.stats; + +public class BrokerStats extends NamespaceStats { + + public int bundleCount; + public BrokerStats(int ratePeriodInSeconds) { + super(ratePeriodInSeconds); + } + + @Override + public void reset() { + super.reset(); + bundleCount = 0; + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index 717846184794a..81b41aa1687a7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -327,7 +327,7 @@ public void testGetMetrics() throws Exception { usage.setDirectMemory(directMemory); usage.setBandwidthIn(bandwidthIn); usage.setBandwidthOut(bandwidthOut); - loadData.update(usage, 1, 2, 3, 4, conf); + loadData.update(usage, 1, 2, 3, 4, 5, conf); brokerLoadMetrics.set(loadData.toMetrics(pulsar.getAdvertisedAddress())); } { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadDataTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadDataTest.java index de5975ba49be0..c85fa4ce9d2cd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadDataTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadDataTest.java @@ -59,7 +59,7 @@ public void testUpdateBySystemResourceUsage() { usage1.setDirectMemory(directMemory); usage1.setBandwidthIn(bandwidthIn); usage1.setBandwidthOut(bandwidthOut); - data.update(usage1, 1,2,3,4, conf); + data.update(usage1, 1, 2, 3, 4, 5, conf); assertEquals(data.getCpu(), cpu); assertEquals(data.getMemory(), memory); @@ -70,6 +70,7 @@ public void testUpdateBySystemResourceUsage() { assertEquals(data.getMsgThroughputOut(), 2.0); assertEquals(data.getMsgRateIn(), 3.0); assertEquals(data.getMsgRateOut(), 4.0); + assertEquals(data.getBundleCount(), 5); assertEquals(data.getMaxResourceUsage(), 0.04); // skips memory usage assertEquals(data.getWeightedMaxEMA(), 2); assertThat(data.getUpdatedAt(), greaterThanOrEqualTo(now)); @@ -86,7 +87,7 @@ public void testUpdateBySystemResourceUsage() { usage2.setDirectMemory(directMemory); usage2.setBandwidthIn(bandwidthIn); usage2.setBandwidthOut(bandwidthOut); - data.update(usage2, 5,6,7,8, conf); + data.update(usage2, 5, 6, 7, 8, 9, conf); assertEquals(data.getCpu(), cpu); assertEquals(data.getMemory(), memory); @@ -97,16 +98,19 @@ public void testUpdateBySystemResourceUsage() { assertEquals(data.getMsgThroughputOut(), 6.0); assertEquals(data.getMsgRateIn(), 7.0); assertEquals(data.getMsgRateOut(), 8.0); + assertEquals(data.getBundleCount(), 9); assertEquals(data.getMaxResourceUsage(), 3.0); assertEquals(data.getWeightedMaxEMA(), 1.875); assertThat(data.getUpdatedAt(), greaterThanOrEqualTo(now)); + assertEquals(data.getReportedAt(), 0l); assertEquals(data.toString(conf), "cpu= 300.00%, memory= 100.00%, directMemory= 2.00%, " + "bandwithIn= 3.00%, bandwithOut= 4.00%, " + "cpuWeight= 0.500000, memoryWeight= 0.500000, directMemoryWeight= 0.500000, " + "bandwithInResourceWeight= 0.500000, bandwithOutResourceWeight= 0.500000, " + "msgThroughputIn= 5.00, msgThroughputOut= 6.00, " - + "msgRateIn= 7.00, msgRateOut= 8.00," - + " maxResourceUsage= 300.00%, weightedMaxEMA= 187.50%, updatedAt= " + data.getUpdatedAt()); + + "msgRateIn= 7.00, msgRateOut= 8.00, bundleCount= 9, " + + "maxResourceUsage= 300.00%, weightedMaxEMA= 187.50%, " + + "updatedAt= " + data.getUpdatedAt() + ", reportedAt= " + data.getReportedAt()); } @Test @@ -133,7 +137,7 @@ public void testUpdateByBrokerLoadData() { usage1.setDirectMemory(directMemory); usage1.setBandwidthIn(bandwidthIn); usage1.setBandwidthOut(bandwidthOut); - other.update(usage1, 1,2,3,4, conf); + other.update(usage1, 1, 2, 3, 4, 5, conf); data.update(other); assertEquals(data, other); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/TopBundlesLoadDataTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/TopBundlesLoadDataTest.java deleted file mode 100644 index 06c232d321990..0000000000000 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/TopBundlesLoadDataTest.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.loadbalance.extensions.data; - -import static org.testng.Assert.assertEquals; - -import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; -import org.testng.annotations.Test; -import java.util.ArrayList; -import java.util.List; - -@Test(groups = "broker") -public class TopBundlesLoadDataTest { - - @Test - public void testTopBundlesLoadData() { - List bundleStats = new ArrayList<>(); - NamespaceBundleStats stats1 = new NamespaceBundleStats(); - stats1.msgRateIn = 100; - bundleStats.add(new TopBundlesLoadData.BundleLoadData("bundle-1", stats1)); - - NamespaceBundleStats stats2 = new NamespaceBundleStats(); - stats2.msgRateIn = 10000; - bundleStats.add(new TopBundlesLoadData.BundleLoadData("bundle-2", stats2)); - - NamespaceBundleStats stats3 = new NamespaceBundleStats(); - stats3.msgRateIn = 100000; - bundleStats.add(new TopBundlesLoadData.BundleLoadData("bundle-3", stats3)); - - NamespaceBundleStats stats4 = new NamespaceBundleStats(); - stats4.msgRateIn = 10; - bundleStats.add(new TopBundlesLoadData.BundleLoadData("bundle-4", stats4)); - - TopBundlesLoadData topBundlesLoadData = TopBundlesLoadData.of(bundleStats, 3); - var top0 = topBundlesLoadData.getTopBundlesLoadData().get(0); - var top1 = topBundlesLoadData.getTopBundlesLoadData().get(1); - var top2 = topBundlesLoadData.getTopBundlesLoadData().get(2); - - assertEquals(top0.bundleName(), "bundle-3"); - assertEquals(top1.bundleName(), "bundle-2"); - assertEquals(top2.bundleName(), "bundle-1"); - } -} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundlesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundlesTest.java new file mode 100644 index 0000000000000..d759dd016955a --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundlesTest.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.models; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class TopKBundlesTest { + + @Test + public void testTopBundlesLoadData() { + Map bundleStats = new HashMap<>(); + var topKBundles = new TopKBundles(); + NamespaceBundleStats stats1 = new NamespaceBundleStats(); + stats1.msgRateIn = 500; + bundleStats.put("bundle-1", stats1); + + NamespaceBundleStats stats2 = new NamespaceBundleStats(); + stats2.msgRateIn = 10000; + bundleStats.put("bundle-2", stats2); + + NamespaceBundleStats stats3 = new NamespaceBundleStats(); + stats3.msgRateIn = 100000; + bundleStats.put("bundle-3", stats3); + + NamespaceBundleStats stats4 = new NamespaceBundleStats(); + stats4.msgRateIn = 0; + bundleStats.put("bundle-4", stats4); + + topKBundles.update(bundleStats, 3); + var top0 = topKBundles.getLoadData().getTopBundlesLoadData().get(0); + var top1 = topKBundles.getLoadData().getTopBundlesLoadData().get(1); + var top2 = topKBundles.getLoadData().getTopBundlesLoadData().get(2); + + assertEquals(top0.bundleName(), "bundle-3"); + assertEquals(top1.bundleName(), "bundle-2"); + assertEquals(top2.bundleName(), "bundle-1"); + } + + @Test + public void testSystemNamespace() { + Map bundleStats = new HashMap<>(); + var topKBundles = new TopKBundles(); + NamespaceBundleStats stats1 = new NamespaceBundleStats(); + stats1.msgRateIn = 500; + bundleStats.put("pulsar/system/bundle-1", stats1); + + NamespaceBundleStats stats2 = new NamespaceBundleStats(); + stats2.msgRateIn = 10000; + bundleStats.put("pulsar/system/bundle-2", stats2); + + topKBundles.update(bundleStats, 2); + assertTrue(topKBundles.getLoadData().getTopBundlesLoadData().isEmpty()); + } + + + @Test + public void testPartitionSort() { + + Random rand = new Random(); + List> actual = new ArrayList<>(); + List> expected = new ArrayList<>(); + + for (int j = 0; j < 100; j++) { + Map map = new HashMap<>(); + int max = rand.nextInt(10) + 1; + for (int i = 0; i < max; i++) { + int val = rand.nextInt(max); + map.put("" + i, val); + } + actual.clear(); + expected.clear(); + for (var etr : map.entrySet()) { + actual.add(etr); + expected.add(etr); + } + int topk = rand.nextInt(max) + 1; + TopKBundles.partitionSort(actual, topk); + Collections.sort(expected, (a, b) -> b.getValue().compareTo(a.getValue())); + String errorMsg = null; + for (int i = 0; i < topk; i++) { + Integer l = (Integer) actual.get(i).getValue(); + Integer r = (Integer) expected.get(i).getValue(); + if (!l.equals(r)) { + errorMsg = String.format("Diff found at i=%d, %d != %d, actual:%s, expected:%s", + i, l, r, actual, expected); + } + assertNull(errorMsg); + } + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java new file mode 100644 index 0000000000000..3e4dc4cb1c07b --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.reporter; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.testng.Assert.assertEquals; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import org.apache.commons.lang.reflect.FieldUtils; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; +import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore; +import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; +import org.apache.pulsar.broker.service.BrokerService; +import org.apache.pulsar.broker.service.PulsarStats; +import org.apache.pulsar.broker.stats.BrokerStats; +import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; +import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class BrokerLoadDataReporterTest { + PulsarService pulsar; + LoadDataStore store; + BrokerService brokerService; + PulsarStats pulsarStats; + ServiceConfiguration config; + BrokerStats brokerStats; + SystemResourceUsage usage; + + @BeforeMethod + void setup() { + config = new ServiceConfiguration(); + pulsar = mock(PulsarService.class); + store = mock(LoadDataStore.class); + brokerService = mock(BrokerService.class); + pulsarStats = mock(PulsarStats.class); + doReturn(brokerService).when(pulsar).getBrokerService(); + doReturn(config).when(pulsar).getConfiguration(); + doReturn(Executors.newSingleThreadScheduledExecutor()).when(pulsar).getLoadManagerExecutor(); + doReturn(pulsarStats).when(brokerService).getPulsarStats(); + brokerStats = new BrokerStats(0); + brokerStats.bundleCount = 5; + brokerStats.msgRateIn = 3; + brokerStats.msgRateOut = 4; + brokerStats.msgThroughputIn = 1; + brokerStats.msgThroughputOut = 2; + doReturn(pulsarStats).when(brokerService).getPulsarStats(); + doReturn(brokerStats).when(pulsarStats).getBrokerStats(); + doReturn(CompletableFuture.completedFuture(null)).when(store).pushAsync(any(), any()); + + usage = new SystemResourceUsage(); + usage.setCpu(new ResourceUsage(1.0, 100.0)); + usage.setMemory(new ResourceUsage(800.0, 200.0)); + usage.setDirectMemory(new ResourceUsage(2.0, 100.0)); + usage.setBandwidthIn(new ResourceUsage(3.0, 100.0)); + usage.setBandwidthOut(new ResourceUsage(4.0, 100.0)); + } + + public void testGenerate() throws IllegalAccessException { + try (MockedStatic mockLoadManagerShared = Mockito.mockStatic(LoadManagerShared.class)) { + mockLoadManagerShared.when(() -> LoadManagerShared.getSystemResourceUsage(any())).thenReturn(usage); + doReturn(0l).when(pulsarStats).getUpdatedAt(); + var target = new BrokerLoadDataReporter(pulsar, "", store); + var expected = new BrokerLoadData(); + expected.update(usage, 1, 2, 3, 4, 5, config); + FieldUtils.writeDeclaredField(expected, "updatedAt", 0l, true); + var actual = target.generateLoadData(); + FieldUtils.writeDeclaredField(actual, "updatedAt", 0l, true); + assertEquals(actual, expected); + } + } + + public void testReport() throws IllegalAccessException { + try (MockedStatic mockLoadManagerShared = Mockito.mockStatic(LoadManagerShared.class)) { + mockLoadManagerShared.when(() -> LoadManagerShared.getSystemResourceUsage(any())).thenReturn(usage); + var target = new BrokerLoadDataReporter(pulsar, "broker-1", store); + var localData = (BrokerLoadData) FieldUtils.readDeclaredField(target, "localData", true); + localData.setReportedAt(System.currentTimeMillis()); + var lastData = (BrokerLoadData) FieldUtils.readDeclaredField(target, "lastData", true); + lastData.update(usage, 1, 2, 3, 4, 5, config); + target.reportAsync(false); + verify(store, times(0)).pushAsync(any(), any()); + + target.reportAsync(true); + verify(store, times(1)).pushAsync(eq("broker-1"), any()); + + target.reportAsync(false); + verify(store, times(1)).pushAsync(eq("broker-1"), any()); + + localData.setReportedAt(0l); + target.reportAsync(false); + verify(store, times(2)).pushAsync(eq("broker-1"), any()); + + lastData.update(usage, 10000, 2, 3, 4, 5, config); + target.reportAsync(false); + verify(store, times(3)).pushAsync(eq("broker-1"), any()); + } + } + +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java new file mode 100644 index 0000000000000..ce2d3d8c3ea93 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.reporter; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import org.apache.commons.lang.reflect.FieldUtils; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData; +import org.apache.pulsar.broker.loadbalance.extensions.models.TopKBundles; +import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore; +import org.apache.pulsar.broker.service.BrokerService; +import org.apache.pulsar.broker.service.PulsarStats; +import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class TopBundleLoadDataReporterTest { + PulsarService pulsar; + LoadDataStore store; + BrokerService brokerService; + PulsarStats pulsarStats; + Map bundleStats; + ServiceConfiguration config; + + @BeforeMethod + void setup() { + config = new ServiceConfiguration(); + pulsar = mock(PulsarService.class); + store = mock(LoadDataStore.class); + brokerService = mock(BrokerService.class); + pulsarStats = mock(PulsarStats.class); + doReturn(brokerService).when(pulsar).getBrokerService(); + doReturn(config).when(pulsar).getConfiguration(); + doReturn(pulsarStats).when(brokerService).getPulsarStats(); + doReturn(CompletableFuture.completedFuture(null)).when(store).pushAsync(any(), any()); + + bundleStats = new HashMap<>(); + NamespaceBundleStats stats1 = new NamespaceBundleStats(); + stats1.msgRateIn = 500; + bundleStats.put("bundle-1", stats1); + NamespaceBundleStats stats2 = new NamespaceBundleStats(); + stats2.msgRateIn = 10000; + bundleStats.put("bundle-2", stats2); + doReturn(bundleStats).when(brokerService).getBundleStats(); + } + + public void testZeroUpdatedAt() { + doReturn(0l).when(pulsarStats).getUpdatedAt(); + var target = new TopBundleLoadDataReporter(pulsar, "", store); + assertNull(target.generateLoadData()); + } + + public void testGenerateLoadData() throws IllegalAccessException { + doReturn(1l).when(pulsarStats).getUpdatedAt(); + config.setLoadBalancerBundleLoadReportPercentage(100); + var target = new TopBundleLoadDataReporter(pulsar, "", store); + var expected = new TopKBundles(); + expected.update(bundleStats, 2); + assertEquals(target.generateLoadData(), expected.getLoadData()); + + config.setLoadBalancerBundleLoadReportPercentage(50); + FieldUtils.writeDeclaredField(target, "lastBundleStatsUpdatedAt", 0l, true); + expected = new TopKBundles(); + expected.update(bundleStats, 1); + assertEquals(target.generateLoadData(), expected.getLoadData()); + + config.setLoadBalancerBundleLoadReportPercentage(1); + FieldUtils.writeDeclaredField(target, "lastBundleStatsUpdatedAt", 0l, true); + expected = new TopKBundles(); + expected.update(bundleStats, 1); + assertEquals(target.generateLoadData(), expected.getLoadData()); + + doReturn(new HashMap()).when(brokerService).getBundleStats(); + FieldUtils.writeDeclaredField(target, "lastBundleStatsUpdatedAt", 0l, true); + expected = new TopKBundles(); + assertEquals(target.generateLoadData(), expected.getLoadData()); + } + + + public void testReportForce() { + var target = new TopBundleLoadDataReporter(pulsar, "broker-1", store); + target.reportAsync(false); + verify(store, times(0)).pushAsync(any(), any()); + target.reportAsync(true); + verify(store, times(1)).pushAsync("broker-1", new TopBundlesLoadData()); + + } + + public void testReport(){ + var target = new TopBundleLoadDataReporter(pulsar, "broker-1", store); + doReturn(1l).when(pulsarStats).getUpdatedAt(); + var expected = new TopKBundles(); + expected.update(bundleStats, 1); + target.reportAsync(false); + verify(store, times(1)).pushAsync("broker-1", expected.getLoadData()); + } + +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java index a16cf2d8a67f1..bdf5f846267e6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java @@ -37,7 +37,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Random; @@ -52,6 +51,7 @@ import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData; +import org.apache.pulsar.broker.loadbalance.extensions.models.TopKBundles; import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision; import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore; @@ -82,11 +82,11 @@ public LoadManagerContext setupContext(){ brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx, 90)); var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); - topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("bundleA", 1, 1)); - topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("bundleB", 3, 1)); - topBundlesLoadDataStore.pushAsync("broker3", getTopBundlesLoad("bundleC", 4, 2)); - topBundlesLoadDataStore.pushAsync("broker4", getTopBundlesLoad("bundleD", 20, 60)); - topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("bundleE", 70, 20)); + topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("bundleA", 2000000, 1000000)); + topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("bundleB", 3000000, 1000000)); + topBundlesLoadDataStore.pushAsync("broker3", getTopBundlesLoad("bundleC", 4000000, 2000000)); + topBundlesLoadDataStore.pushAsync("broker4", getTopBundlesLoad("bundleD", 6000000, 2000000)); + topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("bundleE", 7000000, 2000000)); return ctx; } @@ -121,7 +121,7 @@ public BrokerLoadData getCpuLoad(LoadManagerContext ctx, int load) { usage1.setDirectMemory(directMemory); usage1.setBandwidthIn(bandwidthIn); usage1.setBandwidthOut(bandwidthOut); - loadData.update(usage1, 1,2,3,4, + loadData.update(usage1, 1,2,3,4,5, ctx.brokerConfiguration()); return loadData; } @@ -131,18 +131,18 @@ public TopBundlesLoadData getTopBundlesLoad(String bundlePrefix, int load1, int namespaceBundleStats1.msgThroughputOut = load1; var namespaceBundleStats2 = new NamespaceBundleStats(); namespaceBundleStats2.msgThroughputOut = load2; - var topLoadData = TopBundlesLoadData.of(List.of( - new TopBundlesLoadData.BundleLoadData(bundlePrefix + "-1", namespaceBundleStats1), - new TopBundlesLoadData.BundleLoadData(bundlePrefix + "-2", namespaceBundleStats2)), 2); - return topLoadData; + var topKBundles = new TopKBundles(); + topKBundles.update(Map.of(bundlePrefix + "-1", namespaceBundleStats1, + bundlePrefix + "-2", namespaceBundleStats2), 2); + return topKBundles.getLoadData(); } public TopBundlesLoadData getTopBundlesLoad(String bundlePrefix, int load1) { var namespaceBundleStats1 = new NamespaceBundleStats(); namespaceBundleStats1.msgThroughputOut = load1; - var topLoadData = TopBundlesLoadData.of(List.of( - new TopBundlesLoadData.BundleLoadData(bundlePrefix + "-1", namespaceBundleStats1)), 2); - return topLoadData; + var topKBundles = new TopKBundles(); + topKBundles.update(Map.of(bundlePrefix + "-1", namespaceBundleStats1), 2); + return topKBundles.getLoadData(); } public LoadManagerContext getContext(){ @@ -311,7 +311,7 @@ public void testRecentlyUnloadedBrokers() { unloads.put("broker5", new Unload("broker5", "bundleE-1", Optional.of("broker1"))); unloads.put("broker4", - new Unload("broker4", "bundleD-2", Optional.of("broker2"))); + new Unload("broker4", "bundleD-1", Optional.of("broker2"))); expected.setLabel(Success); expected.setReason(Overloaded); @@ -494,7 +494,7 @@ public void testMinBrokerWithZeroTraffic() throws IllegalAccessException { unloads.put("broker5", new Unload("broker5", "bundleE-1", Optional.of("broker1"))); unloads.put("broker4", - new Unload("broker4", "bundleD-2", Optional.of("broker2"))); + new Unload("broker4", "bundleD-1", Optional.of("broker2"))); expected.setLabel(Success); expected.setReason(Underloaded); expected.setLoadAvg(0.26400000000000007); @@ -515,7 +515,7 @@ public void testMaxNumberOfTransfersPerShedderCycle() { unloads.put("broker5", new Unload("broker5", "bundleE-1", Optional.of("broker1"))); unloads.put("broker4", - new Unload("broker4", "bundleD-2", Optional.of("broker2"))); + new Unload("broker4", "bundleD-1", Optional.of("broker2"))); expected.setLabel(Success); expected.setReason(Overloaded); expected.setLoadAvg(setupLoadAvg); @@ -528,7 +528,7 @@ public void testRemainingTopBundles() { TransferShedder transferShedder = new TransferShedder(); var ctx = setupContext(); var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); - topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("bundleE", 20, 20)); + topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("bundleE", 3000000, 2000000)); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new UnloadDecision(); @@ -536,7 +536,7 @@ public void testRemainingTopBundles() { unloads.put("broker5", new Unload("broker5", "bundleE-1", Optional.of("broker1"))); unloads.put("broker4", - new Unload("broker4", "bundleD-2", Optional.of("broker2"))); + new Unload("broker4", "bundleD-1", Optional.of("broker2"))); expected.setLabel(Success); expected.setReason(Overloaded); expected.setLoadAvg(setupLoadAvg); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeightTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeightTest.java index 2856dde892a8f..da0866c689746 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeightTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeightTest.java @@ -168,7 +168,7 @@ public void testNoLoadDataBrokers() { private BrokerLoadData createBrokerData(LoadManagerContext ctx, double usage, double limit) { var brokerLoadData = new BrokerLoadData(); SystemResourceUsage usages = createUsage(usage, limit); - brokerLoadData.update(usages, 1, 1, 1, 1, + brokerLoadData.update(usages, 1, 1, 1, 1, 1, ctx.brokerConfiguration()); return brokerLoadData; } @@ -185,7 +185,7 @@ private SystemResourceUsage createUsage(double usage, double limit) { private void updateLoad(LoadManagerContext ctx, String broker, double usage) { ctx.brokerLoadDataStore().get(broker).get().update(createUsage(usage, 100.0), - 1, 1, 1, 1, ctx.brokerConfiguration()); + 1, 1, 1, 1, 1, ctx.brokerConfiguration()); } public static LoadManagerContext getContext() { From 950ff441da28e144bdfb71c317a9bc339d4f05b7 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Mon, 13 Feb 2023 19:30:36 +0800 Subject: [PATCH 009/174] [improve][broker] PIP-192: Write the child ownership to `ServiceUnitStateChannel` instead of ZK when handling bundle split (#18858) --- .../broker/admin/impl/NamespacesBase.java | 16 +-- .../channel/ServiceUnitStateChannelImpl.java | 133 +++++++++++++++--- .../broker/namespace/NamespaceService.java | 74 ++++++++-- .../channel/ServiceUnitStateChannelTest.java | 53 ++++++- 4 files changed, 221 insertions(+), 55 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index d5cf6a3e74d0e..5446060ac6502 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -1028,7 +1028,9 @@ protected CompletableFuture internalSplitNamespaceBundleAsync(String bundl validateNamespaceBundleOwnershipAsync(namespaceName, policies.bundles, bundleRange, authoritative, false)) .thenCompose(nsBundle -> pulsar().getNamespaceService().splitAndOwnBundle(nsBundle, unload, - getNamespaceBundleSplitAlgorithmByName(splitAlgorithmName), splitBoundaries)); + pulsar().getNamespaceService() + .getNamespaceBundleSplitAlgorithmByName(splitAlgorithmName), + splitBoundaries)); }); } @@ -1109,18 +1111,6 @@ private CompletableFuture findHotBundleAsync(NamespaceName name .getBundleWithHighestThroughputAsync(namespaceName); } - private NamespaceBundleSplitAlgorithm getNamespaceBundleSplitAlgorithmByName(String algorithmName) { - NamespaceBundleSplitAlgorithm algorithm = NamespaceBundleSplitAlgorithm.of(algorithmName); - if (algorithm == null) { - algorithm = NamespaceBundleSplitAlgorithm.of( - pulsar().getConfig().getDefaultNamespaceBundleSplitAlgorithm()); - } - if (algorithm == null) { - algorithm = NamespaceBundleSplitAlgorithm.RANGE_EQUALLY_DIVIDE_ALGO; - } - return algorithm; - } - protected void internalSetPublishRate(PublishRate maxPublishMessageRate) { validateSuperUserAccess(); log.info("[{}] Set namespace publish-rate {}/{}", clientAppId(), namespaceName, maxPublishMessageRate); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index d5bcd3e1436cb..d10138bda6805 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.channel; +import static java.lang.String.format; +import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Assigned; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Free; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Owned; @@ -35,7 +37,9 @@ import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.MetadataState.Unstable; import static org.apache.pulsar.metadata.api.extended.SessionEvent.SessionLost; import static org.apache.pulsar.metadata.api.extended.SessionEvent.SessionReestablished; +import com.google.common.annotations.VisibleForTesting; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -48,6 +52,7 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import lombok.AllArgsConstructor; import lombok.Getter; @@ -60,18 +65,23 @@ import org.apache.pulsar.broker.loadbalance.extensions.models.Split; import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; +import org.apache.pulsar.broker.namespace.NamespaceService; +import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.TableView; import org.apache.pulsar.common.naming.NamespaceBundle; -import org.apache.pulsar.common.naming.NamespaceBundleSplitAlgorithm; +import org.apache.pulsar.common.naming.NamespaceBundleFactory; +import org.apache.pulsar.common.naming.NamespaceBundles; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.stats.Metrics; +import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; +import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.api.NotificationType; import org.apache.pulsar.metadata.api.coordination.LeaderElectionState; import org.apache.pulsar.metadata.api.extended.SessionEvent; @@ -523,8 +533,7 @@ private void handleReleaseEvent(String serviceUnit, ServiceUnitStateData data) { private void handleSplitEvent(String serviceUnit, ServiceUnitStateData data) { if (isTargetBroker(data.broker())) { - splitServiceUnit(serviceUnit) - .thenCompose(__ -> tombstoneAsync(serviceUnit)) + splitServiceUnit(serviceUnit, data) .whenComplete((__, e) -> log(e, serviceUnit, data, null)); } } @@ -625,25 +634,107 @@ private CompletableFuture closeServiceUnit(String serviceUnit) { }); } - private CompletableFuture splitServiceUnit(String serviceUnit) { - // TODO: after the split we need to write the child ownerships to BSC instead of ZK. + private CompletableFuture splitServiceUnit(String serviceUnit, ServiceUnitStateData data) { + // Write the child ownerships to BSC. long startTime = System.nanoTime(); - return pulsar.getNamespaceService() - .splitAndOwnBundle(getNamespaceBundle(serviceUnit), - false, - NamespaceBundleSplitAlgorithm.of(pulsar.getConfig().getDefaultNamespaceBundleSplitAlgorithm()), - null) - .whenComplete((__, ex) -> { - double splitBundleTime = TimeUnit.NANOSECONDS - .toMillis((System.nanoTime() - startTime)); - if (ex == null) { - log.info("Successfully split {} namespace-bundle in {} ms", - serviceUnit, splitBundleTime); - } else { - log.error("Failed to split {} namespace-bundle in {} ms", - serviceUnit, splitBundleTime, ex); - } - }); + NamespaceService namespaceService = pulsar.getNamespaceService(); + NamespaceBundleFactory bundleFactory = namespaceService.getNamespaceBundleFactory(); + NamespaceBundle bundle = getNamespaceBundle(serviceUnit); + CompletableFuture completionFuture = new CompletableFuture<>(); + final AtomicInteger counter = new AtomicInteger(0); + this.splitServiceUnitOnceAndRetry(namespaceService, bundleFactory, bundle, serviceUnit, data, + counter, startTime, completionFuture); + return completionFuture; + } + + @VisibleForTesting + protected void splitServiceUnitOnceAndRetry(NamespaceService namespaceService, + NamespaceBundleFactory bundleFactory, + NamespaceBundle bundle, + String serviceUnit, + ServiceUnitStateData data, + AtomicInteger counter, + long startTime, + CompletableFuture completionFuture) { + CompletableFuture> updateFuture = new CompletableFuture<>(); + + pulsar.getNamespaceService().getSplitBoundary(bundle, null).thenAccept(splitBundlesPair -> { + // Split and updateNamespaceBundles. Update may fail because of concurrent write to Zookeeper. + if (splitBundlesPair == null) { + String msg = format("Bundle %s not found under namespace", serviceUnit); + updateFuture.completeExceptionally(new BrokerServiceException.ServiceUnitNotReadyException(msg)); + return; + } + ServiceUnitStateData next = new ServiceUnitStateData(Owned, data.broker()); + NamespaceBundles targetNsBundle = splitBundlesPair.getLeft(); + List splitBundles = Collections.unmodifiableList(splitBundlesPair.getRight()); + List successPublishedBundles = + Collections.synchronizedList(new ArrayList<>(splitBundles.size())); + List> futures = new ArrayList<>(splitBundles.size()); + for (NamespaceBundle sBundle : splitBundles) { + futures.add(pubAsync(sBundle.toString(), next).thenAccept(__ -> successPublishedBundles.add(sBundle))); + } + NamespaceName nsname = bundle.getNamespaceObject(); + FutureUtil.waitForAll(futures) + .thenCompose(__ -> namespaceService.updateNamespaceBundles(nsname, targetNsBundle)) + .thenCompose(__ -> namespaceService.updateNamespaceBundlesForPolicies(nsname, targetNsBundle)) + .thenRun(() -> { + bundleFactory.invalidateBundleCache(bundle.getNamespaceObject()); + updateFuture.complete(splitBundles); + }).exceptionally(e -> { + // Clean the new bundle when has exception. + List> futureList = new ArrayList<>(); + for (NamespaceBundle sBundle : successPublishedBundles) { + futureList.add(tombstoneAsync(sBundle.toString()).thenAccept(__ -> {})); + } + FutureUtil.waitForAll(futureList) + .whenComplete((__, ex) -> { + if (ex != null) { + log.warn("Clean new bundles failed,", ex); + } + updateFuture.completeExceptionally(e); + }); + return null; + }); + }).exceptionally(e -> { + updateFuture.completeExceptionally(e); + return null; + }); + + updateFuture.thenAccept(r -> { + // Free the old bundle + tombstoneAsync(serviceUnit).thenRun(() -> { + // Update bundled_topic cache for load-report-generation + pulsar.getBrokerService().refreshTopicToStatsMaps(bundle); + // TODO: Update the load data immediately if needed. + completionFuture.complete(null); + double splitBundleTime = TimeUnit.NANOSECONDS.toMillis((System.nanoTime() - startTime)); + log.info("Successfully split {} parent namespace-bundle to {} in {} ms", serviceUnit, r, + splitBundleTime); + }).exceptionally(e -> { + double splitBundleTime = TimeUnit.NANOSECONDS.toMillis((System.nanoTime() - startTime)); + String msg = format("Failed to free bundle %s in %s ms, under namespace [%s] with error %s", + bundle.getNamespaceObject().toString(), splitBundleTime, bundle, e.getMessage()); + completionFuture.completeExceptionally(new BrokerServiceException.ServiceUnitNotReadyException(msg)); + return null; + }); + }).exceptionally(ex -> { + // Retry several times on BadVersion + Throwable throwable = FutureUtil.unwrapCompletionException(ex); + if ((throwable instanceof MetadataStoreException.BadVersionException) + && (counter.incrementAndGet() < NamespaceService.BUNDLE_SPLIT_RETRY_LIMIT)) { + pulsar.getExecutor().schedule(() -> splitServiceUnitOnceAndRetry(namespaceService, bundleFactory, + bundle, serviceUnit, data, counter, startTime, completionFuture), 100, MILLISECONDS); + } else if (throwable instanceof IllegalArgumentException) { + completionFuture.completeExceptionally(throwable); + } else { + // Retry enough, or meet other exception + String msg = format("Bundle: %s not success update nsBundles, counter %d, reason %s", + bundle.toString(), counter.get(), throwable.getMessage()); + completionFuture.completeExceptionally(new BrokerServiceException.ServiceUnitNotReadyException(msg)); + } + return null; + }); } public void handleMetadataSessionEvent(SessionEvent e) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index abbabcd3b00a1..245c3f896af1f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -124,7 +124,7 @@ public class NamespaceService implements AutoCloseable { private final NamespaceBundleFactory bundleFactory; private final String host; - private static final int BUNDLE_SPLIT_RETRY_LIMIT = 7; + public static final int BUNDLE_SPLIT_RETRY_LIMIT = 7; public static final String SLA_NAMESPACE_PROPERTY = "sla-monitor"; public static final Pattern HEARTBEAT_NAMESPACE_PATTERN = Pattern.compile("pulsar/[^/]+/([^:]+:\\d+)"); public static final Pattern HEARTBEAT_NAMESPACE_PATTERN_V2 = Pattern.compile("pulsar/([^:]+:\\d+)"); @@ -828,18 +828,7 @@ void splitAndOwnBundleOnceAndRetry(NamespaceBundle bundle, CompletableFuture completionFuture, NamespaceBundleSplitAlgorithm splitAlgorithm, List boundaries) { - BundleSplitOption bundleSplitOption; - if (config.getDefaultNamespaceBundleSplitAlgorithm() - .equals(NamespaceBundleSplitAlgorithm.FLOW_OR_QPS_EQUALLY_DIVIDE)) { - Map topicStatsMap = pulsar.getBrokerService().getTopicStats(bundle); - bundleSplitOption = new FlowOrQpsEquallyDivideBundleSplitOption(this, bundle, boundaries, - topicStatsMap, - config.getLoadBalancerNamespaceBundleMaxMsgRate(), - config.getLoadBalancerNamespaceBundleMaxBandwidthMbytes(), - config.getFlowOrQpsDifferenceThresholdPercentage()); - } else { - bundleSplitOption = new BundleSplitOption(this, bundle, boundaries); - } + BundleSplitOption bundleSplitOption = getBundleSplitOption(bundle, boundaries, config); splitAlgorithm.getSplitBoundary(bundleSplitOption).whenComplete((splitBoundaries, ex) -> { CompletableFuture> updateFuture = new CompletableFuture<>(); @@ -957,6 +946,61 @@ void splitAndOwnBundleOnceAndRetry(NamespaceBundle bundle, }); } + /** + * Get the split boundary's. + * + * @param bundle The bundle to split. + * @param boundaries The specified positions, + * use for {@link org.apache.pulsar.common.naming.SpecifiedPositionsBundleSplitAlgorithm}. + * @return A pair, left is target namespace bundle, right is split bundles. + */ + public CompletableFuture>> getSplitBoundary( + NamespaceBundle bundle, List boundaries) { + BundleSplitOption bundleSplitOption = getBundleSplitOption(bundle, boundaries, config); + NamespaceBundleSplitAlgorithm nsBundleSplitAlgorithm = + getNamespaceBundleSplitAlgorithmByName(config.getDefaultNamespaceBundleSplitAlgorithm()); + CompletableFuture> splitBoundary = + nsBundleSplitAlgorithm.getSplitBoundary(bundleSplitOption); + return splitBoundary.thenCompose(splitBoundaries -> { + if (splitBoundaries == null || splitBoundaries.size() == 0) { + LOG.info("[{}] No valid boundary found in {} to split bundle {}", + bundle.getNamespaceObject().toString(), boundaries, bundle.getBundleRange()); + return CompletableFuture.completedFuture(null); + } + return pulsar.getNamespaceService().getNamespaceBundleFactory() + .splitBundles(bundle, splitBoundaries.size() + 1, splitBoundaries); + }); + } + + private BundleSplitOption getBundleSplitOption(NamespaceBundle bundle, + List boundaries, + ServiceConfiguration config) { + BundleSplitOption bundleSplitOption; + if (config.getDefaultNamespaceBundleSplitAlgorithm() + .equals(NamespaceBundleSplitAlgorithm.FLOW_OR_QPS_EQUALLY_DIVIDE)) { + Map topicStatsMap = pulsar.getBrokerService().getTopicStats(bundle); + bundleSplitOption = new FlowOrQpsEquallyDivideBundleSplitOption(this, bundle, boundaries, + topicStatsMap, + config.getLoadBalancerNamespaceBundleMaxMsgRate(), + config.getLoadBalancerNamespaceBundleMaxBandwidthMbytes(), + config.getFlowOrQpsDifferenceThresholdPercentage()); + } else { + bundleSplitOption = new BundleSplitOption(this, bundle, boundaries); + } + return bundleSplitOption; + } + + public NamespaceBundleSplitAlgorithm getNamespaceBundleSplitAlgorithmByName(String algorithmName) { + NamespaceBundleSplitAlgorithm algorithm = NamespaceBundleSplitAlgorithm.of(algorithmName); + if (algorithm == null) { + algorithm = NamespaceBundleSplitAlgorithm.of(pulsar.getConfig().getDefaultNamespaceBundleSplitAlgorithm()); + } + if (algorithm == null) { + algorithm = NamespaceBundleSplitAlgorithm.RANGE_EQUALLY_DIVIDE_ALGO; + } + return algorithm; + } + /** * Update new bundle-range to admin/policies/namespace. * Update may fail because of concurrent write to Zookeeper. @@ -965,7 +1009,7 @@ void splitAndOwnBundleOnceAndRetry(NamespaceBundle bundle, * @param nsBundles * @throws Exception */ - private CompletableFuture updateNamespaceBundlesForPolicies(NamespaceName nsname, + public CompletableFuture updateNamespaceBundlesForPolicies(NamespaceName nsname, NamespaceBundles nsBundles) { Objects.requireNonNull(nsname); Objects.requireNonNull(nsBundles); @@ -994,7 +1038,7 @@ private CompletableFuture updateNamespaceBundlesForPolicies(NamespaceName * @param nsBundles * @throws Exception */ - private CompletableFuture updateNamespaceBundles(NamespaceName nsname, NamespaceBundles nsBundles) { + public CompletableFuture updateNamespaceBundles(NamespaceName nsname, NamespaceBundles nsBundles) { Objects.requireNonNull(nsname); Objects.requireNonNull(nsBundles); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java index 660999365c428..327afa3cb8891 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java @@ -38,9 +38,12 @@ import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -61,6 +64,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.pulsar.broker.PulsarServerException; @@ -69,12 +73,14 @@ import org.apache.pulsar.broker.loadbalance.LeaderElectionService; import org.apache.pulsar.broker.loadbalance.extensions.models.Split; import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; +import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.TypedMessageBuilder; import org.apache.pulsar.client.impl.TableViewImpl; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; +import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.api.NotificationType; import org.apache.pulsar.metadata.api.extended.SessionEvent; import org.awaitility.Awaitility; @@ -113,9 +119,9 @@ protected void setup() throws Exception { pulsar1 = pulsar; additionalPulsarTestContext = createAdditionalPulsarTestContext(getDefaultConf()); pulsar2 = additionalPulsarTestContext.getPulsarService(); - channel1 = new ServiceUnitStateChannelImpl(pulsar1); + channel1 = spy(new ServiceUnitStateChannelImpl(pulsar1)); channel1.start(); - channel2 = new ServiceUnitStateChannelImpl(pulsar2); + channel2 = spy(new ServiceUnitStateChannelImpl(pulsar2)); channel2.start(); lookupServiceAddress1 = (String) FieldUtils.readDeclaredField(channel1, "lookupServiceAddress", true); @@ -480,7 +486,7 @@ public void unloadTestWhenDestBrokerFails() } @Test(priority = 6) - public void splitTest() throws Exception { + public void splitAndRetryTest() throws Exception { channel1.publishAssignEventAsync(bundle, lookupServiceAddress1); waitUntilNewOwner(channel1, bundle, lookupServiceAddress1); waitUntilNewOwner(channel2, bundle, lookupServiceAddress1); @@ -490,17 +496,52 @@ public void splitTest() throws Exception { assertEquals(ownerAddr2, Optional.of(lookupServiceAddress1)); assertTrue(ownerAddr1.isPresent()); + NamespaceService namespaceService = spy(pulsar1.getNamespaceService()); + CompletableFuture future = new CompletableFuture<>(); + int badVersionExceptionCount = 3; + AtomicInteger count = new AtomicInteger(badVersionExceptionCount); + future.completeExceptionally(new MetadataStoreException.BadVersionException("BadVersion")); + doAnswer(invocationOnMock -> { + if (count.decrementAndGet() > 0) { + return future; + } + // Call the real method + reset(namespaceService); + return future; + }).when(namespaceService).updateNamespaceBundles(any(), any()); + doReturn(namespaceService).when(pulsar1).getNamespaceService(); + Split split = new Split(bundle, ownerAddr1.get(), new HashMap<>()); channel1.publishSplitEventAsync(split); waitUntilNewOwner(channel1, bundle, null); waitUntilNewOwner(channel2, bundle, null); - // TODO: assert child bundle ownerships in the channels. - validateHandlerCounters(channel1, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0); - validateHandlerCounters(channel2, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0); + validateHandlerCounters(channel1, 1, 0, 9, 0, 0, 0, 1, 0, 7, 0); + validateHandlerCounters(channel2, 1, 0, 9, 0, 0, 0, 1, 0, 7, 0); validateEventCounters(channel1, 1, 0, 1, 0, 0, 0); validateEventCounters(channel2, 0, 0, 0, 0, 0, 0); + // Verify the retry count + verify(((ServiceUnitStateChannelImpl) channel1), times(badVersionExceptionCount + 1)) + .splitServiceUnitOnceAndRetry(any(), any(), any(), any(), any(), any(), anyLong(), any()); + + // Assert child bundle ownerships in the channels. + String childBundle1 = "public/default/0x7fffffff_0xffffffff"; + String childBundle2 = "public/default/0x00000000_0x7fffffff"; + + waitUntilNewOwner(channel1, childBundle1, lookupServiceAddress1); + waitUntilNewOwner(channel1, childBundle2, lookupServiceAddress1); + waitUntilNewOwner(channel2, childBundle1, lookupServiceAddress1); + waitUntilNewOwner(channel2, childBundle2, lookupServiceAddress1); + assertEquals(Optional.of(lookupServiceAddress1), channel1.getOwnerAsync(childBundle1).get()); + assertEquals(Optional.of(lookupServiceAddress1), channel1.getOwnerAsync(childBundle2).get()); + assertEquals(Optional.of(lookupServiceAddress1), channel2.getOwnerAsync(childBundle1).get()); + assertEquals(Optional.of(lookupServiceAddress1), channel2.getOwnerAsync(childBundle2).get()); + + cleanTableView(channel1, childBundle1); + cleanTableView(channel2, childBundle1); + cleanTableView(channel1, childBundle2); + cleanTableView(channel2, childBundle2); } @Test(priority = 7) From af1b6e16ad9ffc0f5fad532e71c25e3a33e389c5 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Tue, 14 Feb 2023 10:03:40 +0800 Subject: [PATCH 010/174] [improve][broker] PIP-192 Added namespace unload scheduler (#19477) --- .../extensions/ExtensibleLoadManagerImpl.java | 9 +- .../extensions/scheduler/UnloadScheduler.java | 180 ++++++++++++++++++ .../ExtensibleLoadManagerImplTest.java | 3 + .../scheduler/UnloadSchedulerTest.java | 171 +++++++++++++++++ 4 files changed, 362 insertions(+), 1 deletion(-) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index fe28d67227e8a..59c6674676101 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -49,6 +49,8 @@ import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision; import org.apache.pulsar.broker.loadbalance.extensions.reporter.BrokerLoadDataReporter; import org.apache.pulsar.broker.loadbalance.extensions.reporter.TopBundleLoadDataReporter; +import org.apache.pulsar.broker.loadbalance.extensions.scheduler.LoadManagerScheduler; +import org.apache.pulsar.broker.loadbalance.extensions.scheduler.UnloadScheduler; import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore; import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStoreException; import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStoreFactory; @@ -86,6 +88,8 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager { private LoadDataStore brokerLoadDataStore; private LoadDataStore topBundlesLoadDataStore; + private LoadManagerScheduler unloadScheduler; + @Getter private LoadManagerContext context; @@ -194,7 +198,9 @@ public void start() throws PulsarServerException { interval, interval, TimeUnit.MILLISECONDS); - // TODO: Start unload scheduler and bundle split scheduler + // TODO: Start bundle split scheduler. + this.unloadScheduler = new UnloadScheduler(pulsar.getLoadManagerExecutor(), context, serviceUnitStateChannel); + this.unloadScheduler.start(); this.started = true; } @@ -319,6 +325,7 @@ public void close() throws PulsarServerException { this.brokerLoadDataStore.close(); this.topBundlesLoadDataStore.close(); + this.unloadScheduler.close(); } catch (IOException ex) { throw new PulsarServerException(ex); } finally { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java new file mode 100644 index 0000000000000..5cdbd3027104d --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.scheduler; + +import com.google.common.annotations.VisibleForTesting; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel; +import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision; +import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.common.util.Reflections; + +@Slf4j +public class UnloadScheduler implements LoadManagerScheduler { + + private final NamespaceUnloadStrategy namespaceUnloadStrategy; + + private final ScheduledExecutorService loadManagerExecutor; + + private final LoadManagerContext context; + + private final ServiceUnitStateChannel channel; + + private final ServiceConfiguration conf; + + private volatile ScheduledFuture task; + + private final Map recentlyUnloadedBundles; + + private final Map recentlyUnloadedBrokers; + + private volatile CompletableFuture currentRunningFuture = null; + + public UnloadScheduler(ScheduledExecutorService loadManagerExecutor, + LoadManagerContext context, + ServiceUnitStateChannel channel) { + this(loadManagerExecutor, context, channel, createNamespaceUnloadStrategy(context.brokerConfiguration())); + } + + @VisibleForTesting + protected UnloadScheduler(ScheduledExecutorService loadManagerExecutor, + LoadManagerContext context, + ServiceUnitStateChannel channel, + NamespaceUnloadStrategy strategy) { + this.namespaceUnloadStrategy = strategy; + this.recentlyUnloadedBundles = new HashMap<>(); + this.recentlyUnloadedBrokers = new HashMap<>(); + this.loadManagerExecutor = loadManagerExecutor; + this.context = context; + this.conf = context.brokerConfiguration(); + this.channel = channel; + } + + @Override + public synchronized void execute() { + boolean debugMode = conf.isLoadBalancerDebugModeEnabled() || log.isDebugEnabled(); + if (debugMode) { + log.info("Load balancer enabled: {}, Shedding enabled: {}.", + conf.isLoadBalancerEnabled(), conf.isLoadBalancerSheddingEnabled()); + } + if (!isLoadBalancerSheddingEnabled()) { + if (debugMode) { + log.info("The load balancer or load balancer shedding already disabled. Skipping."); + } + return; + } + if (currentRunningFuture != null && !currentRunningFuture.isDone()) { + if (debugMode) { + log.info("Auto namespace unload is running. Skipping."); + } + return; + } + // Remove bundles who have been unloaded for longer than the grace period from the recently unloaded map. + final long timeout = System.currentTimeMillis() + - TimeUnit.MINUTES.toMillis(conf.getLoadBalancerSheddingGracePeriodMinutes()); + recentlyUnloadedBundles.keySet().removeIf(e -> recentlyUnloadedBundles.get(e) < timeout); + + this.currentRunningFuture = channel.isChannelOwnerAsync().thenCompose(isChannelOwner -> { + if (!isChannelOwner) { + if (debugMode) { + log.info("Current broker is not channel owner. Skipping."); + } + return CompletableFuture.completedFuture(null); + } + return context.brokerRegistry().getAvailableBrokersAsync().thenCompose(availableBrokers -> { + if (debugMode) { + log.info("Available brokers: {}", availableBrokers); + } + if (availableBrokers.size() <= 1) { + log.info("Only 1 broker available: no load shedding will be performed. Skipping."); + return CompletableFuture.completedFuture(null); + } + final UnloadDecision unloadDecision = namespaceUnloadStrategy + .findBundlesForUnloading(context, recentlyUnloadedBundles, recentlyUnloadedBrokers); + if (debugMode) { + log.info("[{}] Unload decision result: {}", + namespaceUnloadStrategy.getClass().getSimpleName(), unloadDecision.toString()); + } + if (unloadDecision.getUnloads().isEmpty()) { + if (debugMode) { + log.info("[{}] Unload decision unloads is empty. Skipping.", + namespaceUnloadStrategy.getClass().getSimpleName()); + } + return CompletableFuture.completedFuture(null); + } + List> futures = new ArrayList<>(); + unloadDecision.getUnloads().forEach((broker, unload) -> { + log.info("[{}] Unloading bundle: {}", namespaceUnloadStrategy.getClass().getSimpleName(), unload); + futures.add(channel.publishUnloadEventAsync(unload).thenAccept(__ -> { + recentlyUnloadedBundles.put(unload.serviceUnit(), System.currentTimeMillis()); + recentlyUnloadedBrokers.put(unload.sourceBroker(), System.currentTimeMillis()); + })); + }); + return FutureUtil.waitForAll(futures).exceptionally(ex -> { + log.error("[{}] Namespace unload has exception.", + namespaceUnloadStrategy.getClass().getSimpleName(), ex); + return null; + }); + }); + }); + } + + @Override + public void start() { + long loadSheddingInterval = TimeUnit.MINUTES + .toMillis(conf.getLoadBalancerSheddingIntervalMinutes()); + this.task = loadManagerExecutor.scheduleAtFixedRate( + this::execute, loadSheddingInterval, loadSheddingInterval, TimeUnit.MILLISECONDS); + } + + @Override + public void close() { + if (this.task != null) { + this.task.cancel(false); + } + this.recentlyUnloadedBundles.clear(); + this.recentlyUnloadedBrokers.clear(); + } + + private static NamespaceUnloadStrategy createNamespaceUnloadStrategy(ServiceConfiguration conf) { + try { + return Reflections.createInstance(conf.getLoadBalancerLoadSheddingStrategy(), NamespaceUnloadStrategy.class, + Thread.currentThread().getContextClassLoader()); + } catch (Exception e) { + log.error("Error when trying to create namespace unload strategy: {}", + conf.getLoadBalancerLoadPlacementStrategy(), e); + } + log.error("create namespace unload strategy failed. using TransferShedder instead."); + return new TransferShedder(); + } + + private boolean isLoadBalancerSheddingEnabled() { + return conf.isLoadBalancerEnabled() && conf.isLoadBalancerSheddingEnabled(); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index 81b41aa1687a7..1ef4f660e4af3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -99,6 +99,7 @@ import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage; +import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -125,6 +126,7 @@ public class ExtensibleLoadManagerImplTest extends MockedPulsarServiceBaseTest { private ServiceUnitStateChannelImpl channel2; @BeforeClass + @Override public void setup() throws Exception { conf.setAllowAutoTopicCreation(true); conf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); @@ -186,6 +188,7 @@ protected void createNamespaceIfNotExists(PulsarResources resources, } @Override + @AfterClass protected void cleanup() throws Exception { pulsar1 = null; pulsar2.close(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java new file mode 100644 index 0000000000000..cda5f81d81b93 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.scheduler; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import com.google.common.collect.Lists; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.extensions.BrokerRegistry; +import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel; +import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; +import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision; +import org.apache.pulsar.client.util.ExecutorProvider; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +@Test(groups = "broker") +public class UnloadSchedulerTest { + + private ScheduledExecutorService loadManagerExecutor; + + public LoadManagerContext setupContext(){ + var ctx = getContext(); + ctx.brokerConfiguration().setLoadBalancerDebugModeEnabled(true); + return ctx; + } + + @BeforeMethod + public void setUp() { + this.loadManagerExecutor = Executors + .newSingleThreadScheduledExecutor(new ExecutorProvider.ExtendedThreadFactory("pulsar-load-manager")); + } + + @AfterMethod + public void tearDown() { + this.loadManagerExecutor.shutdown(); + } + + @Test(timeOut = 30 * 1000) + public void testExecuteSuccess() { + LoadManagerContext context = setupContext(); + BrokerRegistry registry = context.brokerRegistry(); + ServiceUnitStateChannel channel = mock(ServiceUnitStateChannel.class); + NamespaceUnloadStrategy unloadStrategy = mock(NamespaceUnloadStrategy.class); + doReturn(CompletableFuture.completedFuture(true)).when(channel).isChannelOwnerAsync(); + doReturn(CompletableFuture.completedFuture(Lists.newArrayList("broker-1", "broker-2"))) + .when(registry).getAvailableBrokersAsync(); + doReturn(CompletableFuture.completedFuture(null)).when(channel).publishUnloadEventAsync(any()); + UnloadDecision decision = new UnloadDecision(); + Unload unload = new Unload("broker-1", "bundle-1"); + decision.getUnloads().put("broker-1", unload); + doReturn(decision).when(unloadStrategy).findBundlesForUnloading(any(), any(), any()); + + UnloadScheduler scheduler = new UnloadScheduler(loadManagerExecutor, context, channel, unloadStrategy); + + scheduler.execute(); + + verify(channel, times(1)).publishUnloadEventAsync(eq(unload)); + + // Test empty unload. + UnloadDecision emptyUnload = new UnloadDecision(); + doReturn(emptyUnload).when(unloadStrategy).findBundlesForUnloading(any(), any(), any()); + + scheduler.execute(); + + verify(channel, times(1)).publishUnloadEventAsync(eq(unload)); + } + + @Test(timeOut = 30 * 1000) + public void testExecuteMoreThenOnceWhenFirstNotDone() throws InterruptedException { + LoadManagerContext context = setupContext(); + BrokerRegistry registry = context.brokerRegistry(); + ServiceUnitStateChannel channel = mock(ServiceUnitStateChannel.class); + NamespaceUnloadStrategy unloadStrategy = mock(NamespaceUnloadStrategy.class); + doReturn(CompletableFuture.completedFuture(true)).when(channel).isChannelOwnerAsync(); + doAnswer(__ -> CompletableFuture.supplyAsync(() -> { + try { + // Delay 5 seconds to finish. + TimeUnit.SECONDS.sleep(5); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return Lists.newArrayList("broker-1", "broker-2"); + }, Executors.newFixedThreadPool(1))).when(registry).getAvailableBrokersAsync(); + UnloadScheduler scheduler = new UnloadScheduler(loadManagerExecutor, context, channel, unloadStrategy); + + ExecutorService executorService = Executors.newFixedThreadPool(10); + CountDownLatch latch = new CountDownLatch(10); + for (int i = 0; i < 10; i++) { + executorService.execute(() -> { + scheduler.execute(); + latch.countDown(); + }); + } + latch.await(); + + verify(registry, times(1)).getAvailableBrokersAsync(); + } + + @Test(timeOut = 30 * 1000) + public void testDisableLoadBalancer() { + LoadManagerContext context = setupContext(); + context.brokerConfiguration().setLoadBalancerEnabled(false); + ServiceUnitStateChannel channel = mock(ServiceUnitStateChannel.class); + NamespaceUnloadStrategy unloadStrategy = mock(NamespaceUnloadStrategy.class); + UnloadScheduler scheduler = new UnloadScheduler(loadManagerExecutor, context, channel, unloadStrategy); + + scheduler.execute(); + + verify(channel, times(0)).isChannelOwnerAsync(); + + context.brokerConfiguration().setLoadBalancerEnabled(true); + context.brokerConfiguration().setLoadBalancerSheddingEnabled(false); + scheduler.execute(); + + verify(channel, times(0)).isChannelOwnerAsync(); + } + + @Test(timeOut = 30 * 1000) + public void testNotChannelOwner() { + LoadManagerContext context = setupContext(); + context.brokerConfiguration().setLoadBalancerEnabled(false); + ServiceUnitStateChannel channel = mock(ServiceUnitStateChannel.class); + NamespaceUnloadStrategy unloadStrategy = mock(NamespaceUnloadStrategy.class); + UnloadScheduler scheduler = new UnloadScheduler(loadManagerExecutor, context, channel, unloadStrategy); + doReturn(CompletableFuture.completedFuture(false)).when(channel).isChannelOwnerAsync(); + + scheduler.execute(); + + verify(context.brokerRegistry(), times(0)).getAvailableBrokersAsync(); + } + + public LoadManagerContext getContext(){ + var ctx = mock(LoadManagerContext.class); + var registry = mock(BrokerRegistry.class); + var conf = new ServiceConfiguration(); + doReturn(conf).when(ctx).brokerConfiguration(); + doReturn(registry).when(ctx).brokerRegistry(); + return ctx; + } +} From bb83502ebe9a7b50070b988735925b247cfa8fef Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Tue, 14 Feb 2023 11:17:25 +0800 Subject: [PATCH 011/174] [improve][broker] introduce consistent hash ring to distribute the load of multiple topics subscription (#19502) --- ...bstractDispatcherSingleActiveConsumer.java | 73 ++++++++++++------- .../broker/service/ResendRequestTest.java | 11 ++- .../client/impl/TopicsConsumerImplTest.java | 52 +++++++++++++ .../pulsar/client/impl/ZeroQueueSizeTest.java | 8 +- 4 files changed, 113 insertions(+), 31 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractDispatcherSingleActiveConsumer.java index 6380fb8384b04..2e22a80250cc3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractDispatcherSingleActiveConsumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractDispatcherSingleActiveConsumer.java @@ -19,8 +19,12 @@ package org.apache.pulsar.broker.service; import static com.google.common.base.Preconditions.checkArgument; +import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.NavigableMap; import java.util.Objects; +import java.util.TreeMap; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicBoolean; @@ -30,7 +34,9 @@ import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.service.BrokerServiceException.ConsumerBusyException; import org.apache.pulsar.broker.service.BrokerServiceException.ServerMetadataException; +import org.apache.pulsar.client.impl.Murmur3Hash32; import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; +import org.apache.pulsar.common.util.Murmur3_32Hash; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,6 +63,7 @@ public abstract class AbstractDispatcherSingleActiveConsumer extends AbstractBas private volatile int isClosed = FALSE; protected boolean isFirstRead = true; + private static final int CONSUMER_CONSISTENT_HASH_REPLICAS = 100; public AbstractDispatcherSingleActiveConsumer(SubType subscriptionType, int partitionIndex, String topicName, Subscription subscription, @@ -93,35 +100,31 @@ protected void notifyActiveConsumerChanged(Consumer activeConsumer) { */ protected boolean pickAndScheduleActiveConsumer() { checkArgument(!consumers.isEmpty()); - // By default always pick the first connected consumer for non partitioned topic. - int index = 0; - - // If it's a partitioned topic, sort consumers based on priority level then consumer name. - if (partitionIndex >= 0) { - AtomicBoolean hasPriorityConsumer = new AtomicBoolean(false); - consumers.sort((c1, c2) -> { - int priority = c1.getPriorityLevel() - c2.getPriorityLevel(); - if (priority != 0) { - hasPriorityConsumer.set(true); - return priority; - } - return c1.consumerName().compareTo(c2.consumerName()); - }); - - int consumersSize = consumers.size(); - // find number of consumers which are having the highest priorities. so partitioned-topic assignment happens - // evenly across highest priority consumers - if (hasPriorityConsumer.get()) { - int highestPriorityLevel = consumers.get(0).getPriorityLevel(); - for (int i = 0; i < consumers.size(); i++) { - if (highestPriorityLevel != consumers.get(i).getPriorityLevel()) { - consumersSize = i; - break; - } + AtomicBoolean hasPriorityConsumer = new AtomicBoolean(false); + consumers.sort((c1, c2) -> { + int priority = c1.getPriorityLevel() - c2.getPriorityLevel(); + if (priority != 0) { + hasPriorityConsumer.set(true); + return priority; + } + return c1.consumerName().compareTo(c2.consumerName()); + }); + + int consumersSize = consumers.size(); + // find number of consumers which are having the highest priorities. so partitioned-topic assignment happens + // evenly across highest priority consumers + if (hasPriorityConsumer.get()) { + int highestPriorityLevel = consumers.get(0).getPriorityLevel(); + for (int i = 0; i < consumers.size(); i++) { + if (highestPriorityLevel != consumers.get(i).getPriorityLevel()) { + consumersSize = i; + break; } } - index = partitionIndex % consumersSize; } + int index = partitionIndex >= 0 + ? partitionIndex % consumersSize + : peekConsumerIndexFromHashRing(makeHashRing(consumersSize)); Consumer prevConsumer = ACTIVE_CONSUMER_UPDATER.getAndSet(this, consumers.get(index)); @@ -136,6 +139,24 @@ protected boolean pickAndScheduleActiveConsumer() { } } + private int peekConsumerIndexFromHashRing(NavigableMap hashRing) { + int hash = Murmur3Hash32.getInstance().makeHash(topicName); + Map.Entry ceilingEntry = hashRing.ceilingEntry(hash); + return ceilingEntry != null ? ceilingEntry.getValue() : hashRing.firstEntry().getValue(); + } + + private NavigableMap makeHashRing(int consumerSize) { + NavigableMap hashRing = new TreeMap<>(); + for (int i = 0; i < consumerSize; i++) { + for (int j = 0; j < CONSUMER_CONSISTENT_HASH_REPLICAS; j++) { + String key = consumers.get(i).consumerName() + j; + int hash = Murmur3_32Hash.getInstance().makeHash(key.getBytes()); + hashRing.put(hash, i); + } + } + return Collections.unmodifiableNavigableMap(hashRing); + } + public synchronized void addConsumer(Consumer consumer) throws BrokerServiceException { if (IS_CLOSED_UPDATER.get(this) == TRUE) { log.warn("[{}] Dispatcher is already closed. Closing consumer {}", this.topicName, consumer); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ResendRequestTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ResendRequestTest.java index 3aedf75762d0a..113d766a8ab64 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ResendRequestTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ResendRequestTest.java @@ -707,12 +707,19 @@ public void testFailoverInactiveConsumer() throws Exception { receivedConsumer1 += 1; } } while (message1 != null); + do { + message2 = consumer2.receive(500, TimeUnit.MILLISECONDS); + if (message2 != null) { + log.info("Consumer 2 Received: " + new String(message2.getData())); + receivedConsumer2 += 1; + } + } while (message2 != null); log.info("Consumer 1 receives = " + receivedConsumer1); log.info("Consumer 2 receives = " + receivedConsumer2); log.info("Total receives = " + (receivedConsumer2 + receivedConsumer1)); assertEquals(receivedConsumer2 + receivedConsumer1, totalMessages); // Consumer 2 is on Stand By - assertEquals(receivedConsumer2, 0); + assertEquals(receivedConsumer1, 0); // 5. Consumer 2 asks for a redelivery but the request is ignored log.info("Consumer 2 asks for resend"); @@ -722,7 +729,7 @@ public void testFailoverInactiveConsumer() throws Exception { message1 = consumer1.receive(500, TimeUnit.MILLISECONDS); message2 = consumer2.receive(500, TimeUnit.MILLISECONDS); assertNull(message1); - assertNull(message2); + assertNotNull(message2); } @SuppressWarnings("unchecked") diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java index 8fcf378079ea5..ce4a0ae86ac4e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java @@ -30,6 +30,7 @@ import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.ConsumerEventListener; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.MessageRouter; @@ -1304,4 +1305,55 @@ public void testPartitionsUpdatesForMultipleTopics() throws Exception { }); } + @Test + public void testTopicsDistribution() throws Exception { + final String topic = "topics-distribution"; + final int topicCount = 100; + final int consumers = 10; + + for (int i = 0; i < topicCount; i++) { + admin.topics().createNonPartitionedTopic(topic + "-" + i); + } + + CustomizedConsumerEventListener eventListener = new CustomizedConsumerEventListener(); + + List> consumerList = new ArrayList<>(consumers); + for (int i = 0; i < consumers; i++) { + consumerList.add(pulsarClient.newConsumer() + .topics(IntStream.range(0, topicCount).mapToObj(j -> topic + "-" + j).toList()) + .subscriptionType(SubscriptionType.Failover) + .subscriptionName("my-sub") + .consumerName("consumer-" + i) + .consumerEventListener(eventListener) + .subscribe()); + } + + log.info("Topics are distributed to consumers as {}", eventListener.getActiveConsumers()); + Map assigned = new HashMap<>(); + eventListener.getActiveConsumers().forEach((k, v) -> assigned.compute(v, (t, c) -> c == null ? 1 : ++ c)); + assertEquals(assigned.size(), consumers); + for (Consumer consumer : consumerList) { + consumer.close(); + } + } + + private static class CustomizedConsumerEventListener implements ConsumerEventListener { + + private final Map activeConsumers = new HashMap<>(); + + @Override + public void becameActive(Consumer consumer, int partitionId) { + activeConsumers.put(consumer.getTopic(), consumer.getConsumerName()); + } + + @Override + public void becameInactive(Consumer consumer, int partitionId) { + //no-op + } + + public Map getActiveConsumers() { + return activeConsumers; + } + } + } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ZeroQueueSizeTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ZeroQueueSizeTest.java index fd18cc06fce7f..a75838c1eb895 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ZeroQueueSizeTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ZeroQueueSizeTest.java @@ -240,9 +240,7 @@ public void zeroQueueSizeFailoverSubscription() throws PulsarClientException { ConsumerImpl consumer1 = (ConsumerImpl) pulsarClient.newConsumer().topic(topicName) .subscriptionName(subscriptionName).receiverQueueSize(0).subscriptionType(SubscriptionType.Failover) .consumerName("consumer-1").subscribe(); - ConsumerImpl consumer2 = (ConsumerImpl) pulsarClient.newConsumer().topic(topicName) - .subscriptionName(subscriptionName).receiverQueueSize(0).subscriptionType(SubscriptionType.Failover) - .consumerName("consumer-2").subscribe(); + // 4. Produce Messages for (int i = 0; i < totalMessages; i++) { @@ -260,6 +258,10 @@ public void zeroQueueSizeFailoverSubscription() throws PulsarClientException { log.info("Consumer received : " + new String(message.getData())); } + ConsumerImpl consumer2 = (ConsumerImpl) pulsarClient.newConsumer().topic(topicName) + .subscriptionName(subscriptionName).receiverQueueSize(0).subscriptionType(SubscriptionType.Failover) + .consumerName("consumer-2").subscribe(); + // 6. Trigger redelivery consumer1.redeliverUnacknowledgedMessages(); From 70c4003c5c5ef6d911f639d946c235721b6d0e67 Mon Sep 17 00:00:00 2001 From: yws-tracy <34879511+yws-tracy@users.noreply.github.com> Date: Tue, 14 Feb 2023 11:19:52 +0800 Subject: [PATCH 012/174] [fix][broker]: make log4j2 delete strategy work (#19495) --- conf/log4j2.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/log4j2.yaml b/conf/log4j2.yaml index fff2e063ebd0c..9c261a6b89a50 100644 --- a/conf/log4j2.yaml +++ b/conf/log4j2.yaml @@ -78,7 +78,7 @@ Configuration: basePath: ${sys:pulsar.log.dir} maxDepth: 2 IfFileName: - glob: "*/${sys:pulsar.log.file}*log.gz" + glob: "${sys:pulsar.log.file}*log.gz" IfLastModified: age: 30d @@ -120,7 +120,7 @@ Configuration: basePath: ${sys:pulsar.log.dir} maxDepth: 2 IfFileName: - glob: "*/${sys:pulsar.log.file}*log.gz" + glob: "${sys:pulsar.log.file}*log.gz" IfLastModified: age: 30d - ref: "${sys:pulsar.routing.appender.default}" From c4c174424192202bc0aec9414a1fec0334600fca Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Mon, 13 Feb 2023 23:37:07 -0600 Subject: [PATCH 013/174] [fix][broker] Make ServerCnx#originalAuthData volatile (#19507) Fixes #19431 ### Motivation `authenticationData` is already `volatile`. We use `originalAuthData` when set, so we should match the style. In #19431, I proposed that we find a way to not use `volatile`. I still think this might be a "better" approach, but it will be a larger change, and since we already use `volatile` for `authenticationData`, I think this is the right change for now. It's possible that this is not a bug, given that the `originalAuthData` does not change frequently. However, we always want to use up to date values for authorization. ### Modifications * Add `volatile` keyword to `ServerCnx#originalAuthData`. ### Verifying this change This change is a trivial rework / code cleanup without any test coverage. ### Documentation - [x] `doc-not-needed` ### Matching PR in forked repository PR in forked repository: skipping test in fork. --- .../main/java/org/apache/pulsar/broker/service/ServerCnx.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 9cbdbcf1ae4d5..c6534cbc301c7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -195,7 +195,7 @@ public class ServerCnx extends PulsarHandler implements TransportCnx { // In case of proxy, if the authentication credentials are forwardable, // it will hold the credentials of the original client private AuthenticationState originalAuthState; - private AuthenticationDataSource originalAuthData; + private volatile AuthenticationDataSource originalAuthData; // Keep temporarily in order to verify after verifying proxy's authData private AuthData originalAuthDataCopy; private boolean pendingAuthChallengeResponse = false; From aa63a5567a9e5d466b311a54d5dcc2cb05c2b5cd Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Tue, 14 Feb 2023 00:20:27 -0600 Subject: [PATCH 014/174] [improve][broker] Require authRole is proxyRole to set originalPrincipal (#19455) Co-authored-by: Lari Hotari --- .../authorization/AuthorizationService.java | 67 ++++++++++++------ .../admin/impl/PersistentTopicsBase.java | 2 +- .../pulsar/broker/service/ServerCnx.java | 31 +------- .../pulsar/broker/web/PulsarWebResource.java | 27 +++---- .../pulsar/broker/auth/AuthorizationTest.java | 39 ++++++++++ .../auth/MockedPulsarServiceBaseTest.java | 7 ++ .../pulsar/broker/service/ServerCnxTest.java | 4 ++ .../impl/AdminApiKeyStoreTlsAuthTest.java | 16 ++--- ...roxyAuthenticatedProducerConsumerTest.java | 44 ++++++++---- .../server/ProxyWithAuthorizationNegTest.java | 2 + .../server/ProxyWithJwtAuthorizationTest.java | 4 +- .../generate_keystore.sh | 11 +++ .../jks/broker.keystore.jks | Bin 2254 -> 2254 bytes .../jks/broker.truststore.jks | Bin 978 -> 969 bytes .../jks/broker.truststore.nopassword.jks | Bin 978 -> 969 bytes .../jks/client.keystore.jks | Bin 2258 -> 2257 bytes .../jks/client.truststore.jks | Bin 980 -> 971 bytes .../jks/client.truststore.nopassword.jks | Bin 980 -> 971 bytes .../jks/proxy-and-client.truststore.jks | Bin 0 -> 1891 bytes ...proxy-and-client.truststore.nopassword.jks | Bin 0 -> 1891 bytes .../jks/proxy.keystore.jks | Bin 0 -> 2245 bytes .../jks/proxy.truststore.jks | Bin 0 -> 971 bytes .../jks/proxy.truststore.nopassword.jks | Bin 0 -> 971 bytes 23 files changed, 161 insertions(+), 93 deletions(-) create mode 100644 tests/certificate-authority/jks/proxy-and-client.truststore.jks create mode 100644 tests/certificate-authority/jks/proxy-and-client.truststore.nopassword.jks create mode 100644 tests/certificate-authority/jks/proxy.keystore.jks create mode 100644 tests/certificate-authority/jks/proxy.truststore.jks create mode 100644 tests/certificate-authority/jks/proxy.truststore.nopassword.jks diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java index 526e8430d0f1a..39f401b493f17 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java @@ -19,10 +19,10 @@ package org.apache.pulsar.broker.authorization; import static java.util.concurrent.TimeUnit.SECONDS; +import java.net.SocketAddress; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; -import javax.ws.rs.core.Response; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.ServiceConfiguration; @@ -37,7 +37,6 @@ import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.common.policies.data.TenantOperation; import org.apache.pulsar.common.policies.data.TopicOperation; -import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.RestException; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.slf4j.Logger; @@ -293,19 +292,39 @@ public CompletableFuture allowSinkOpsAsync(NamespaceName namespaceName, return provider.allowSinkOpsAsync(namespaceName, role, authenticationData); } - private static void validateOriginalPrincipal(Set proxyRoles, String authenticatedPrincipal, - String originalPrincipal) { - if (proxyRoles.contains(authenticatedPrincipal)) { - // Request has come from a proxy + public boolean isValidOriginalPrincipal(String authenticatedPrincipal, + String originalPrincipal, + AuthenticationDataSource authDataSource) { + SocketAddress remoteAddress = authDataSource != null ? authDataSource.getPeerAddress() : null; + return isValidOriginalPrincipal(authenticatedPrincipal, originalPrincipal, remoteAddress); + } + + /** + * Validates that the authenticatedPrincipal and the originalPrincipal are a valid combination. + * Valid combinations fulfill the following rule: the authenticatedPrincipal is in + * {@link ServiceConfiguration#getProxyRoles()}, if, and only if, the originalPrincipal is set to a role + * that is not also in {@link ServiceConfiguration#getProxyRoles()}. + * @return true when roles are a valid combination and false when roles are an invalid combination + */ + public boolean isValidOriginalPrincipal(String authenticatedPrincipal, + String originalPrincipal, + SocketAddress remoteAddress) { + String errorMsg = null; + if (conf.getProxyRoles().contains(authenticatedPrincipal)) { if (StringUtils.isBlank(originalPrincipal)) { - log.warn("Original principal empty in request authenticated as {}", authenticatedPrincipal); - throw new RestException(Response.Status.UNAUTHORIZED, "Original principal cannot be empty if the " - + "request is via proxy."); - } - if (proxyRoles.contains(originalPrincipal)) { - log.warn("Original principal {} cannot be a proxy role ({})", originalPrincipal, proxyRoles); - throw new RestException(Response.Status.UNAUTHORIZED, "Original principal cannot be a proxy role"); + errorMsg = "originalPrincipal must be provided when connecting with a proxy role."; + } else if (conf.getProxyRoles().contains(originalPrincipal)) { + errorMsg = "originalPrincipal cannot be a proxy role."; } + } else if (StringUtils.isNotBlank(originalPrincipal)) { + errorMsg = "cannot specify originalPrincipal when connecting without valid proxy role."; + } + if (errorMsg != null) { + log.warn("[{}] Illegal combination of role [{}] and originalPrincipal [{}]: {}", remoteAddress, + authenticatedPrincipal, originalPrincipal, errorMsg); + return false; + } else { + return true; } } @@ -340,7 +359,9 @@ public CompletableFuture allowTenantOperationAsync(String tenantName, String originalRole, String role, AuthenticationDataSource authData) { - validateOriginalPrincipal(conf.getProxyRoles(), role, originalRole); + if (!isValidOriginalPrincipal(role, originalRole, authData)) { + return CompletableFuture.completedFuture(false); + } if (isProxyRole(role)) { CompletableFuture isRoleAuthorizedFuture = allowTenantOperationAsync( tenantName, operation, role, authData); @@ -400,7 +421,9 @@ public CompletableFuture allowNamespaceOperationAsync(NamespaceName nam String originalRole, String role, AuthenticationDataSource authData) { - validateOriginalPrincipal(conf.getProxyRoles(), role, originalRole); + if (!isValidOriginalPrincipal(role, originalRole, authData)) { + return CompletableFuture.completedFuture(false); + } if (isProxyRole(role)) { CompletableFuture isRoleAuthorizedFuture = allowNamespaceOperationAsync( namespaceName, operation, role, authData); @@ -442,7 +465,9 @@ public CompletableFuture allowNamespacePolicyOperationAsync(NamespaceNa String originalRole, String role, AuthenticationDataSource authData) { - validateOriginalPrincipal(conf.getProxyRoles(), role, originalRole); + if (!isValidOriginalPrincipal(role, originalRole, authData)) { + return CompletableFuture.completedFuture(false); + } if (isProxyRole(role)) { CompletableFuture isRoleAuthorizedFuture = allowNamespacePolicyOperationAsync( namespaceName, policy, operation, role, authData); @@ -503,10 +528,8 @@ public CompletableFuture allowTopicPolicyOperationAsync(TopicName topic String originalRole, String role, AuthenticationDataSource authData) { - try { - validateOriginalPrincipal(conf.getProxyRoles(), role, originalRole); - } catch (RestException e) { - return FutureUtil.failedFuture(e); + if (!isValidOriginalPrincipal(role, originalRole, authData)) { + return CompletableFuture.completedFuture(false); } if (isProxyRole(role)) { CompletableFuture isRoleAuthorizedFuture = allowTopicPolicyOperationAsync( @@ -594,7 +617,9 @@ public CompletableFuture allowTopicOperationAsync(TopicName topicName, String originalRole, String role, AuthenticationDataSource authData) { - validateOriginalPrincipal(conf.getProxyRoles(), role, originalRole); + if (!isValidOriginalPrincipal(role, originalRole, authData)) { + return CompletableFuture.completedFuture(false); + } if (isProxyRole(role)) { CompletableFuture isRoleAuthorizedFuture = allowTopicOperationAsync( topicName, operation, role, authData); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index c5d465e747e12..0214079335bb3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -4317,7 +4317,7 @@ protected void internalOffloadStatus(AsyncResponse asyncResponse, boolean author }); } - public static CompletableFuture getPartitionedTopicMetadata( + public CompletableFuture getPartitionedTopicMetadata( PulsarService pulsar, String clientAppId, String originalPrincipal, AuthenticationDataSource authenticationData, TopicName topicName) { CompletableFuture metadataFuture = new CompletableFuture<>(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index c6534cbc301c7..1351c6fe715a9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -210,7 +210,6 @@ public class ServerCnx extends PulsarHandler implements TransportCnx { private int nonPersistentPendingMessages = 0; private final int maxNonPersistentPendingMessages; private String originalPrincipal = null; - private Set proxyRoles; private final boolean schemaValidationEnforced; private String authMethod = "none"; private final int maxMessageSize; @@ -282,7 +281,6 @@ public ServerCnx(PulsarService pulsar, String listenerName) { this.recentlyClosedProducers = new HashMap<>(); this.replicatorPrefix = conf.getReplicatorPrefix(); this.maxNonPersistentPendingMessages = conf.getMaxConcurrentNonPersistentMessagePerConnection(); - this.proxyRoles = conf.getProxyRoles(); this.schemaValidationEnforced = conf.isSchemaValidationEnforced(); this.maxMessageSize = conf.getMaxMessageSize(); this.maxPendingSendRequests = conf.getMaxPendingPublishRequestsPerConnection(); @@ -400,32 +398,6 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws E ctx.close(); } - /** - * When transitioning from Connecting to Connected, this method validates the roles. - * If the authRole is one of proxyRoles, the following must be true: - * - the originalPrincipal is given while connecting - * - originalPrincipal is not blank - * - originalPrincipal is not a proxy principal. - * @return true when roles are valid and false when roles are invalid - */ - private boolean isValidRoleAndOriginalPrincipal() { - String errorMsg = null; - if (proxyRoles.contains(authRole)) { - if (StringUtils.isBlank(originalPrincipal)) { - errorMsg = "originalPrincipal must be provided when connecting with a proxy role."; - } else if (proxyRoles.contains(originalPrincipal)) { - errorMsg = "originalPrincipal cannot be a proxy role."; - } - } - if (errorMsg != null) { - log.warn("[{}] Illegal combination of role [{}] and originalPrincipal [{}]: {}", remoteAddress, authRole, - originalPrincipal, errorMsg); - return false; - } else { - return true; - } - } - // //// // // Incoming commands handling // //// @@ -694,7 +666,8 @@ ByteBuf createConsumerStatsResponse(Consumer consumer, long requestId) { // complete the connect and sent newConnected command private void completeConnect(int clientProtoVersion, String clientVersion) { if (service.isAuthenticationEnabled() && service.isAuthorizationEnabled()) { - if (!isValidRoleAndOriginalPrincipal()) { + if (!service.getAuthorizationService() + .isValidOriginalPrincipal(authRole, originalPrincipal, remoteAddress)) { state = State.Failed; service.getPulsarStats().recordConnectionCreateFail(); final ByteBuf msg = Commands.newError(-1, ServerError.AuthorizationError, "Invalid roles."); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java index af8eb78de7dce..fb80a3e79834f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java @@ -150,19 +150,11 @@ public static boolean isClientAuthenticated(String appId) { return appId != null; } - private static void validateOriginalPrincipal(Set proxyRoles, String authenticatedPrincipal, - String originalPrincipal) { - if (proxyRoles.contains(authenticatedPrincipal)) { - // Request has come from a proxy - if (StringUtils.isBlank(originalPrincipal)) { - log.warn("Original principal empty in request authenticated as {}", authenticatedPrincipal); - throw new RestException(Status.UNAUTHORIZED, - "Original principal cannot be empty if the request is via proxy."); - } - if (proxyRoles.contains(originalPrincipal)) { - log.warn("Original principal {} cannot be a proxy role ({})", originalPrincipal, proxyRoles); - throw new RestException(Status.UNAUTHORIZED, "Original principal cannot be a proxy role"); - } + private void validateOriginalPrincipal(String authenticatedPrincipal, String originalPrincipal) { + if (!pulsar.getBrokerService().getAuthorizationService() + .isValidOriginalPrincipal(authenticatedPrincipal, originalPrincipal, clientAuthData())) { + throw new RestException(Status.UNAUTHORIZED, + "Invalid combination of Original principal cannot be empty if the request is via proxy."); } } @@ -185,7 +177,7 @@ public CompletableFuture validateSuperUserAccessAsync(){ isClientAuthenticated(appId), appId); } String originalPrincipal = originalPrincipal(); - validateOriginalPrincipal(pulsar.getConfiguration().getProxyRoles(), appId, originalPrincipal); + validateOriginalPrincipal(appId, originalPrincipal); if (pulsar.getConfiguration().getProxyRoles().contains(appId)) { BrokerService brokerService = pulsar.getBrokerService(); @@ -260,7 +252,7 @@ protected void validateAdminAccessForTenant(String tenant) { } } - protected static void validateAdminAccessForTenant(PulsarService pulsar, String clientAppId, + protected void validateAdminAccessForTenant(PulsarService pulsar, String clientAppId, String originalPrincipal, String tenant, AuthenticationDataSource authenticationData, long timeout, TimeUnit unit) { @@ -287,7 +279,7 @@ protected CompletableFuture validateAdminAccessForTenantAsync(String tenan clientAuthData()); } - protected static CompletableFuture validateAdminAccessForTenantAsync( + protected CompletableFuture validateAdminAccessForTenantAsync( PulsarService pulsar, String clientAppId, String originalPrincipal, String tenant, AuthenticationDataSource authenticationData) { @@ -306,8 +298,7 @@ protected static CompletableFuture validateAdminAccessForTenantAsync( if (!isClientAuthenticated(clientAppId)) { throw new RestException(Status.FORBIDDEN, "Need to authenticate to perform the request"); } - validateOriginalPrincipal(pulsar.getConfiguration().getProxyRoles(), clientAppId, - originalPrincipal); + validateOriginalPrincipal(clientAppId, originalPrincipal); if (pulsar.getConfiguration().getProxyRoles().contains(clientAppId)) { AuthorizationService authorizationService = pulsar.getBrokerService().getAuthorizationService(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java index 11afe889ee935..c578d9ec94162 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java @@ -23,8 +23,13 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import com.google.common.collect.Sets; +import java.net.SocketAddress; +import java.util.Collections; import java.util.EnumSet; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.authentication.AuthenticationDataSource; import org.apache.pulsar.broker.authorization.AuthorizationService; +import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminBuilder; import org.apache.pulsar.common.naming.TopicDomain; @@ -33,6 +38,7 @@ import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.SubscriptionAuthMode; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -229,6 +235,39 @@ public void simple() throws Exception { admin.clusters().deleteCluster("c1"); } + @Test + public void testOriginalRoleValidation() throws Exception { + ServiceConfiguration conf = new ServiceConfiguration(); + conf.setProxyRoles(Collections.singleton("proxy")); + AuthorizationService auth = new AuthorizationService(conf, Mockito.mock(PulsarResources.class)); + + // Original principal should be supplied when authenticatedPrincipal is proxy role + assertTrue(auth.isValidOriginalPrincipal("proxy", "client", (SocketAddress) null)); + + // Non proxy role should not supply originalPrincipal + assertTrue(auth.isValidOriginalPrincipal("client", "", (SocketAddress) null)); + assertTrue(auth.isValidOriginalPrincipal("client", null, (SocketAddress) null)); + + // Only likely in cases when authentication is disabled, but we still define these to be valid. + assertTrue(auth.isValidOriginalPrincipal(null, null, (SocketAddress) null)); + assertTrue(auth.isValidOriginalPrincipal(null, "", (SocketAddress) null)); + assertTrue(auth.isValidOriginalPrincipal("", null, (SocketAddress) null)); + assertTrue(auth.isValidOriginalPrincipal("", "", (SocketAddress) null)); + + // Proxy role must supply an original principal + assertFalse(auth.isValidOriginalPrincipal("proxy", "", (SocketAddress) null)); + assertFalse(auth.isValidOriginalPrincipal("proxy", null, (SocketAddress) null)); + + // OriginalPrincipal cannot be proxy role + assertFalse(auth.isValidOriginalPrincipal("proxy", "proxy", (SocketAddress) null)); + assertFalse(auth.isValidOriginalPrincipal("client", "proxy", (SocketAddress) null)); + assertFalse(auth.isValidOriginalPrincipal("", "proxy", (SocketAddress) null)); + assertFalse(auth.isValidOriginalPrincipal(null, "proxy", (SocketAddress) null)); + + // Must gracefully handle a missing AuthenticationDataSource + assertTrue(auth.isValidOriginalPrincipal("proxy", "client", (AuthenticationDataSource) null)); + } + @Test public void testGetListWithGetBundleOp() throws Exception { String tenant = "p1"; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java index ce21449d89ee1..cd31f9150e619 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java @@ -84,6 +84,13 @@ public abstract class MockedPulsarServiceBaseTest extends TestRetrySupport { public final static String CLIENT_KEYSTORE_PW = "111111"; public final static String CLIENT_TRUSTSTORE_PW = "111111"; + public final static String PROXY_KEYSTORE_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/jks/proxy.keystore.jks"); + public final static String PROXY_KEYSTORE_PW = "111111"; + public final static String PROXY_AND_CLIENT_TRUSTSTORE_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/jks/proxy-and-client.truststore.jks"); + public final static String PROXY_AND_CLIENT_TRUSTSTORE_PW = "111111"; + public final static String CLIENT_KEYSTORE_CN = "clientuser"; public final static String KEYSTORE_TYPE = "JKS"; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java index b7e2839832c5d..afa1cd4e2528c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java @@ -602,6 +602,10 @@ public void testConnectCommandWithInvalidRoleCombinations() throws Exception { verifyAuthRoleAndOriginalPrincipalBehavior(authMethodName, "pass.proxy", "pass.proxy"); verifyAuthRoleAndOriginalPrincipalBehavior(authMethodName, "pass.proxy", ""); verifyAuthRoleAndOriginalPrincipalBehavior(authMethodName, "pass.proxy", null); + // Invalid combinations where original principal is set to a pass.proxy role + verifyAuthRoleAndOriginalPrincipalBehavior(authMethodName, "pass.client", "pass.proxy"); + // Invalid combinations where the original principal is set to a non-proxy role + verifyAuthRoleAndOriginalPrincipalBehavior(authMethodName, "pass.client1", "pass.client"); } private void verifyAuthRoleAndOriginalPrincipalBehavior(String authMethodName, String authData, diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/AdminApiKeyStoreTlsAuthTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/AdminApiKeyStoreTlsAuthTest.java index 3194a91c6e4b3..4f9f49a0842a1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/AdminApiKeyStoreTlsAuthTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/AdminApiKeyStoreTlsAuthTest.java @@ -50,7 +50,6 @@ import org.apache.pulsar.client.impl.auth.AuthenticationToken; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; -import org.apache.pulsar.common.tls.NoopHostnameVerifier; import org.apache.pulsar.common.util.keystoretls.KeyStoreSSLContext; import org.glassfish.jersey.client.ClientConfig; import org.glassfish.jersey.client.ClientProperties; @@ -83,8 +82,8 @@ public void setup() throws Exception { conf.setTlsKeyStorePassword(BROKER_KEYSTORE_PW); conf.setTlsTrustStoreType(KEYSTORE_TYPE); - conf.setTlsTrustStore(CLIENT_TRUSTSTORE_FILE_PATH); - conf.setTlsTrustStorePassword(CLIENT_TRUSTSTORE_PW); + conf.setTlsTrustStore(PROXY_AND_CLIENT_TRUSTSTORE_FILE_PATH); + conf.setTlsTrustStorePassword(PROXY_AND_CLIENT_TRUSTSTORE_PW); conf.setClusterName(clusterName); conf.setTlsRequireTrustedClientCertOnConnect(true); @@ -94,6 +93,7 @@ public void setup() throws Exception { // config for authentication and authorization. conf.setSuperUserRoles(Sets.newHashSet(CLIENT_KEYSTORE_CN)); + conf.setProxyRoles(Sets.newHashSet("proxy")); conf.setAuthenticationEnabled(true); conf.setAuthorizationEnabled(true); Set providers = new HashSet<>(); @@ -139,13 +139,13 @@ WebTarget buildWebClient() throws Exception { SSLContext sslCtx = KeyStoreSSLContext.createClientSslContext( KEYSTORE_TYPE, - CLIENT_KEYSTORE_FILE_PATH, - CLIENT_KEYSTORE_PW, + PROXY_KEYSTORE_FILE_PATH, + PROXY_KEYSTORE_PW, KEYSTORE_TYPE, BROKER_TRUSTSTORE_FILE_PATH, BROKER_TRUSTSTORE_PW); - clientBuilder.sslContext(sslCtx).hostnameVerifier(NoopHostnameVerifier.INSTANCE); + clientBuilder.sslContext(sslCtx); Client client = clientBuilder.build(); return client.target(brokerUrlTls.toString()); @@ -178,11 +178,11 @@ public void testSuperUserCanListTenants() throws Exception { } @Test - public void testSuperUserCantListNamespaces() throws Exception { + public void testSuperUserCanListNamespaces() throws Exception { try (PulsarAdmin admin = buildAdminClient()) { admin.clusters().createCluster("test", ClusterData.builder().serviceUrl(brokerUrl.toString()).build()); admin.tenants().createTenant("tenant1", - new TenantInfoImpl(Set.of("proxy"), + new TenantInfoImpl(Set.of(""), Set.of("test"))); admin.namespaces().createNamespace("tenant1/ns1"); Assert.assertTrue(admin.namespaces().getNamespaces("tenant1").contains("tenant1/ns1")); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticatedProducerConsumerTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticatedProducerConsumerTest.java index 904883c18540d..bfe86f86976ee 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticatedProducerConsumerTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticatedProducerConsumerTest.java @@ -23,6 +23,7 @@ import com.google.common.collect.Sets; +import java.util.Collections; import java.util.HashSet; import java.util.Map; import java.util.Optional; @@ -57,9 +58,17 @@ public class ProxyAuthenticatedProducerConsumerTest extends ProducerConsumerBase { private static final Logger log = LoggerFactory.getLogger(ProxyAuthenticatedProducerConsumerTest.class); + // Root for both proxy and client certificates private final String TLS_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem"; - private final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/server-cert.pem"; - private final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/server-key.pem"; + + // Borrow certs for broker and proxy from other test + private final String TLS_PROXY_CERT_FILE_PATH = "./src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cert.pem"; + private final String TLS_PROXY_KEY_FILE_PATH = "./src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-key.pem"; + private final String TLS_BROKER_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-cacert.pem"; + private final String TLS_BROKER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-cert.pem"; + private final String TLS_BROKER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-key.pem"; + + // This client cert is a superUser, so use that one private final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem"; private final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem"; @@ -78,20 +87,23 @@ protected void setup() throws Exception { conf.setBrokerServicePortTls(Optional.of(0)); conf.setWebServicePortTls(Optional.of(0)); conf.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); - conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH); - conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH); - conf.setTlsAllowInsecureConnection(true); + conf.setTlsCertificateFilePath(TLS_BROKER_CERT_FILE_PATH); + conf.setTlsKeyFilePath(TLS_BROKER_KEY_FILE_PATH); + conf.setTlsAllowInsecureConnection(false); conf.setNumExecutorThreadPoolSize(5); Set superUserRoles = new HashSet<>(); superUserRoles.add("localhost"); superUserRoles.add("superUser"); + superUserRoles.add("Proxy"); conf.setSuperUserRoles(superUserRoles); + conf.setProxyRoles(Collections.singleton("Proxy")); conf.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName()); conf.setBrokerClientAuthenticationParameters( - "tlsCertFile:" + TLS_CLIENT_CERT_FILE_PATH + "," + "tlsKeyFile:" + TLS_SERVER_KEY_FILE_PATH); - conf.setBrokerClientTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); + "tlsCertFile:" + TLS_CLIENT_CERT_FILE_PATH + "," + "tlsKeyFile:" + TLS_CLIENT_KEY_FILE_PATH); + conf.setBrokerClientTrustCertsFilePath(TLS_BROKER_TRUST_CERT_FILE_PATH); + conf.setBrokerClientTlsEnabled(true); Set providers = new HashSet<>(); providers.add(AuthenticationProviderTls.class.getName()); conf.setAuthenticationProviders(providers); @@ -102,7 +114,6 @@ protected void setup() throws Exception { // start proxy service proxyConfig.setAuthenticationEnabled(true); - proxyConfig.setAuthenticationEnabled(true); proxyConfig.setServicePort(Optional.of(0)); proxyConfig.setBrokerProxyAllowedTargetPorts("*"); @@ -110,16 +121,18 @@ protected void setup() throws Exception { proxyConfig.setWebServicePort(Optional.of(0)); proxyConfig.setWebServicePortTls(Optional.of(0)); proxyConfig.setTlsEnabledWithBroker(true); + // Setting advertised address to localhost to avoid hostname verification failure + proxyConfig.setAdvertisedAddress("localhost"); // enable tls and auth&auth at proxy - proxyConfig.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH); - proxyConfig.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH); + proxyConfig.setTlsCertificateFilePath(TLS_PROXY_CERT_FILE_PATH); + proxyConfig.setTlsKeyFilePath(TLS_PROXY_KEY_FILE_PATH); proxyConfig.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); proxyConfig.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName()); proxyConfig.setBrokerClientAuthenticationParameters( - "tlsCertFile:" + TLS_CLIENT_CERT_FILE_PATH + "," + "tlsKeyFile:" + TLS_CLIENT_KEY_FILE_PATH); - proxyConfig.setBrokerClientTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); + "tlsCertFile:" + TLS_PROXY_CERT_FILE_PATH + "," + "tlsKeyFile:" + TLS_PROXY_KEY_FILE_PATH); + proxyConfig.setBrokerClientTrustCertsFilePath(TLS_BROKER_TRUST_CERT_FILE_PATH); proxyConfig.setAuthenticationProviders(providers); proxyConfig.setMetadataStoreUrl(DUMMY_VALUE); @@ -207,10 +220,11 @@ public void testTlsSyncProducerAndConsumer() throws Exception { } protected final PulsarClient createPulsarClient(Authentication auth, String lookupUrl) throws Exception { - admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrlTls.toString()).tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH) - .allowTlsInsecureConnection(true).authentication(auth).build()); + admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrlTls.toString()) + .tlsTrustCertsFilePath(TLS_BROKER_TRUST_CERT_FILE_PATH) + .enableTlsHostnameVerification(true).authentication(auth).build()); return PulsarClient.builder().serviceUrl(lookupUrl).statsInterval(0, TimeUnit.SECONDS) - .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).allowTlsInsecureConnection(true).authentication(auth) + .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).enableTlsHostnameVerification(true).authentication(auth) .enableTls(true).build(); } diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationNegTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationNegTest.java index 3466662e3c319..e8bb128c8c190 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationNegTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationNegTest.java @@ -22,6 +22,7 @@ import com.google.common.collect.Sets; +import java.util.Collections; import java.util.HashSet; import java.util.Map; import java.util.Optional; @@ -91,6 +92,7 @@ protected void setup() throws Exception { Set superUserRoles = new HashSet<>(); superUserRoles.add("superUser"); conf.setSuperUserRoles(superUserRoles); + conf.setProxyRoles(Collections.singleton("Proxy")); conf.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName()); conf.setBrokerClientAuthenticationParameters( diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithJwtAuthorizationTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithJwtAuthorizationTest.java index 6764a99a9d1b8..f42cbe4c30e87 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithJwtAuthorizationTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithJwtAuthorizationTest.java @@ -25,6 +25,7 @@ import io.jsonwebtoken.Jwts; import io.jsonwebtoken.SignatureAlgorithm; import java.util.Base64; +import java.util.Collections; import java.util.HashSet; import java.util.Optional; import java.util.Set; @@ -57,7 +58,7 @@ import org.testng.annotations.Test; public class ProxyWithJwtAuthorizationTest extends ProducerConsumerBase { - private static final Logger log = LoggerFactory.getLogger(ProxyWithAuthorizationTest.class); + private static final Logger log = LoggerFactory.getLogger(ProxyWithJwtAuthorizationTest.class); private final String ADMIN_ROLE = "admin"; private final String PROXY_ROLE = "proxy"; @@ -86,6 +87,7 @@ protected void setup() throws Exception { superUserRoles.add(PROXY_ROLE); superUserRoles.add(BROKER_ROLE); conf.setSuperUserRoles(superUserRoles); + conf.setProxyRoles(Collections.singleton(PROXY_ROLE)); conf.setBrokerClientAuthenticationPlugin(AuthenticationToken.class.getName()); conf.setBrokerClientAuthenticationParameters(BROKER_TOKEN); diff --git a/tests/certificate-authority/generate_keystore.sh b/tests/certificate-authority/generate_keystore.sh index cbddd53c456af..faf808324b0d9 100755 --- a/tests/certificate-authority/generate_keystore.sh +++ b/tests/certificate-authority/generate_keystore.sh @@ -31,19 +31,30 @@ keytool -genkeypair -keystore broker.keystore.jks $COMMON_PARAMS -keyalg RSA -ke -dname 'CN=localhost,OU=Unknown,O=Unknown,L=Unknown,ST=Unknown,C=Unknown' keytool -genkeypair -keystore client.keystore.jks $COMMON_PARAMS -keyalg RSA -keysize 2048 -alias client -validity $DAYS \ -dname 'CN=clientuser,OU=Unknown,O=Unknown,L=Unknown,ST=Unknown,C=Unknown' +keytool -genkeypair -keystore proxy.keystore.jks $COMMON_PARAMS -keyalg RSA -keysize 2048 -alias proxy -validity $DAYS \ + -dname 'CN=proxy,OU=Unknown,O=Unknown,L=Unknown,ST=Unknown,C=Unknown' # export certificate keytool -exportcert -keystore broker.keystore.jks $COMMON_PARAMS -file broker.cer -alias broker keytool -exportcert -keystore client.keystore.jks $COMMON_PARAMS -file client.cer -alias client +keytool -exportcert -keystore proxy.keystore.jks $COMMON_PARAMS -file proxy.cer -alias proxy # generate truststore keytool -importcert -keystore client.truststore.jks $COMMON_PARAMS -file client.cer -alias truststore keytool -importcert -keystore broker.truststore.jks $COMMON_PARAMS -file broker.cer -alias truststore +keytool -importcert -keystore proxy.truststore.jks $COMMON_PARAMS -file proxy.cer -file client.cer -alias truststore + +# generate trust store with proxy and client public certs +keytool -importcert -keystore proxy-and-client.truststore.jks $COMMON_PARAMS -file proxy.cer -alias proxy +keytool -importcert -keystore proxy-and-client.truststore.jks $COMMON_PARAMS -file client.cer -alias client # generate a truststore without password java ../RemoveJksPassword.java client.truststore.jks 111111 client.truststore.nopassword.jks java ../RemoveJksPassword.java broker.truststore.jks 111111 broker.truststore.nopassword.jks +java ../RemoveJksPassword.java proxy.truststore.jks 111111 proxy.truststore.nopassword.jks +java ../RemoveJksPassword.java proxy-and-client.truststore.jks 111111 proxy-and-client.truststore.nopassword.jks # cleanup rm broker.cer rm client.cer +rm proxy.cer diff --git a/tests/certificate-authority/jks/broker.keystore.jks b/tests/certificate-authority/jks/broker.keystore.jks index 3495891ce9bd5f78627c21d43c309838910e526c..6f2df055f26add400c012950df1967fbc2e2581c 100644 GIT binary patch delta 1942 zcmV;H2Wj}u5zY~i8wG|*RQ+m^ARK>H9L|`JM|M34y$VJ)h$=LTLjpCI69g$M0g@;& zUh97nNdl(tn(ASb^rUYBAagD{>Q&_Mm9{h_l(Mqb3wITU%wVvDS$%&dQ2iJO zdox>JR}>-|W(2jVwJt&Y;q|tAcgN?hJy%6hel1FX6U*&2F;-Zi&D$rP(+1~`GK~|> zIJy=pq{?){G-!P#$dMycR|M^{Mh5+(MQm-)MXwx5%+SLft@EL%rgaNx#ZF5+Zn^JY z^Gx~q7Azyd4AAb>qu-7nhHrnZ8io?iw7n6-HcMYz0V*1F(ySHZC3jBE!I7|z9CZ2w zjVn#%(}X}AVApdN<&!b{xjcQeN0{KNi(=n=-V7D9@;KMSm&vDx+7zRKH+yE1q1CWaXcOA^rpKny7zPOJSgu)G7Vd zw}^d=oDTpiE0!zuula8RTQN=-#&HvaCn4QZnn6eyWf{sf(!6Ez(}lV3`s+wP3$xJ? zS)}*-Pb#Gs;l&Lvq2hSVim@Gq-C!Q9QXM3S4Ss_Gib@XOEsX6Ya=mGRP^2novH~vl z1ZhBzV|Dcs$Apxh&x?QS2?DocWIh;uFhCGk<~fmun&1;x!(^BnoE};2L_u0bxViMk zjC;|d4^gYHQ_oyWF_0E&iT2ggf_B*n&y-7tcIZ%PC^Lv_a&06ItMVa09?!(oE~3CHPjm-ziQ!`% zC1l}c4B#Qb!IH&rS{9_^Ll&`udtC#W5->`D2WiSdm{yR+S2C26C0IZjS|K2yX1)%p z?*x3<1-?fX^` zP=1_KO*vfY9D?(NRTMe&_f@(jIPGF#nSnpghKOM|TPfakcHnDfOH|LRVt< zgNaHN?U4B8@FCNJ|L+k<)*nbjP|)pjIn+a0RRP9E;TSY z005IR1riAI55_oi1L{(fOa+7-GcYnSGcYwZGBYw-7!NWrGBYqSGB7YTH8L|Ylk5eO zf9SX0?lOp&4hI-P{n+9($bM9Yo7kg!M{en%jCAu4yQYIf5qHFzt5bAjM*e}1as!tO zH+XLUZe(cdp0I1Kd1_GM9vwN~Q40i)iA&@HYvdBsK}Rf(bubhf?J|?5{&RiY;M9}Z zj6_w^Q%Vdj`ITQ!%P-|Y#k@0!>)28be;HLa0x=JP3&Kw|W2Oe;s8s9WRY;LB!~w`MSU5=!6gDv`1A^+ zdukEGZ@JnApe;RRM~*1c7>Y%^d3(Cm*eOj;+WoQ~>$X`N@yUwntbKCajpsvsf78bT z0|5X5qaiRKFdYU1RUHll76cTCI$52va;#+g^No4!RQ@2tgGnSX4F(A+hDe6@4FLfQ z1potr0RaGUi0jAl@`N?_tGqnUz7hG!Hnkp;md|DeO7XV!%$K>%lMjk!Vuj27!DD_n z*#Zn}1UP%_g61XjjG2wmAja!ne|4U=@a(Ocb|OK&i0Q+BJ&v03O0(iH%HU`@jeO?Y z-2Fs#W>wRtkL&v=hP|#nlb!ZA%)WpFLTq0_i1hnbTyTXyW^Hjno{F%|r~-%-TX1;2i*xp<}J{l=Kufz delta 1942 zcmV;H2Wj}u5zY~i8wG%xPw1wRARK@CTKzU4mBV8CppD$_B8-T}(K``~{*Q3$7W17vfm+j^xs`uFvdb3`U;^RE?G^HB{yq7o>KXxff9ZN=z19dhR)t19 z$U91>-PnS02w$_R&n0U+!LxF73O@W#>R;?DKo44wWIJ34I8Ow}rvLyp;qnw_qy4^R zQd@)g)WEhgqw?ay^L`!z_;lfr8D2~A4@6(J)RALxs(9=o*{4xIbNPQXDszmJWhhqA z;aO)|R3rZ8aOZ*gMI1%jGZAuX1BxIGmE8uB2+@>C(-!Fz>s?e{L{#NNPS6yB_CU$8 zl8cYE!PRj+-@#1sivk}Ogvg(fIx!=~`$NO9`xwWQ=gxkack81e|vHzUNA|FGYmxfFfq6kyoUi3d(I}vtB zHOdC5?f$Fpv+94{b>`D|IWh35cw~PN1B9EO08w^o3z_ifLNpe{2g;!0@W1h!DWlzG zYOsMx*v+J~ZDF;5GVY+`HQ}SGMuq4&3{sY6oX>)4t zom{26P=uWKJ(F;6eLIe9Y&i$s4%;s9J`k|}zSMR(r+;!XG?Lx-&B%9B`0`#2u}Yb8 zX(qM${ezp$WvZtmYa2cq#O?PXuIjOYZWMz|SGs`TVklIiY@?>t-{+%zaTKkoBi5L+ zQDulw@C$#H?ce?Sv)@#UJtJ~&Hm*gbpUpV$h5t|=JV2tMverzxOteH$mn+_SFXUV%nH(HR-;G&Cbv)F+UCg1(+L-RTI+(TAEamd@`L)lCWc30h)Ub1{^z!q0&C;AsfMW-`K@a+lge+)40fV0%EbYsx-f6zP9Mw2*~bL~-jBEDAy?$M5$)4>J2H1bZX zrqhzA>4SjgdWILF53BwfRgqmY8waGnr!HeHH0004oNmT^Nku4ht z@(;#1a|7y9kx+yjGcYnSGcYwZGBYw-7!NWrGBYqSGB7YTH8L|Yk?@j#=(peQGKiQC z2N*&9*y1$EepH5=*rR(#Zt0lT{(+Bj1D6Xocy9l0WN7Q2 zuxqY)YEa=G9Xa1o3j~gdOXLD;0852rk#MdghzCbSeHp{SB?3_R^a`VUY7xV4x!MMx zEj?vNjwsR?ibcD5d%D%wDNRn={jwhGwpknT$%^W%eRAB5=R0RRD`Aut~> z9R>qc9S#H*1QdumS)H&LNQU&No)gf;i8ygbgn5&6kBwH}j}&t?Wn@wWBMm$}W84~k}Dh0FcHV}3Z<0t{;eID71Z z<|XrtnT^pP#_L{xb)L5H?5&!1B0;@~>BE0Lj+*gGv*Iwy;AlCGeCFHS{X}(URnw=B z>-#8%y{JU3HIK?CgdEuw68u2SeEjq{BKEr5@&%W{CiTm}?Yb7`GYmI62eYHfRCS_U TA*~mDjFxr~!~}Q`DH#}yU=uk& delta 702 zcmV;v0zv)B2hs;n{_Xzl000020000100002025+zZ);_8UvzSHb97&0Y-wd~bN~PW zfSOP3(f|cmE;TSY000AqFoFYuFoFVVpaTK{0s;vDkSu!2rK8!vkw=6aGB7nTHZe9e zIWjp~7!NWrGBPkUF)}eWH90alk>`?s%@Yj-R*6LicH!(*sdkZ9(3>mjCx$ONn8b`4 z&j2J(Vod??p|V+1G>G3eb9f7ApzwmzyN05dn9W*32hVJRwxdQ=DA{I?m;#X=@huoJu{s-B3?PsjrYg*F$l6v*)Kany-%Ct?(Oic( z040+hi`<78bf*f26Pc&OMk#uNjH_CMQm^6`)?rse*-Q*}U+%*M?)yJ-x=B+uEoX+) za_{Oj-oeRr_~I{*eF4&w1>#o<7B8CgjR|n7T0!?UyD}m(F@EOskZW?0>|s39HJD9a zvV3d67FTH}`WHn-r!z_L?FsBECEo%A0RRD`Aut~>9R>qc9S#H*1Qd0N+Y<6W0lRZs zBV2FxBBK8Toxzi#0u+B0N@+)A>+)40fV0%EbYsx-f6zP9Mw2*~bL~-jBEDAy?$M5$ z)4>J2H1bZXrqhzA>4SjgdWILF53BwfRg2mWc00BV>|m9mSb zIbM}nDMov|0ovhx``Hvj+t diff --git a/tests/certificate-authority/jks/broker.truststore.nopassword.jks b/tests/certificate-authority/jks/broker.truststore.nopassword.jks index e4d6dff0047e28d0e089032e39d439d4b0fae11a..75c3fd8012f9625d6d5160f8905235c314b3ccdf 100644 GIT binary patch delta 685 zcmV;e0#g0b2gwH{{_Xzl000020000100002019++b#rucbZ>HH0004oNmT^Nku4ht z@(;#1a|7y9kx+yjGcYnSGcYwZGBYw-7!NWrGBYqSGB7YTH8L|Yk?@j#=(peQGKiQC z2N*&9*y1$EepH5=*rR(#Zt0lT{(+Bj1D6Xocy9l0WN7Q2 zuxqY)YEa=G9Xa1o3j~gdOXLD;0852rk#MdghzCbSeHp{SB?3_R^a`VUY7xV4x!MMx zEj?vNjwsR?ibcD5d%D%wDNRn={jwhGwpknT$%^W%eRAB5=R0RRD`Aut~> z9R>qc9S#H*1QdumS)H&LNQU&No)gf;i8ygbgn5&6kBwH}j}&t?Wn@wWBMm$}W84~k}Dh0FcHV}3Z<0t{;eID71Z z<|XrtnT^pP#_L{xb)L5H?5&!1B0;@~>BE0Lj+*gGv*Iwy;AlCGeCFHS{X}(URnw=B z>-#8%y{JU3HIK?CgdEuw68u2SeEjq{BKEr5@&%W{CiTm}?Yb7`GYmI62eYHf`LMXf TGu4Q)0S1WScn6opgCYEOhR!_& delta 702 zcmV;v0zv)B2hs;n{_Xzl000020000100002025+zZ);_8UvzSHb97&0Y-wd~bN~PW zfSOP3(f|cmE;TSY000AqFoFYuFoFVVpaTK{0s;vDkSu!2rK8!vkw=6aGB7nTHZe9e zIWjp~7!NWrGBPkUF)}eWH90alk>`?s%@Yj-R*6LicH!(*sdkZ9(3>mjCx$ONn8b`4 z&j2J(Vod??p|V+1G>G3eb9f7ApzwmzyN05dn9W*32hVJRwxdQ=DA{I?m;#X=@huoJu{s-B3?PsjrYg*F$l6v*)Kany-%Ct?(Oic( z040+hi`<78bf*f26Pc&OMk#uNjH_CMQm^6`)?rse*-Q*}U+%*M?)yJ-x=B+uEoX+) za_{Oj-oeRr_~I{*eF4&w1>#o<7B8CgjR|n7T0!?UyD}m(F@EOskZW?0>|s39HJD9a zvV3d67FTH}`WHn-r!z_L?FsBECEo%A0RRD`Aut~>9R>qc9S#H*1Qd0N+Y<6W0lRZs zBV2FxBBK8Toxzi#0u+B0N@+)A>+)40fV0%EbYsx-f6zP9Mw2*~bL~-jBEDAy?$M5$ z)4>J2H1bZXrqhzA>4SjgdWILF53BwfRg2mWc00BV>|m9mSb zIbM}nDMov|Ob9WR`y;c7c)5aGO2%$9LOYoNjwNwH1mu9)%iO{&yvPQ}8HM zg7^KE8LSPl!T7<>%t%^D@?xsZM^cn`xdDb$YKE2n{4Q2&4-F{sKkiYcOvHn_wCz;V zTZ=z@Ub5(B9)AoStQlVPthTrfWiGGj*7r)Sog2qiFMY}9{?7b}F@ac>2oDr1LQehw zsNwIpkBsKiO4y+^m?{Q^Rll%M^AS4GY9c+avJA7G}V5ITI3^N&vf+~azEP|ouHeoJss`tTG!nhKSY1Ju7^_m0*PwfXQIZYBqa6YoTX`1Pvhq7YpvK_}` z6X<%*`hWKHE;6MZ7G%kcj6`D-B2XAZy&vaqpn?%I$p_!RPw>-Jsi2!}$$xh7WEjZm zG0iL7NWn>$F)J%%4d|tA&GJsz@&pRGJsj6RO5<3jpS+(QSPsecuogHmyxSks|96>j8OvIYpX!=<=Dvb@2G-dxqlX1z~}FOI!n!WWKg?( zTnv}^@sBg6@ylkC09apGasAx%Fdca53{-rxgnK(KDdRTV%7lq5YL~N0@*~Bpa36n% zHGgzNOkezD_tYwq4qaeUcESMkTD?p+E7?ZrW>Ab(jDlRRIMPq^kZ@n3QT6~HTDntY43{@j@nm$#whL~ughw#KGd0X>G z>Xu}b9TU4^$`@(S>+-2-+7W6YVe$<%IDgxvO7#g8-4hfCfUrK*W*5C(q$o}Kp-(q3 zZgDDdO2fHb^Gz#@cqJM+$|;2)1Wp#{VwvN_T=zdaZe5OdNTbztC5+F4{692L=|NCZ zm6SCXk4rq-EV<0AUzUW{rm@sx`&2iKCc;#8If05UUXXxl7ZIYKrekM4o zGwyA8Jf2=!gJO%kyAqWzVx}dFDs?DUvOEO9&CjY35~~PfjRzqw9vLe#sXf>kZWWW- zl$zHau`?(6&Fk)m{E&|2kw`so3$|yuf?*o-CzjMHSS+?cX=B za|~h`;LFP#pp_-_xQj@W8#C&4 zXHlu4?@E&;>~+$b?yISAMTZnYr#-OEpG~F=DYl#{eA^-h>YBc6F(p~VL4T1YD$ttLv0&|vcUN(E zApwB%cV4m6JFK5>AVKYgVEx+xjo3w`Yp4E5+!D^3jg}j)7)ITjqdC9ay0Up9ztu*> z09mrBBUoBaMedJ_GLAb6Bw-z?l0z=a4CNDU=@_zzWTSt~-k(HScFni)4$prA21yT163Uk1QrAohEOFvm%ZL>VL^J9Cl})DIiV`x zFbxI?Duzgg_YDC73k3iJf&l>lp!bD=j-W3hX=1pubG+=hwHLgHekK}IGgN~u(_Q#f zlD1jXaxo(QH+-GlhidMI0{?Y{&?_sikuZ!w95%=p&420hF@PORBDo~;#Z;b(KWuR^ zyHJxope5{kD)zh%q9>r58klFK!5I0z7JX+Tk&kgO5Z>Lx=*<=!Kg&T8ixoykf$r`Z zG%vQvaw&CUU&1_*;6sMmj_4T5v{%O-7Ey(kbzvylwY0&#U^#g90d~ca=hDyy7DE$V zR#AK2MN|k|`4t69>qbf4aoBmW2#g)*f+&*&(t+239DLlAI)%Cwg5zmYG%r%AmrUd3E8dE9tyrHDpzm0fl|-v*z>J9D&a0vKb*m6-ql delta 1941 zcmV;G2Wt4y5z-Nm8-IYBPwGhk00jduf&~9C4h9M<1_1;CDgqG!0R;dAf&}aPMI@4% zC_#XKWVqC$M*Ga*oq1_dNf#_l0nj5)BPb|~AXei?pyM))e}k3BvE8$Ulj-NCl>+%d zZM?08h8G_i#;gMnXJhacTsYhY6eUpOS`hf7RV{)S0k-qw~{ z-0t1XZ4MlpN~ebR6@x8sOOPK@mr)f2T=@-q-%uX)PAxN%?wRs(-UP?(qov@hqngQR zv?$ERR7|PME)d90Fr5Lc${E&^o6+bj=QqZ;NU+L3VWG3rag3*ZIF)qlONJtnU%aVJ zJik@XI@TfU%zt&*x;SFo(*PrTK^_2K!ocs_Of1lTuGz)F-ixIBq3*cgg7VGJBuV~_ z?dCZfli1XSzR?L>16M{(zAdqel_~FV@S7!|>xdKn&3;A-qtu67l6-tV$A6we7G`y= zWI-Htn^@Nc5zW+6hG__hFNyt?&JUrbkeHwSMnr^{EeVD8E3WqK|Oheho{ zO3V3aS?%vc3)-BQWkS&O4>_Mcap>tX`3oGzvV!)NT9F}Oxn1W0*IR5n!rwQA_rMrg zxU7*aoQg4dp5tbQK~TS1-f{|0jSv7CMeT zI`ejZ5|a;dYz;baRl6O%5Z@Xs^g%mED`l&41Pgrn!SaI-K6Ww=|quAJX__ztco5+71 z34hNmP~*vnDUPO1S?rIeP$O2o38$?sguM#K3#Wq=d7}rZYLmOueS)42^GU2jAGG^( z%MIg*-voCq#uGJPOlHLvtnNt`Od$ls><+%EIz@HXeGOSTPd;2*G02Di!OV=Ndq9?8 zUy}}*2=@3KOoZ-kEeMuSBe8B&NQ0u^Eq`N=s!R{gi^;S1^S)%WfpsL$m^vmpMng6O zu%Iy#82}^=iG1AsETrofssSRX@weQai#(EaOP;dHgW>6$xM1J)>Mss2N zM`)h|W)tDRbc}SoiI#M&A$JcQp!ZELvZ7pA1`Uu1QjnZM0zI(C6w^Li0-*12=zm^D z;&C&B&6q(dxln`5e_Tp;7g7rkD|;%F&cc~@95S15AJ_#0K|#8+<{0HCRWDm2M^%br z9U8jck#^S%>|Adovm74gK3^?yAWdRAjVW@un$P;!n-gm=T|I2^+zc}l;~E>iAm)5k z9L=-*sk{Xq6WJ#iry{$Iz^hK&m4C#ZNPanuwAZ+j!#oO_bB7WKz-e)rs591p_B%Xm8n835GF#;_P!mvaK(;K<$p@1@rh@GdK&)fCn+{z#;YlzpQ()Osebvjvk6FaG z*1-gc=ZrG;VWEiRPjHVlytk#a)gx3{hXzQ60Kt4FJG|A^zr=qK3AR34T2~eHwOlYp zgCf~Z`st>H3-BN!OkP3J^t?y-DF$OPsAaLy+_vJM_i71^Mp_)sF@90hXpz-UD~BXP z4+o+k$iEYWyT(ixh5_yit*pJ!3R=ek|2a@t5JK(JD35|Y9%|KWz#SX1_>&LNQUHH0004oNmT?7ku4ht zj`{i05LRf+kx+#kGcYnSGcYwZGBYw-7!NWrGBYqSGB7YTH8L|Yk@1s%w`{?83~-cD zPW?Od4tw*9G(qD2myb}1beo`+CG)t8NRt~g>UC#PsiE&mlO^nR(wgqEx96~CtL>-Q zem+;Ix*p8w8MgB&NwzGVYA&zfE(rxUrj5dJSBK`Gn2CHD3%*5%6hWswu+5)MrVJ^z zoGN_VA_nT3zHBiiS;RqqktHh7n$)pi?T&X>ad;sCfb(}=vC})OpKc&Q?S)|d+X0Q( zMWkz|{z%*s&YF#u8?P8f-J7F1zudaAc_P2nM#KPFvZ*6jT24jokBc&nI|?LW9jTH- zF3Jq$6K?4kvWR4(f6U&WL|JytxAG3pe?cGgEf^N?+^8BEsqs30jsF4z0RRD`Aut~> z9R>qc9S#H*1Qdo)B|VqD-fUq(dX^^_;_Eq~D&Q~;1_>&LNQU zE3c6-j6obW$QaFk>GLsw9ZVv*B=W^no{B$gaWT74lRcm%?0YKqybhu#pqd((XQaUx z`Mwr?XCjf0aWD|x-NWe3792mzK@p1;Mn{3}?in;Mw#jlSbz)z_Jdxl-hT4wk7|OI) z#~v0@g_d<;DB88O!M$KPc=iEy#gXUI&;}Mm6J1tOd)`G<2wV9T1xo8iN!@YSd9etL z9q586lLXR%*MS^-+>|m-4mxty)j{u`*lbF;Ejwa6q;!(8&jrKBj{05J0SByPFPOIN?b!EmL6G zQ+?IT#E)6Tx7NV~iRX+m_F<)xX4l5DB(E zTUu8Y^tD_tMuQ^RPWtJlg$wW?B1~RE(e%7W_$dZsF{ov+(cHG;pZ97BjYe7=&M|&b z)M$~_PAi8bLJtR`AjrQHguBK}7lr}u3$3iZ&9R>qc9S#H*1Qd|~^~J>@t5JK( zJD35|Y9%|KWz#SX1_>&LNQUHH0004oNmT?7ku4ht zj`{i05LRf+kx+#kGcYnSGcYwZGBYw-7!NWrGBYqSGB7YTH8L|Yk@1s%w`{?83~-cD zPW?Od4tw*9G(qD2myb}1beo`+CG)t8NRt~g>UC#PsiE&mlO^nR(wgqEx96~CtL>-Q zem+;Ix*p8w8MgB&NwzGVYA&zfE(rxUrj5dJSBK`Gn2CHD3%*5%6hWswu+5)MrVJ^z zoGN_VA_nT3zHBiiS;RqqktHh7n$)pi?T&X>ad;sCfb(}=vC})OpKc&Q?S)|d+X0Q( zMWkz|{z%*s&YF#u8?P8f-J7F1zudaAc_P2nM#KPFvZ*6jT24jokBc&nI|?LW9jTH- zF3Jq$6K?4kvWR4(f6U&WL|JytxAG3pe?cGgEf^N?+^8BEsqs30jsF4z0RRD`Aut~> z9R>qc9S#H*1Qdo)B|VqD-fUq(dX^^_;_Eq~D&Q~;1_>&LNQU zE3c6-j6obW$QaFk>GLsw9ZVv*B=W^no{B$gaWT74lRcm%?0YKqybhu#pqd((XQaUx z`Mwr?XCjf0aWD|x-NWe3792mzK@p1;Mn{3}?in;Mw#jlSbz)z_Jdxl-hT4wk7|OI) z#~v0@g_d<;DB88O!M$KPc=iEy#gXUI&;}Mm6J1tOd)`G<2wV9T1xo8iN!@YSd9etL z9q586lLXR%*MS^-+>|m-4mxty)j{u`*lbF;Ejwa6q;!(8&jrKBj{05J0SByPFPOIN?b!EmL6G zQ+?IT#E)6Tx7NV~iRX+m_F<)xX4l5DB(E zTUu8Y^tD_tMuQ^RPWtJlg$wW?B1~RE(e%7W_$dZsF{ov+(cHG;pZ97BjYe7=&M|&b z)M$~_PAi8bLJtR`AjrQHguBK}7lr}u3$3iZ&9R>qc9S#H*1Qd|~^~J>@t5JK( zJD35|Y9%|KWz#SX1_>&LNQU6{p#5%x3g diff --git a/tests/certificate-authority/jks/proxy-and-client.truststore.jks b/tests/certificate-authority/jks/proxy-and-client.truststore.jks new file mode 100644 index 0000000000000000000000000000000000000000..45a49018d8da3f7abb1b2afbd7e99bcdccbd3ba9 GIT binary patch literal 1891 zcmezO_TO6u1_mZLW=qb=OwB7{U|?+X3}w5=z#5@vYGBF0z}#Wb#N1-g#FVpunTe5! ziIbu4=g&(5VHsx(c-c6$+C196^D;7WvoaXu8wwcku`!3TunDt==4I#Qm*>GmI50%m zF+{j9M7R+m!UlpMlbMCNAkHi;PAxJ}5a%^AHZU?YHZV0cGB%2m;5RY^3K$w0Kn3XU zq$WlsH^eBWg}n%m}R)!bHMdo^ocj-hJg z5yy$DS{G(r*_iOQuROf4LXolIb9vmx%hqe==P5Y8ZB6)lo3ZzX>yqr{|2*ypotxD= zO?thA+uhlVE%)EqwW(5R|5djm43V2wDn~^5xxVY~HtMtHQc09uImub?6wg!Pyq6N2 zI#L$bpSd^RC9>@7_D_80>mB94=t_uvxU)h^a^**>-v3O@j0}v66%FJKWP$N1%f}+d zBGMM1YCC=Jy_`hHs%h%tk6v3Y)Ovs%h`_W33`9nT1>aj6`xfXcWhU*|T)gMaj;-Q* z+UwM$f{jC(buY*M2${4k@^X=((m(T>xp&*M-nB9PFKxM?t-XGtL64)1*$IiWFFzYL z$oeSlRQYr?WL}qjPNCuMfXTKCRNqux83S{A#rL;_;ES0z}9J{i5j=I zZaKI&!Ls5TW7*M(&o5nI6LS`h4GXNk=gJZNQQ zQKsh36sy)#>2b@(|3ySh9Dxta5 z0Fg`UfVs2^m`jsjxpd;9&>3p}7v#`F(jW_#CuxvN`3!gi|Q=9WQQ*f4Sx|CFR)D zB@-j3D^C7<*KYBYxFceVu2japTzf&BXZEDd=Y{)~AI)L4^UGO#Ez$6Xy=Awcu&S)V zQdI-bOouC>mvaBgW6PV{`tF}BD7q@1Hb?ZsQsdH9>L&x1qNONs-ZU#%Zd&QUu|oWF zQ_b}&OI|ogdRy6cUS4@bI3%9)LEN!Lu@jGfmh+CAohE81uDr`%Iq8g6gs4dB-)Ghu zM|d7Ab?ds#a4)L)x=!L#hy4z-p6I^FEZ@D3y-iq8G(+yX|CjY<)fHY3?(*Gq4>)zB zd$ra)CRWEc7RIyI1jJ5s-OoBp-BytE`XtS)6CV~OGADb@OFS0nocj1PtKp)VmTxWw z20Z1S@li_U4$F)9^L~~Mw~oFr=d+l-W`P;|-pxl(i7oxh|L#u>|L1L5xbu)ugk6ON3z59Vue@VB{e0#td6vgrO Zx8`3syLB6TaPH})I}R`O2y{Br4**4GmI50%m zF+{j9M7R+m!UlpMlbMCNAkHi;PAxJ}5a%^AHZU?YHZV0cGB%2m;5RY^3K$w0Kn3XU zq$WlsH^eBWg}n%m}R)!bHMdo^ocj-hJg z5yy$DS{G(r*_iOQuROf4LXolIb9vmx%hqe==P5Y8ZB6)lo3ZzX>yqr{|2*ypotxD= zO?thA+uhlVE%)EqwW(5R|5djm43V2wDn~^5xxVY~HtMtHQc09uImub?6wg!Pyq6N2 zI#L$bpSd^RC9>@7_D_80>mB94=t_uvxU)h^a^**>-v3O@j0}v66%FJKWP$N1%f}+d zBGMM1YCC=Jy_`hHs%h%tk6v3Y)Ovs%h`_W33`9nT1>aj6`xfXcWhU*|T)gMaj;-Q* z+UwM$f{jC(buY*M2${4k@^X=((m(T>xp&*M-nB9PFKxM?t-XGtL64)1*$IiWFFzYL z$oeSlRQYr?WL}qjPNCuMfXTKCRNqux83S{A#rL;_;ES0z}9J{i5j=I zZaKI&!Ls5TW7*M(&o5nI6LS`h4GXNk=gJZNQQ zQKsh36sy)#>2b@(|3ySh9Dxta5 z0Fg`UfVs2^m`jsjxpd;9&>3p}7v#`F(jW_#CuxvN`3!gi|Q=9WQQ*f4Sx|CFR)D zB@-j3D^C7<*KYBYxFceVu2japTzf&BXZEDd=Y{)~AI)L4^UGO#Ez$6Xy=Awcu&S)V zQdI-bOouC>mvaBgW6PV{`tF}BD7q@1Hb?ZsQsdH9>L&x1qNONs-ZU#%Zd&QUu|oWF zQ_b}&OI|ogdRy6cUS4@bI3%9)LEN!Lu@jGfmh+CAohE81uDr`%Iq8g6gs4dB-)Ghu zM|d7Ab?ds#a4)L)x=!L#hy4z-p6I^FEZ@D3y-iq8G(+yX|CjY<)fHY3?(*Gq4>)zB zd$ra)CRWEc7RIyI1jJ5s-OoBp-BytE`XtS)6CV~OGADb@OFS0nocj1PtKp)VmTxWw z20Z1S@li_U4$F)9^L~~Mw~oFr=d+l-W`P;|-pxl(i7oxh|L#u>|L1L5xbu)ugk6ON3z59Vue@VB{e0#td6vc8O YwVx8L~?o-{(2sXLo&f9Rvb__XPY~ z(C~<`nAp84+1eS<2Z5jfmJH!x5&S~x`~U<{1B(Fw6a*ndSYz!^D;aNelr_rp%dOSF z`;6uq|G7elU#q^<^x+2}vbZ)$p7pI7QcmhxPv}%#`@S})krmXJKa02+L1^99#kKiF zz-4Gr5ig%c9CN7`i0$5Sf;f@tgfVlO>pT3t=!`U?u?x?mWS&>(UZ00_c5;v*YzK(A z1vuC5AM2?pDWCn`(th){4rHD9w21Qb!p)($c-YwYe!CSp4GeAiszcP?Tta_w%1;KdK}R9 zJGzH+w#oxPUpSvnd>5H#&$BR*z-DGYARaP652$EF0sBgAEAUS9HG#E@5i@B+M4J$k!ia9?V=C)llk7 z4ln5L<;!2L;TYz+ttv@z|9g)ecDpsI@yw8 zXs)B|?b3f)csZwyU?Ol=(0oOqES!ToO@9$-$XvQmV^x=5K=l=A*E#JTU``v2W?U^I z?<)Qp3SU~Zt9R*ZQR$-f{0M9}i$WAjy5i_`%x#Kik*BD-Cr@f888r03C6^GOc-<+Q zdRLq-<=;w~M?xF3!UB$FDGxRp|BzK2g5z6)K9$GR2ab_gHh!C|fk$ZulhyYcF5`y! zsdwM1o;c#nsS#O>)R+;Il=GoUvd;dY(zYT@nJ9ua8=1udhvD)1jkg}TmQB0z$JZo7 zDARR>O;0bFN2z{uzq><7TtzK29PLu4!$jrES29BlRm9!e)H$bo`&nw31-G0yHf9*p zL|x>n#dob$N%SRE*?hb@&0wRB$&zanXzzDk=_a)qVcMR{BzjOIQZj%dVf@V zq9>}drj#B(TRkjvCSSU)TcRN6?|c zZ$%RbXP#x8^!+8yktFXb>({@#PjJ%D=TF{{MYgRu9--|X_D*RP?;lq9j=O@ze7W zim4Q}_HgidvS8s(E{!sitGVoY)V1T(oq@Csn@}{f$Kcya=81(VmZW@L!r(bevzhln zF$Bx#4X!}Zx1DwL#;KS|88TM2fmBwg>y-S)&q&Pu0qKU-M|puGiPH8B@x#4evE?6p zD(zhv;AhKE`FloCE9|NIBUf-?<{a(A$t|Z7B-!i8^tWMHrX#Z~u;$)^MBH-#5f*`|4Ox{1F!(=#)n<@=5 z3`Edh(h6bua=(4`X5{hN6VVcmC^`T63t2f{F!|s)IXXIntR{a9a_i)1mY+)5D1OX` zA~*MX{Mzg^=ybDiL+OH?gGO(hV_s}y;_QH>)rC6F7yq=04834~J!IM6x#+5)i!}g& z_!6*WzBnuy>{G}G2Ebq#D4pt@t>o|#g%#n4sU@FE=86D-FcgFh(%QEdn|#g?esKY3 zQZOkjn)F{H_!ANMi3t5fg#SknSops>^NIf**H|=CL>q_I*1}~}h!uk*EBK-Zq(-flxx zl8Rc!ZVI^~`qyh0KE;dQtS8t5+!o^^(N#9w%=bxls%_fw!Ga1Pl;k(xTL$kOI6#1; zKXE0z0_7GcB@UM@+NBh&=NC*I6uNdDi9C9WMd|icY3%S<-m#iK7TDFsC_E`)>As5B zQhc>Mj^vD-%=BX{mZZC7DL&*)n-phywM$WlV?9~bACd69jFhEI^h5JcpeFVq)guJ0 zvEv4*V&aDtu%(Bw))!5Ood-jC2f%y)2q;#>9>6N>ZBYRMk%CAxq|J7PM|?sOpGbZ! z#YH_;?!uS;)5N`t+A|RV>4lf;#+nLNAlc;j(czM1QyE)BdU)^66D`m6Y)3F^yE(;d7;ycP2qI?v&AA2#cn4)7;NTmM=!Xt`SHU z*N_ZEjXH2@^{&R)%uOT4t?=DrsWi0$Fw|`AC@!}O@0o5vhvq69i@`=SRD<4sr4smj zZ3+ks#}obwY^WAB$6#&15x*#${Y@4#1zGm)w>P*k(Xp(LIGR^gs3*|!qN7`?lqb5e z6EDhcu$J+@R*JrLr4w8c(86$Q*>;O;ilpb#U(JNdTRT&QL i{P+#1@^3FkmCk+N@SMCpqT;2;PwtGsG|-N-8vg?A72!Sr literal 0 HcmV?d00001 diff --git a/tests/certificate-authority/jks/proxy.truststore.jks b/tests/certificate-authority/jks/proxy.truststore.jks new file mode 100644 index 0000000000000000000000000000000000000000..0e13895b1c21d47158ef56c67a8e02b54fa79b5f GIT binary patch literal 971 zcmezO_TO6u1_mY|W(3n*B}JvhCB-HAMX5lcHqTJjN(R;lJyQcq1_tI1gC^z{gC?e& z1njW}v;d`&is&7g!&3dR zmy+8)YkF?enVY4z{(&ARtNF6tLxth(&*snQtdZp2=h`mfxZHNb+4;W9cr>@o(W<$v z#P({|z8pi<$RmyuRkbe6y0S6hZC`nKVTB@N!{_q2jhC(0%+FJBeA}Av_cmki4c8^v z%l~=Y5jr=kcbfEi3Aekm7hCSXvujhO(*CP%M;IbEtyGSP@^gLH-)+=q&83nkyK<7V z-YK4^!g((xHg%*du0L~czDs1;+3lbB&euE2f6C%u7b_ab z8^{9VQ;L#2P_HFNK_XT57<`d`{|L0fzMM1vkj z8M6}-XJ39cY>@R)+NtvCXvn-S`gL|~?wzEG{-mhj_p zVu7vGN)t71ZQXKkZ-QmTH^#D~6Q5tYz$WG_92*u`eb1F6`ll$X*K0S=yM;F@H*)mI zzG%{z%yOymdZSFuoheqWyF{BFX9hj)-ao@?{^zT*J~Q16O799U_uOk3cQnSR@(!1e XzeQq9k7D@MBHn+Q{(@?=OQrw-xX@_- literal 0 HcmV?d00001 diff --git a/tests/certificate-authority/jks/proxy.truststore.nopassword.jks b/tests/certificate-authority/jks/proxy.truststore.nopassword.jks new file mode 100644 index 0000000000000000000000000000000000000000..2a2729f1c3f60f9bbfcfb597b063b849b316f76b GIT binary patch literal 971 zcmezO_TO6u1_mY|W(3n*B}JvhCB-HAMX5lcHqTJjN(R;lJyQcq1_tI1gC^z{gC?e& z1njW}v;d`&is&7g!&3dR zmy+8)YkF?enVY4z{(&ARtNF6tLxth(&*snQtdZp2=h`mfxZHNb+4;W9cr>@o(W<$v z#P({|z8pi<$RmyuRkbe6y0S6hZC`nKVTB@N!{_q2jhC(0%+FJBeA}Av_cmki4c8^v z%l~=Y5jr=kcbfEi3Aekm7hCSXvujhO(*CP%M;IbEtyGSP@^gLH-)+=q&83nkyK<7V z-YK4^!g((xHg%*du0L~czDs1;+3lbB&euE2f6C%u7b_ab z8^{9VQ;L#2P_HFNK_XT57<`d`{|L0fzMM1vkj z8M6}-XJ39cY>@R)+NtvCXvn-S`gL|~?wzEG{-mhj_p zVu7vGN)t71ZQXKkZ-QmTH^#D~6Q5tYz$WG_92*u`eb1F6`ll$X*K0S=yM;F@H*)mI zzG%{z%yOymdZSFuoheqWyF{BFX9hj)-ao@?{^zT*J~Q16O799U_uOk3cQnSR@{ZH{ Xma`KN$7IP=^x3So&x-lo@#!)E-?MCQ literal 0 HcmV?d00001 From 0f025f3fb4abf13fe2c7f290a7039ca44b91b5cc Mon Sep 17 00:00:00 2001 From: Hang Chen Date: Tue, 14 Feb 2023 15:39:32 +0800 Subject: [PATCH 015/174] [fix][client] Fix authentication not update after changing the serviceUrl (#19510) --- .../java/org/apache/pulsar/client/impl/AutoClusterFailover.java | 1 + .../apache/pulsar/client/impl/ControlledClusterFailover.java | 1 + .../org/apache/pulsar/client/impl/AutoClusterFailoverTest.java | 2 ++ .../pulsar/client/impl/ControlledClusterFailoverTest.java | 1 + 4 files changed, 5 insertions(+) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AutoClusterFailover.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AutoClusterFailover.java index 94e8026b7010e..68b781e67d29c 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AutoClusterFailover.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AutoClusterFailover.java @@ -161,6 +161,7 @@ private void updateServiceUrl(String target, } pulsarClient.updateServiceUrl(target); + pulsarClient.reloadLookUp(); currentPulsarServiceUrl = target; } catch (IOException e) { log.error("Current Pulsar service is {}, " diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ControlledClusterFailover.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ControlledClusterFailover.java index d9a2862030080..080d328e3f02c 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ControlledClusterFailover.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ControlledClusterFailover.java @@ -138,6 +138,7 @@ public void initialize(PulsarClient client) { } pulsarClient.updateServiceUrl(serviceUrl); + pulsarClient.reloadLookUp(); currentPulsarServiceUrl = serviceUrl; currentControlledConfiguration = controlledConfiguration; } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AutoClusterFailoverTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AutoClusterFailoverTest.java index a2e7719330ae1..36ffa30296bb0 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AutoClusterFailoverTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AutoClusterFailoverTest.java @@ -224,12 +224,14 @@ public void testAutoClusterFailoverSwitchWithAuthentication() throws IOException autoClusterFailover.initialize(pulsarClient); Awaitility.await().untilAsserted(() -> assertEquals(autoClusterFailover.getServiceUrl(), secondary)); + Mockito.verify(pulsarClient, Mockito.atLeastOnce()).reloadLookUp(); Mockito.verify(pulsarClient, Mockito.atLeastOnce()).updateTlsTrustCertsFilePath(secondaryTlsTrustCertsFilePath); Mockito.verify(pulsarClient, Mockito.atLeastOnce()).updateAuthentication(secondaryAuthentication); // primary cluster came back Mockito.doReturn(true).when(autoClusterFailover).probeAvailable(primary); Awaitility.await().untilAsserted(() -> assertEquals(autoClusterFailover.getServiceUrl(), primary)); + Mockito.verify(pulsarClient, Mockito.atLeastOnce()).reloadLookUp(); Mockito.verify(pulsarClient, Mockito.atLeastOnce()).updateTlsTrustCertsFilePath(primaryTlsTrustCertsFilePath); Mockito.verify(pulsarClient, Mockito.atLeastOnce()).updateAuthentication(primaryAuthentication); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ControlledClusterFailoverTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ControlledClusterFailoverTest.java index d2d31ab85c59c..570b139832806 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ControlledClusterFailoverTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ControlledClusterFailoverTest.java @@ -121,6 +121,7 @@ public void testControlledClusterFailoverSwitch() throws IOException { Awaitility.await().untilAsserted(() -> Assert.assertEquals(backupServiceUrlV1, controlledClusterFailover.getServiceUrl())); + Mockito.verify(pulsarClient, Mockito.atLeastOnce()).reloadLookUp(); Mockito.verify(pulsarClient, Mockito.atLeastOnce()).updateServiceUrl(backupServiceUrlV1); Mockito.verify(pulsarClient, Mockito.atLeastOnce()) .updateTlsTrustCertsFilePath(tlsTrustCertsFilePathV1); From 5d1fc6d5f3b5b68e9760a572172cf163a1e9785a Mon Sep 17 00:00:00 2001 From: AloysZhang Date: Tue, 14 Feb 2023 16:15:30 +0800 Subject: [PATCH 016/174] [fix][broker] catch exception for brokerInterceptor (#19147) --- .../BrokerInterceptorWithClassLoader.java | 6 + .../broker/intercept/BrokerInterceptors.java | 6 + .../pulsar/broker/service/ServerCnx.java | 22 +++- .../ExceptionsBrokerInterceptor.java | 102 +++++++++++++++ .../ExceptionsBrokerInterceptorTest.java | 117 ++++++++++++++++++ .../apache/pulsar/client/impl/ClientCnx.java | 20 +++ 6 files changed, 267 insertions(+), 6 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/ExceptionsBrokerInterceptor.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/ExceptionsBrokerInterceptorTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptorWithClassLoader.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptorWithClassLoader.java index a74730d23e102..faee5799289d0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptorWithClassLoader.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptorWithClassLoader.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.intercept; +import com.google.common.annotations.VisibleForTesting; import io.netty.buffer.ByteBuf; import java.io.IOException; import java.util.Map; @@ -208,4 +209,9 @@ public void close() { log.warn("Failed to close the broker interceptor class loader", e); } } + + @VisibleForTesting + public BrokerInterceptor getInterceptor() { + return interceptor; + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java index e7f82742a97cc..cef3f0eb609a1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.intercept; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; import io.netty.buffer.ByteBuf; import java.io.IOException; @@ -277,4 +278,9 @@ public void close() { private boolean interceptorsEnabled() { return interceptors != null && !interceptors.isEmpty(); } + + @VisibleForTesting + public Map getInterceptors() { + return interceptors; + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 1351c6fe715a9..4c81b46601ea7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -1185,7 +1185,11 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { remoteAddress, topicName, subscriptionName); commandSender.sendSuccessResponse(requestId); if (brokerInterceptor != null) { - brokerInterceptor.consumerCreated(this, consumer, metadata); + try { + brokerInterceptor.consumerCreated(this, consumer, metadata); + } catch (Throwable t) { + log.error("Exception occur when intercept consumer created.", t); + } } } else { // The consumer future was completed before by a close command @@ -1223,8 +1227,7 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { } // If client timed out, the future would have been completed by subsequent close. - // Send error - // back to client, only if not completed already. + // Send error back to client, only if not completed already. if (consumerFuture.completeExceptionally(exception)) { commandSender.sendErrorResponse(requestId, BrokerServiceException.getClientErrorCode(exception), @@ -1521,8 +1524,11 @@ private void buildProducerAndAddTopic(Topic topic, long producerId, String produ producer.getLastSequenceId(), producer.getSchemaVersion(), newTopicEpoch, true /* producer is ready now */); if (brokerInterceptor != null) { - brokerInterceptor. - producerCreated(this, producer, metadata); + try { + brokerInterceptor.producerCreated(this, producer, metadata); + } catch (Throwable t) { + log.error("Exception occur when intercept producer created.", t); + } } return; } else { @@ -1689,7 +1695,11 @@ protected void handleAck(CommandAck ack) { requestId, null, null, consumerId)); } if (brokerInterceptor != null) { - brokerInterceptor.messageAcked(this, consumer, copyOfAckForInterceptor); + try { + brokerInterceptor.messageAcked(this, consumer, copyOfAckForInterceptor); + } catch (Throwable t) { + log.error("Exception occur when intercept message acked.", t); + } } }).exceptionally(e -> { if (hasRequestId) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/ExceptionsBrokerInterceptor.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/ExceptionsBrokerInterceptor.java new file mode 100644 index 0000000000000..f58d56c05a951 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/ExceptionsBrokerInterceptor.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.intercept; + +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.service.Consumer; +import org.apache.pulsar.broker.service.Producer; +import org.apache.pulsar.broker.service.ServerCnx; +import org.apache.pulsar.common.api.proto.BaseCommand; +import org.apache.pulsar.common.api.proto.CommandAck; +import org.apache.pulsar.common.intercept.InterceptException; + +public class ExceptionsBrokerInterceptor implements BrokerInterceptor { + + + private AtomicInteger producerCount = new AtomicInteger(); + private AtomicInteger consumerCount = new AtomicInteger(); + private AtomicInteger messageAckCount = new AtomicInteger(); + + public AtomicInteger getProducerCount() { + return producerCount; + } + + public AtomicInteger getConsumerCount() { + return consumerCount; + } + + public AtomicInteger getMessageAckCount() { + return messageAckCount; + } + + @Override + public void producerCreated(ServerCnx cnx, Producer producer, Map metadata) { + producerCount.incrementAndGet(); + throw new RuntimeException("exception when intercept producer created"); + } + + @Override + public void consumerCreated(ServerCnx cnx, Consumer consumer, Map metadata) { + consumerCount.incrementAndGet(); + throw new RuntimeException("exception when intercept consumer created"); + } + + @Override + public void messageAcked(ServerCnx cnx, Consumer consumer, CommandAck ackCmd) { + messageAckCount.incrementAndGet(); + throw new RuntimeException("exception when intercept consumer ack message"); + } + + @Override + public void onPulsarCommand(BaseCommand command, ServerCnx cnx) throws InterceptException { + + } + + @Override + public void onConnectionClosed(ServerCnx cnx) { + + } + + @Override + public void onWebserviceRequest(ServletRequest request) throws IOException, ServletException, InterceptException { + + } + + @Override + public void onWebserviceResponse(ServletRequest request, ServletResponse response) + throws IOException, ServletException { + + } + + @Override + public void initialize(PulsarService pulsarService) throws Exception { + + } + + @Override + public void close() { + + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/ExceptionsBrokerInterceptorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/ExceptionsBrokerInterceptorTest.java new file mode 100644 index 0000000000000..aa254a8ac168a --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/ExceptionsBrokerInterceptorTest.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.intercept; + +import static org.mockito.Mockito.mock; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; + +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.apache.pulsar.broker.testcontext.PulsarTestContext; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.client.impl.ClientCnx; +import org.apache.pulsar.client.impl.ConsumerImpl; +import org.apache.pulsar.common.nar.NarClassLoader; +import org.awaitility.Awaitility; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +public class ExceptionsBrokerInterceptorTest extends ProducerConsumerBase { + + private String interceptorName = "exception_interceptor"; + + @BeforeMethod + public void setup() throws Exception { + conf.setSystemTopicEnabled(false); + conf.setTopicLevelPoliciesEnabled(false); + this.conf.setDisableBrokerInterceptors(false); + + + this.enableBrokerInterceptor = true; + super.internalSetup(); + super.producerBaseSetup(); + } + + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Override + protected void customizeMainPulsarTestContextBuilder(PulsarTestContext.Builder pulsarTestContextBuilder) { + Map listenerMap = new HashMap<>(); + BrokerInterceptor interceptor = new ExceptionsBrokerInterceptor(); + NarClassLoader narClassLoader = mock(NarClassLoader.class); + listenerMap.put(interceptorName, new BrokerInterceptorWithClassLoader(interceptor, narClassLoader)); + pulsarTestContextBuilder.brokerInterceptor(new BrokerInterceptors(listenerMap)); + } + + @Test + public void testMessageAckedExceptions() throws Exception { + String topic = "persistent://public/default/test"; + String subName = "test-sub"; + int messageNumber = 10; + admin.topics().createNonPartitionedTopic(topic); + + BrokerInterceptors listener = (BrokerInterceptors) pulsar.getBrokerInterceptor(); + assertNotNull(listener); + BrokerInterceptorWithClassLoader brokerInterceptor = listener.getInterceptors().get(interceptorName); + assertNotNull(brokerInterceptor); + BrokerInterceptor interceptor = brokerInterceptor.getInterceptor(); + assertTrue(interceptor instanceof ExceptionsBrokerInterceptor); + + Producer producer = pulsarClient.newProducer().topic(topic).create(); + + ConsumerImpl consumer = (ConsumerImpl) pulsarClient + .newConsumer() + .topic(topic) + .subscriptionName(subName) + .acknowledgmentGroupTime(0, TimeUnit.MILLISECONDS) + .isAckReceiptEnabled(true) + .subscribe(); + + Awaitility.await().until(() -> ((ExceptionsBrokerInterceptor) interceptor).getProducerCount().get() == 1); + Awaitility.await().until(() -> ((ExceptionsBrokerInterceptor) interceptor).getConsumerCount().get() == 1); + + for (int i = 0; i < messageNumber; i ++) { + producer.send("test".getBytes(StandardCharsets.UTF_8)); + } + + int receiveCounter = 0; + Message message; + while((message = consumer.receive(3, TimeUnit.SECONDS)) != null) { + receiveCounter ++; + consumer.acknowledge(message); + } + assertEquals(receiveCounter, 10); + Awaitility.await().until(() + -> ((ExceptionsBrokerInterceptor) interceptor).getMessageAckCount().get() == messageNumber); + + ClientCnx clientCnx = consumer.getClientCnx(); + // no duplicated responses received from broker + assertEquals(clientCnx.getDuplicatedResponseCount(), 0); + } + +} diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java index f2ebb12f957ec..5074d0f55ef62 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java @@ -45,6 +45,7 @@ import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; +import java.util.concurrent.atomic.AtomicLong; import lombok.AccessLevel; import lombok.Getter; import org.apache.commons.lang3.exception.ExceptionUtils; @@ -115,6 +116,8 @@ public class ClientCnx extends PulsarHandler { protected final Authentication authentication; protected State state; + private AtomicLong duplicatedResponseCounter = new AtomicLong(0); + @Getter private final ConcurrentLongHashMap> pendingRequests = ConcurrentLongHashMap.>newBuilder() @@ -352,6 +355,11 @@ public static boolean isKnownException(Throwable t) { return t instanceof NativeIoException || t instanceof ClosedChannelException; } + @VisibleForTesting + public long getDuplicatedResponseCount() { + return duplicatedResponseCounter.get(); + } + @Override protected void handleConnected(CommandConnected connected) { checkArgument(state == State.SentConnectFrame || state == State.Connecting); @@ -475,6 +483,7 @@ protected void handleAckResponse(CommandAckResponse ackResponse) { buildError(ackResponse.getRequestId(), ackResponse.getMessage()))); } } else { + duplicatedResponseCounter.incrementAndGet(); log.warn("AckResponse has complete when receive response! requestId : {}, consumerId : {}", ackResponse.getRequestId(), ackResponse.hasConsumerId()); } @@ -519,6 +528,7 @@ protected void handleSuccess(CommandSuccess success) { if (requestFuture != null) { requestFuture.complete(null); } else { + duplicatedResponseCounter.incrementAndGet(); log.warn("{} Received unknown request id from server: {}", ctx.channel(), success.getRequestId()); } } @@ -537,6 +547,7 @@ protected void handleGetLastMessageIdSuccess(CommandGetLastMessageIdResponse suc if (requestFuture != null) { requestFuture.complete(new CommandGetLastMessageIdResponse().copyFrom(success)); } else { + duplicatedResponseCounter.incrementAndGet(); log.warn("{} Received unknown request id from server: {}", ctx.channel(), success.getRequestId()); } } @@ -572,6 +583,7 @@ protected void handleProducerSuccess(CommandProducerSuccess success) { success.hasTopicEpoch() ? Optional.of(success.getTopicEpoch()) : Optional.empty()); requestFuture.complete(pr); } else { + duplicatedResponseCounter.incrementAndGet(); log.warn("{} Received unknown request id from server: {}", ctx.channel(), success.getRequestId()); } } @@ -719,6 +731,8 @@ private CompletableFuture getAndRemovePendingLookupRequest(lon } else { pendingLookupRequestSemaphore.release(); } + } else { + duplicatedResponseCounter.incrementAndGet(); } return result; } @@ -775,6 +789,7 @@ protected void handleError(CommandError error) { getPulsarClientException(error.getError(), buildError(error.getRequestId(), error.getMessage()))); } else { + duplicatedResponseCounter.incrementAndGet(); log.warn("{} Received unknown request id from server: {}", ctx.channel(), error.getRequestId()); } } @@ -882,6 +897,7 @@ protected void handleGetTopicsOfNamespaceSuccess(CommandGetTopicsOfNamespaceResp success.isFiltered(), success.isChanged())); } else { + duplicatedResponseCounter.incrementAndGet(); log.warn("{} Received unknown request id from server: {}", ctx.channel(), success.getRequestId()); } } @@ -895,6 +911,7 @@ protected void handleGetSchemaResponse(CommandGetSchemaResponse commandGetSchema CompletableFuture future = (CompletableFuture) pendingRequests.remove(requestId); if (future == null) { + duplicatedResponseCounter.incrementAndGet(); log.warn("{} Received unknown request id from server: {}", ctx.channel(), requestId); return; } @@ -908,6 +925,7 @@ protected void handleGetOrCreateSchemaResponse(CommandGetOrCreateSchemaResponse CompletableFuture future = (CompletableFuture) pendingRequests.remove(requestId); if (future == null) { + duplicatedResponseCounter.incrementAndGet(); log.warn("{} Received unknown request id from server: {}", ctx.channel(), requestId); return; } @@ -1080,6 +1098,7 @@ protected void handleTcClientConnectResponse(CommandTcClientConnectResponse resp requestFuture.completeExceptionally(getExceptionByServerError(error, response.getMessage())); } } else { + duplicatedResponseCounter.incrementAndGet(); log.warn("Tc client connect command has been completed and get response for request: {}", response.getRequestId()); } @@ -1133,6 +1152,7 @@ protected void handleCommandWatchTopicListSuccess(CommandWatchTopicListSuccess c if (requestFuture != null) { requestFuture.complete(commandWatchTopicListSuccess); } else { + duplicatedResponseCounter.incrementAndGet(); log.warn("{} Received unknown request id from server: {}", ctx.channel(), commandWatchTopicListSuccess.getRequestId()); } From 153e4d4cc3b56aaee224b0a68e0186c08125c975 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Tue, 14 Feb 2023 03:09:55 -0600 Subject: [PATCH 017/174] [fix][broker] Make authentication refresh threadsafe (#19506) Co-authored-by: Lari Hotari --- .../service/PulsarChannelInitializer.java | 30 ----- .../pulsar/broker/service/ServerCnx.java | 108 +++++++++++------- .../service/PersistentTopicE2ETest.java | 2 - .../pulsar/broker/service/ServerCnxTest.java | 14 ++- 4 files changed, 75 insertions(+), 79 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarChannelInitializer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarChannelInitializer.java index a96625af4686c..5308b3c981eb4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarChannelInitializer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarChannelInitializer.java @@ -18,9 +18,6 @@ */ package org.apache.pulsar.broker.service; -import static org.apache.bookkeeper.util.SafeRunnable.safeRun; -import com.github.benmanes.caffeine.cache.Cache; -import com.github.benmanes.caffeine.cache.Caffeine; import com.google.common.annotations.VisibleForTesting; import io.netty.channel.ChannelInitializer; import io.netty.channel.socket.SocketChannel; @@ -30,8 +27,6 @@ import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslHandler; import io.netty.handler.ssl.SslProvider; -import java.net.SocketAddress; -import java.util.concurrent.TimeUnit; import lombok.Builder; import lombok.Data; import lombok.extern.slf4j.Slf4j; @@ -57,15 +52,6 @@ public class PulsarChannelInitializer extends ChannelInitializer private final ServiceConfiguration brokerConf; private NettySSLContextAutoRefreshBuilder nettySSLContextAutoRefreshBuilder; - // This cache is used to maintain a list of active connections to iterate over them - // We keep weak references to have the cache to be auto cleaned up when the connections - // objects are GCed. - @VisibleForTesting - protected final Cache connections = Caffeine.newBuilder() - .weakKeys() - .weakValues() - .build(); - /** * @param pulsar * An instance of {@link PulsarService} @@ -114,10 +100,6 @@ public PulsarChannelInitializer(PulsarService pulsar, PulsarChannelOptions opts) this.sslCtxRefresher = null; } this.brokerConf = pulsar.getConfiguration(); - - pulsar.getExecutor().scheduleAtFixedRate(safeRun(this::refreshAuthenticationCredentials), - pulsar.getConfig().getAuthenticationRefreshCheckSeconds(), - pulsar.getConfig().getAuthenticationRefreshCheckSeconds(), TimeUnit.SECONDS); } @Override @@ -148,18 +130,6 @@ protected void initChannel(SocketChannel ch) throws Exception { ch.pipeline().addLast("flowController", new FlowControlHandler()); ServerCnx cnx = newServerCnx(pulsar, listenerName); ch.pipeline().addLast("handler", cnx); - - connections.put(ch.remoteAddress(), cnx); - } - - private void refreshAuthenticationCredentials() { - connections.asMap().values().forEach(cnx -> { - try { - cnx.refreshAuthenticationCredentials(); - } catch (Throwable t) { - log.warn("[{}] Failed to refresh auth credentials", cnx.clientAddress()); - } - }); } @VisibleForTesting diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 4c81b46601ea7..71b31eeeeda2b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -39,6 +39,7 @@ import io.netty.handler.ssl.SslHandler; import io.netty.util.concurrent.FastThreadLocal; import io.netty.util.concurrent.Promise; +import io.netty.util.concurrent.ScheduledFuture; import io.prometheus.client.Gauge; import java.io.IOException; import java.net.InetSocketAddress; @@ -199,6 +200,7 @@ public class ServerCnx extends PulsarHandler implements TransportCnx { // Keep temporarily in order to verify after verifying proxy's authData private AuthData originalAuthDataCopy; private boolean pendingAuthChallengeResponse = false; + private ScheduledFuture authRefreshTask; // Max number of pending requests per connections. If multiple producers are sharing the same connection the flow // control done by a single producer might not be enough to prevent write spikes on the broker. @@ -332,6 +334,9 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception { } cnxsPerThread.get().remove(this); + if (authRefreshTask != null) { + authRefreshTask.cancel(false); + } // Connection is gone, close the producers immediately producers.forEach((__, producerFuture) -> { @@ -665,15 +670,18 @@ ByteBuf createConsumerStatsResponse(Consumer consumer, long requestId) { // complete the connect and sent newConnected command private void completeConnect(int clientProtoVersion, String clientVersion) { - if (service.isAuthenticationEnabled() && service.isAuthorizationEnabled()) { - if (!service.getAuthorizationService() + if (service.isAuthenticationEnabled()) { + if (service.isAuthorizationEnabled()) { + if (!service.getAuthorizationService() .isValidOriginalPrincipal(authRole, originalPrincipal, remoteAddress)) { - state = State.Failed; - service.getPulsarStats().recordConnectionCreateFail(); - final ByteBuf msg = Commands.newError(-1, ServerError.AuthorizationError, "Invalid roles."); - NettyChannelUtil.writeAndFlushWithClosePromise(ctx, msg); - return; + state = State.Failed; + service.getPulsarStats().recordConnectionCreateFail(); + final ByteBuf msg = Commands.newError(-1, ServerError.AuthorizationError, "Invalid roles."); + NettyChannelUtil.writeAndFlushWithClosePromise(ctx, msg); + return; + } } + maybeScheduleAuthenticationCredentialsRefresh(); } writeAndFlush(Commands.newConnected(clientProtoVersion, maxMessageSize, enableSubscriptionPatternEvaluation)); state = State.Connected; @@ -772,7 +780,7 @@ public void authChallengeSuccessCallback(AuthData authChallenge, log.debug("[{}] Authentication in progress client by method {}.", remoteAddress, authMethod); } } - } catch (Exception e) { + } catch (Exception | AssertionError e) { authenticationFailed(e); } } @@ -799,7 +807,7 @@ private void authenticateOriginalData(int clientProtoVersion, String clientVersi remoteAddress, originalPrincipal); } completeConnect(clientProtoVersion, clientVersion); - } catch (Exception e) { + } catch (Exception | AssertionError e) { authenticationFailed(e); } } @@ -821,61 +829,75 @@ private void authenticationFailed(Throwable t) { NettyChannelUtil.writeAndFlushWithClosePromise(ctx, msg); } - public void refreshAuthenticationCredentials() { - AuthenticationState authState = this.originalAuthState != null ? originalAuthState : this.authState; - + /** + * Method to initialize the {@link #authRefreshTask} task. + */ + private void maybeScheduleAuthenticationCredentialsRefresh() { + assert ctx.executor().inEventLoop(); + assert authRefreshTask == null; if (authState == null) { // Authentication is disabled or there's no local state to refresh return; - } else if (getState() != State.Connected || !isActive) { - // Connection is either still being established or already closed. + } + authRefreshTask = ctx.executor().scheduleAtFixedRate(this::refreshAuthenticationCredentials, + service.getPulsar().getConfig().getAuthenticationRefreshCheckSeconds(), + service.getPulsar().getConfig().getAuthenticationRefreshCheckSeconds(), + TimeUnit.SECONDS); + } + + private void refreshAuthenticationCredentials() { + assert ctx.executor().inEventLoop(); + AuthenticationState authState = this.originalAuthState != null ? originalAuthState : this.authState; + if (getState() == State.Failed) { + // Happens when an exception is thrown that causes this connection to close. return; } else if (!authState.isExpired()) { // Credentials are still valid. Nothing to do at this point return; } else if (originalPrincipal != null && originalAuthState == null) { + // This case is only checked when the authState is expired because we've reached a point where + // authentication needs to be refreshed, but the protocol does not support it unless the proxy forwards + // the originalAuthData. log.info( "[{}] Cannot revalidate user credential when using proxy and" + " not forwarding the credentials. Closing connection", remoteAddress); + ctx.close(); return; } - ctx.executor().execute(SafeRun.safeRun(() -> { - log.info("[{}] Refreshing authentication credentials for originalPrincipal {} and authRole {}", - remoteAddress, originalPrincipal, this.authRole); - - if (!supportsAuthenticationRefresh()) { - log.warn("[{}] Closing connection because client doesn't support auth credentials refresh", - remoteAddress); - ctx.close(); - return; - } + if (!supportsAuthenticationRefresh()) { + log.warn("[{}] Closing connection because client doesn't support auth credentials refresh", + remoteAddress); + ctx.close(); + return; + } - if (pendingAuthChallengeResponse) { - log.warn("[{}] Closing connection after timeout on refreshing auth credentials", - remoteAddress); - ctx.close(); - return; - } + if (pendingAuthChallengeResponse) { + log.warn("[{}] Closing connection after timeout on refreshing auth credentials", + remoteAddress); + ctx.close(); + return; + } - try { - AuthData brokerData = authState.refreshAuthentication(); + log.info("[{}] Refreshing authentication credentials for originalPrincipal {} and authRole {}", + remoteAddress, originalPrincipal, this.authRole); + try { + AuthData brokerData = authState.refreshAuthentication(); - writeAndFlush(Commands.newAuthChallenge(authMethod, brokerData, - getRemoteEndpointProtocolVersion())); - if (log.isDebugEnabled()) { - log.debug("[{}] Sent auth challenge to client to refresh credentials with method: {}.", + writeAndFlush(Commands.newAuthChallenge(authMethod, brokerData, + getRemoteEndpointProtocolVersion())); + if (log.isDebugEnabled()) { + log.debug("[{}] Sent auth challenge to client to refresh credentials with method: {}.", remoteAddress, authMethod); - } + } - pendingAuthChallengeResponse = true; + pendingAuthChallengeResponse = true; - } catch (AuthenticationException e) { - log.warn("[{}] Failed to refresh authentication: {}", remoteAddress, e); - ctx.close(); - } - })); + } catch (AuthenticationException e) { + log.warn("[{}] Failed to refresh authentication: {}", remoteAddress, e); + ctx.close(); + } } private static final byte[] emptyArray = new byte[0]; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicE2ETest.java index 7506053b28d15..63f80911ae62b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicE2ETest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicE2ETest.java @@ -1948,8 +1948,6 @@ protected void initChannel(SocketChannel ch) throws Exception { ch.pipeline().remove("handler"); PersistentTopicE2ETest.ServerCnxForTest serverCnxForTest = new PersistentTopicE2ETest.ServerCnxForTest(this.pulsar, this.opts.getListenerName()); ch.pipeline().addAfter("flowController", "testHandler", serverCnxForTest); - //override parent - connections.put(ch.remoteAddress(), serverCnxForTest); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java index afa1cd4e2528c..7e7c2110533ae 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java @@ -487,10 +487,13 @@ public void testAuthChallengePrincipalChangeFails() throws Exception { when(brokerService.getAuthenticationService()).thenReturn(authenticationService); when(authenticationService.getAuthenticationProvider(authMethodName)).thenReturn(authenticationProvider); svcConfig.setAuthenticationEnabled(true); + svcConfig.setAuthenticationRefreshCheckSeconds(30); resetChannel(); assertTrue(channel.isActive()); assertEquals(serverCnx.getState(), State.Start); + // Don't want the keep alive task affecting which messages are handled + serverCnx.cancelKeepAliveTask(); ByteBuf clientCommand = Commands.newConnect(authMethodName, "pass.client", ""); channel.writeInbound(clientCommand); @@ -503,7 +506,7 @@ public void testAuthChallengePrincipalChangeFails() throws Exception { // Trigger the ServerCnx to check if authentication is expired (it is because of our special implementation) // and then force channel to run the task - serverCnx.refreshAuthenticationCredentials(); + channel.advanceTimeBy(30, TimeUnit.SECONDS); channel.runPendingTasks(); Object responseAuthChallenge1 = getResponse(); assertTrue(responseAuthChallenge1 instanceof CommandAuthChallenge); @@ -513,7 +516,7 @@ public void testAuthChallengePrincipalChangeFails() throws Exception { channel.writeInbound(authResponse1); // Trigger the ServerCnx to check if authentication is expired again - serverCnx.refreshAuthenticationCredentials(); + channel.advanceTimeBy(30, TimeUnit.SECONDS); assertTrue(channel.hasPendingTasks(), "This test assumes there are pending tasks to run."); channel.runPendingTasks(); Object responseAuthChallenge2 = getResponse(); @@ -539,10 +542,13 @@ public void testAuthChallengeOriginalPrincipalChangeFails() throws Exception { svcConfig.setAuthenticationEnabled(true); svcConfig.setAuthenticateOriginalAuthData(true); svcConfig.setProxyRoles(Collections.singleton("pass.proxy")); + svcConfig.setAuthenticationRefreshCheckSeconds(30); resetChannel(); assertTrue(channel.isActive()); assertEquals(serverCnx.getState(), State.Start); + // Don't want the keep alive task affecting which messages are handled + serverCnx.cancelKeepAliveTask(); ByteBuf clientCommand = Commands.newConnect(authMethodName, "pass.proxy", 1, null, null, "pass.client", "pass.client", authMethodName); @@ -559,7 +565,7 @@ public void testAuthChallengeOriginalPrincipalChangeFails() throws Exception { // Trigger the ServerCnx to check if authentication is expired (it is because of our special implementation) // and then force channel to run the task - serverCnx.refreshAuthenticationCredentials(); + channel.advanceTimeBy(30, TimeUnit.SECONDS); assertTrue(channel.hasPendingTasks(), "This test assumes there are pending tasks to run."); channel.runPendingTasks(); Object responseAuthChallenge1 = getResponse(); @@ -570,7 +576,7 @@ public void testAuthChallengeOriginalPrincipalChangeFails() throws Exception { channel.writeInbound(authResponse1); // Trigger the ServerCnx to check if authentication is expired again - serverCnx.refreshAuthenticationCredentials(); + channel.advanceTimeBy(30, TimeUnit.SECONDS); channel.runPendingTasks(); Object responseAuthChallenge2 = getResponse(); assertTrue(responseAuthChallenge2 instanceof CommandAuthChallenge); From 8b7e4ce6f1d22d20e840a8de123f810b07ca2df8 Mon Sep 17 00:00:00 2001 From: wenbingshen Date: Tue, 14 Feb 2023 21:55:18 +0800 Subject: [PATCH 018/174] [fix][io] Config autoCommitEnabled when it disabled (#19499) --- .../java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java index c01158da50a7f..565c36047474b 100644 --- a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java +++ b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java @@ -117,6 +117,8 @@ public void open(Map config, SourceContext sourceContext) throws } props.put(ConsumerConfig.GROUP_ID_CONFIG, kafkaSourceConfig.getGroupId()); props.put(ConsumerConfig.FETCH_MIN_BYTES_CONFIG, String.valueOf(kafkaSourceConfig.getFetchMinBytes())); + props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, + String.valueOf(kafkaSourceConfig.isAutoCommitEnabled())); props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, String.valueOf(kafkaSourceConfig.getAutoCommitIntervalMs())); props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, String.valueOf(kafkaSourceConfig.getSessionTimeoutMs())); From 8639585bfe50720f6791fa512704100ff0541d48 Mon Sep 17 00:00:00 2001 From: houxiaoyu Date: Tue, 14 Feb 2023 22:04:23 +0800 Subject: [PATCH 019/174] [cleanup][broker] Cleanup finalPosition null-check in asyncFindPosition (#19497) --- .../apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index f6c9fb3bfd806..d541f60a0d3cc 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -1779,7 +1779,7 @@ public void closeComplete(int rc, LedgerHandle lh, Object o) { @Override public CompletableFuture asyncFindPosition(Predicate predicate) { - CompletableFuture future = new CompletableFuture(); + CompletableFuture future = new CompletableFuture<>(); Long firstLedgerId = ledgers.firstKey(); final PositionImpl startPosition = firstLedgerId == null ? null : new PositionImpl(firstLedgerId, 0); if (startPosition == null) { @@ -1792,11 +1792,6 @@ public void findEntryComplete(Position position, Object ctx) { final Position finalPosition; if (position == null) { finalPosition = startPosition; - if (finalPosition == null) { - log.warn("[{}] Unable to find position for predicate {}.", name, predicate); - future.complete(null); - return; - } log.info("[{}] Unable to find position for predicate {}. Use the first position {} instead.", name, predicate, startPosition); } else { From 6bafe8d21bf2b51c32892e12f74719e2b28e2cdd Mon Sep 17 00:00:00 2001 From: Kai Date: Tue, 14 Feb 2023 06:08:05 -0800 Subject: [PATCH 020/174] [improve][client] Remove default 30s ackTimeout when setting DLQ policy on java consumer (#19486) --- .../java/org/apache/pulsar/client/api/ConsumerBuilder.java | 2 -- .../org/apache/pulsar/client/impl/ConsumerBuilderImpl.java | 4 ---- .../apache/pulsar/client/impl/ConsumerBuilderImplTest.java | 2 +- 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerBuilder.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerBuilder.java index 8978584b2b47a..14a94cb8286dc 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerBuilder.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerBuilder.java @@ -602,8 +602,6 @@ public interface ConsumerBuilder extends Cloneable { * .build()) * .subscribe(); * - * When a dead letter policy is specified, and no ackTimeoutMillis is specified, - * then the acknowledgment timeout is set to 30000 milliseconds. */ ConsumerBuilder deadLetterPolicy(DeadLetterPolicy deadLetterPolicy); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBuilderImpl.java index 3eb1ea4c87457..f644c6a18398f 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBuilderImpl.java @@ -71,7 +71,6 @@ public class ConsumerBuilderImpl implements ConsumerBuilder { private static final long MIN_ACK_TIMEOUT_MILLIS = 1000; private static final long MIN_TICK_TIME_MILLIS = 100; - private static final long DEFAULT_ACK_TIMEOUT_MILLIS_FOR_DEAD_LETTER = 30000L; public ConsumerBuilderImpl(PulsarClientImpl client, Schema schema) { @@ -440,9 +439,6 @@ public ConsumerBuilder intercept(ConsumerInterceptor... interceptors) { @Override public ConsumerBuilder deadLetterPolicy(DeadLetterPolicy deadLetterPolicy) { if (deadLetterPolicy != null) { - if (conf.getAckTimeoutMillis() == 0) { - conf.setAckTimeoutMillis(DEFAULT_ACK_TIMEOUT_MILLIS_FOR_DEAD_LETTER); - } checkArgument(deadLetterPolicy.getMaxRedeliverCount() > 0, "MaxRedeliverCount must be > 0."); } conf.setDeadLetterPolicy(deadLetterPolicy); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerBuilderImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerBuilderImplTest.java index daa3fbf8eba2d..8dbd23f9c29c9 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerBuilderImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerBuilderImplTest.java @@ -537,7 +537,7 @@ public void testLoadConfNotModified() { assertEquals(configurationData.getNegativeAckRedeliveryDelayMicros(), TimeUnit.MINUTES.toMicros(1)); assertEquals(configurationData.getMaxTotalReceiverQueueSizeAcrossPartitions(), 50000); assertEquals(configurationData.getConsumerName(), "consumer"); - assertEquals(configurationData.getAckTimeoutMillis(), 30000); + assertEquals(configurationData.getAckTimeoutMillis(), 0); assertEquals(configurationData.getTickDurationMillis(), 1000); assertEquals(configurationData.getPriorityLevel(), 0); assertEquals(configurationData.getMaxPendingChunkedMessage(), 10); From 456d1122525bb5a0b794a6f6f15313e40b886047 Mon Sep 17 00:00:00 2001 From: gaozhangmin Date: Wed, 15 Feb 2023 17:53:37 +0800 Subject: [PATCH 021/174] [fix][broker] Fix loadbalance score caculation problem (#19420) --- .../impl/ModularLoadManagerImpl.java | 29 +++++++++---------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java index e25ec981e7705..7a933908962ec 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java @@ -40,6 +40,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.SystemUtils; @@ -60,6 +61,7 @@ import org.apache.pulsar.broker.stats.prometheus.metrics.Summary; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.util.ExecutorProvider; +import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.NamespaceBundleFactory; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.ServiceUnitId; @@ -568,24 +570,19 @@ private void updateBundleData() { // Remove all loaded bundles from the preallocated maps. final Map preallocatedBundleData = brokerData.getPreallocatedBundleData(); + Set ownedNsBundles = pulsar.getNamespaceService().getOwnedServiceUnits() + .stream().map(NamespaceBundle::toString).collect(Collectors.toSet()); synchronized (preallocatedBundleData) { - for (String preallocatedBundleName : brokerData.getPreallocatedBundleData().keySet()) { - if (brokerData.getLocalData().getBundles().contains(preallocatedBundleName)) { - final Iterator> preallocatedIterator = - preallocatedBundleData.entrySet() - .iterator(); - while (preallocatedIterator.hasNext()) { - final String bundle = preallocatedIterator.next().getKey(); - - if (bundleData.containsKey(bundle)) { - preallocatedIterator.remove(); - preallocatedBundleToBroker.remove(bundle); - } - } + preallocatedBundleToBroker.keySet().removeAll(preallocatedBundleData.keySet()); + final Iterator> preallocatedIterator = + preallocatedBundleData.entrySet().iterator(); + while (preallocatedIterator.hasNext()) { + final String bundle = preallocatedIterator.next().getKey(); + if (!ownedNsBundles.contains(bundle) + || (brokerData.getLocalData().getBundles().contains(bundle) + && bundleData.containsKey(bundle))) { + preallocatedIterator.remove(); } - - // This is needed too in case a broker which was assigned a bundle dies and comes back up. - preallocatedBundleToBroker.remove(preallocatedBundleName); } } From ca0b25ecba50dd86de28ad221d1c29ec6419a973 Mon Sep 17 00:00:00 2001 From: Masahiro Sakamoto Date: Wed, 15 Feb 2023 22:40:19 +0900 Subject: [PATCH 022/174] [fix][sec] Upgrade kafka client to 3.4.0 to fix CVE-2023-25194 (#19527) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index bf1ad305e3b20..07e7ad634dfb8 100644 --- a/pom.xml +++ b/pom.xml @@ -167,7 +167,7 @@ flexible messaging model and an intuitive client API. 2.2.0 3.11.2 4.4.20 - 2.8.2 + 3.4.0 5.5.3 1.12.262 1.10.2 From f9af4245e0b05c382656fc674fdaeda26487258c Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Wed, 15 Feb 2023 23:02:30 +0800 Subject: [PATCH 023/174] [fix] [broker] Incorrect service name selection logic (#19505) When calling the method `PulsarWebResource.getRedirectionUrl`, reuse the same `PulsarServiceNameResolver` instance. --- .../pulsar/broker/web/PulsarWebResource.java | 31 ++++++++++++++++--- .../pulsar/broker/service/ReplicatorTest.java | 3 +- .../impl/PulsarServiceNameResolver.java | 5 ++- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java index fb80a3e79834f..82246ad649441 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java @@ -21,12 +21,16 @@ import static com.google.common.base.Preconditions.checkArgument; import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.commons.lang3.StringUtils.isBlank; +import com.github.benmanes.caffeine.cache.CacheLoader; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.LoadingCache; import com.google.common.collect.BoundType; import com.google.common.collect.Range; import com.google.common.collect.Sets; import java.net.MalformedURLException; import java.net.URI; import java.net.URL; +import java.time.Duration; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -86,6 +90,8 @@ import org.apache.pulsar.common.policies.path.PolicyPath; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.api.MetadataStoreException; +import org.checkerframework.checker.nullness.qual.NonNull; +import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -96,6 +102,17 @@ public abstract class PulsarWebResource { private static final Logger log = LoggerFactory.getLogger(PulsarWebResource.class); + private static final LoadingCache SERVICE_NAME_RESOLVER_CACHE = + Caffeine.newBuilder().expireAfterWrite(Duration.ofMinutes(5)).build( + new CacheLoader<>() { + @Override + public @Nullable PulsarServiceNameResolver load(@NonNull String serviceUrl) throws Exception { + PulsarServiceNameResolver serviceNameResolver = new PulsarServiceNameResolver(); + serviceNameResolver.updateServiceUrl(serviceUrl); + return serviceNameResolver; + } + }); + static final String ORIGINAL_PRINCIPAL_HEADER = "X-Original-Principal"; @Context @@ -476,17 +493,21 @@ protected void validateClusterOwnership(String cluster) throws WebApplicationExc private URI getRedirectionUrl(ClusterData differentClusterData) throws MalformedURLException { try { - PulsarServiceNameResolver serviceNameResolver = new PulsarServiceNameResolver(); + PulsarServiceNameResolver serviceNameResolver; if (isRequestHttps() && pulsar.getConfiguration().getWebServicePortTls().isPresent() && StringUtils.isNotBlank(differentClusterData.getServiceUrlTls())) { - serviceNameResolver.updateServiceUrl(differentClusterData.getServiceUrlTls()); + serviceNameResolver = SERVICE_NAME_RESOLVER_CACHE.get(differentClusterData.getServiceUrlTls()); } else { - serviceNameResolver.updateServiceUrl(differentClusterData.getServiceUrl()); + serviceNameResolver = SERVICE_NAME_RESOLVER_CACHE.get(differentClusterData.getServiceUrl()); } URL webUrl = new URL(serviceNameResolver.resolveHostUri().toString()); return UriBuilder.fromUri(uri.getRequestUri()).host(webUrl.getHost()).port(webUrl.getPort()).build(); - } catch (PulsarClientException.InvalidServiceURL exception) { - throw new MalformedURLException(exception.getMessage()); + } catch (Exception exception) { + if (exception.getCause() != null + && exception.getCause() instanceof PulsarClientException.InvalidServiceURL) { + throw new MalformedURLException(exception.getMessage()); + } + throw exception; } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java index 1c8c86c9434a3..ab4f6a5c7f8ba 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java @@ -233,7 +233,8 @@ public void activeBrokerParse() throws Exception { pulsar1.getConfiguration().setAuthorizationEnabled(true); //init clusterData - String cluster2ServiceUrls = String.format("%s,localhost:1234,localhost:5678", pulsar2.getWebServiceAddress()); + String cluster2ServiceUrls = String.format("%s,localhost:1234,localhost:5678,localhost:5677,localhost:5676", + pulsar2.getWebServiceAddress()); ClusterData cluster2Data = ClusterData.builder().serviceUrl(cluster2ServiceUrls).build(); String cluster2 = "activeCLuster2"; admin2.clusters().createCluster(cluster2, cluster2Data); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarServiceNameResolver.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarServiceNameResolver.java index 32f5aa4975c6c..e47750be46219 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarServiceNameResolver.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarServiceNameResolver.java @@ -52,9 +52,8 @@ public InetSocketAddress resolveHost() { if (list.size() == 1) { return list.get(0); } else { - CURRENT_INDEX_UPDATER.getAndUpdate(this, last -> (last + 1) % list.size()); - return list.get(currentIndex); - + int originalIndex = CURRENT_INDEX_UPDATER.getAndUpdate(this, last -> (last + 1) % list.size()); + return list.get((originalIndex + 1) % list.size()); } } From 89c1de36933c4dfa5bccc82d02a43f563f037d82 Mon Sep 17 00:00:00 2001 From: StevenLuMT Date: Thu, 16 Feb 2023 10:30:29 +0800 Subject: [PATCH 024/174] [cleanup][broker] Cleanup ManagedLedgerImpl's nouse method: isLedgersReadonly (#19513) Co-authored-by: lushiji --- .../apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java | 4 ---- .../bookkeeper/mledger/impl/ShadowManagedLedgerImpl.java | 5 ----- 2 files changed, 9 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index d541f60a0d3cc..e334adf078a9e 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -465,10 +465,6 @@ public void operationFailed(MetaStoreException e) { scheduleTimeoutTask(); } - protected boolean isLedgersReadonly() { - return false; - } - protected synchronized void initializeBookKeeper(final ManagedLedgerInitializeLedgerCallback callback) { if (log.isDebugEnabled()) { log.debug("[{}] initializing bookkeeper; ledgers {}", name, ledgers); diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImpl.java index 39e7b6b42ec0b..9a029778fe01c 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImpl.java @@ -161,11 +161,6 @@ public void operationFailed(ManagedLedgerException.MetaStoreException e) { }); } - @Override - protected boolean isLedgersReadonly() { - return true; - } - @Override protected synchronized void initializeBookKeeper(ManagedLedgerInitializeLedgerCallback callback) { if (log.isDebugEnabled()) { From b7b2053b4e4342a16d5ac9ee5ca02b4ff8571564 Mon Sep 17 00:00:00 2001 From: HuangWei Date: Thu, 16 Feb 2023 14:07:50 +0800 Subject: [PATCH 025/174] [fix][fn] log4j root level (#19500) --- conf/functions_log4j2.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/functions_log4j2.xml b/conf/functions_log4j2.xml index 190d9be92940b..6902a3acd8736 100644 --- a/conf/functions_log4j2.xml +++ b/conf/functions_log4j2.xml @@ -120,11 +120,11 @@ - info + ${sys:pulsar.log.level} ${sys:pulsar.log.appender} ${sys:pulsar.log.level} - \ No newline at end of file + From cb306c8e6c5da3a05ea312a5e5746eb178d9ab10 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Thu, 16 Feb 2023 17:07:32 +0800 Subject: [PATCH 026/174] [fix] [broker] Make the service name resolver cache of PulsarWebResource expire after access (#19532) --- .../java/org/apache/pulsar/broker/web/PulsarWebResource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java index 82246ad649441..5484a70e1aad0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java @@ -103,7 +103,7 @@ public abstract class PulsarWebResource { private static final Logger log = LoggerFactory.getLogger(PulsarWebResource.class); private static final LoadingCache SERVICE_NAME_RESOLVER_CACHE = - Caffeine.newBuilder().expireAfterWrite(Duration.ofMinutes(5)).build( + Caffeine.newBuilder().expireAfterAccess(Duration.ofMinutes(5)).build( new CacheLoader<>() { @Override public @Nullable PulsarServiceNameResolver load(@NonNull String serviceUrl) throws Exception { From fe547c7b7e1d4cb344419cbe37dc2ae771ca630f Mon Sep 17 00:00:00 2001 From: Masahiro Sakamoto Date: Thu, 16 Feb 2023 20:02:44 +0900 Subject: [PATCH 027/174] [fix][client] Shade com.fasterxml.jackson.datatype.* to prevent ClassNotFoundException (#19458) --- pulsar-client-admin-shaded/pom.xml | 9 +++++++-- pulsar-client-all/pom.xml | 12 +++++++----- pulsar-client-shaded/pom.xml | 11 +++++++---- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/pulsar-client-admin-shaded/pom.xml b/pulsar-client-admin-shaded/pom.xml index c11aaa5b57d34..4fca4b1bb25ad 100644 --- a/pulsar-client-admin-shaded/pom.xml +++ b/pulsar-client-admin-shaded/pom.xml @@ -123,7 +123,7 @@ com.google.protobuf:protobuf-java com.google.guava:guava com.google.code.gson:gson - com.fasterxml.jackson.core + com.fasterxml.jackson.*:* io.netty:* io.netty.incubator:* org.apache.pulsar:pulsar-common @@ -133,7 +133,6 @@ javax.ws.rs:* jakarta.annotation:* org.glassfish.hk2*:* - com.fasterxml.jackson.*:* io.grpc:* io.perfmark:* com.yahoo.datasketches:* @@ -150,6 +149,9 @@ org.apache.pulsar:pulsar-client-messagecrypto-bc + + com.fasterxml.jackson.core:jackson-annotations + @@ -192,6 +194,9 @@ com.fasterxml.jackson org.apache.pulsar.shade.com.fasterxml.jackson + + com.fasterxml.jackson.annotation.* + io.netty diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml index 9a31313d9c82b..62f98b8316eec 100644 --- a/pulsar-client-all/pom.xml +++ b/pulsar-client-all/pom.xml @@ -151,10 +151,7 @@ com.google.errorprone:* com.google.j2objc:* com.google.code.gson:gson - com.fasterxml.jackson.core - com.fasterxml.jackson.module - com.fasterxml.jackson.core:jackson-core - com.fasterxml.jackson.dataformat + com.fasterxml.jackson.*:* io.netty:netty io.netty:netty-all io.netty:netty-tcnative-boringssl-static @@ -171,7 +168,6 @@ javax.ws.rs:* jakarta.annotation:* org.glassfish.hk2*:* - com.fasterxml.jackson.*:* io.grpc:* io.perfmark:* com.yahoo.datasketches:* @@ -194,6 +190,9 @@ org.apache.pulsar:pulsar-client-messagecrypto-bc + + com.fasterxml.jackson.core:jackson-annotations + @@ -230,6 +229,9 @@ com.fasterxml.jackson org.apache.pulsar.shade.com.fasterxml.jackson + + com.fasterxml.jackson.annotation.* + io.netty diff --git a/pulsar-client-shaded/pom.xml b/pulsar-client-shaded/pom.xml index fc87b98371bb4..ce5e6acf152d8 100644 --- a/pulsar-client-shaded/pom.xml +++ b/pulsar-client-shaded/pom.xml @@ -145,10 +145,7 @@ com.google.errorprone:* com.google.j2objc:* com.google.code.gson:gson - com.fasterxml.jackson.core - com.fasterxml.jackson.module - com.fasterxml.jackson.core:jackson-core - com.fasterxml.jackson.dataformat + com.fasterxml.jackson.*:* io.netty:* io.netty.incubator:* io.perfmark:* @@ -171,6 +168,9 @@ org.apache.pulsar:pulsar-client-messagecrypto-bc + + com.fasterxml.jackson.core:jackson-annotations + @@ -207,6 +207,9 @@ com.fasterxml.jackson org.apache.pulsar.shade.com.fasterxml.jackson + + com.fasterxml.jackson.annotation.* + io.netty From c0f89dc981fbe36be834303f13bd09f3b62b3d67 Mon Sep 17 00:00:00 2001 From: xiaolong ran Date: Thu, 16 Feb 2023 23:18:11 +0800 Subject: [PATCH 028/174] [improve][broker] Use shrink map for trackerCache (#19534) Signed-off-by: xiaolongran --- .../broker/service/InMemoryRedeliveryTracker.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/InMemoryRedeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/InMemoryRedeliveryTracker.java index df999ab139d8e..8c992d2f7a90b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/InMemoryRedeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/InMemoryRedeliveryTracker.java @@ -21,12 +21,16 @@ import java.util.List; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.bookkeeper.util.collections.ConcurrentLongLongPairHashMap; -import org.apache.bookkeeper.util.collections.ConcurrentLongLongPairHashMap.LongPair; +import org.apache.pulsar.common.util.collections.ConcurrentLongLongPairHashMap; +import org.apache.pulsar.common.util.collections.ConcurrentLongLongPairHashMap.LongPair; public class InMemoryRedeliveryTracker implements RedeliveryTracker { - private ConcurrentLongLongPairHashMap trackerCache = new ConcurrentLongLongPairHashMap(256, 1); + private ConcurrentLongLongPairHashMap trackerCache = ConcurrentLongLongPairHashMap.newBuilder() + .concurrencyLevel(1) + .expectedItems(256) + .autoShrink(true) + .build(); @Override public int incrementAndGetRedeliveryCount(Position position) { From 2d90089dfa2a6af99cfe257bd1f2b3d24166303a Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Fri, 17 Feb 2023 00:30:27 +0800 Subject: [PATCH 029/174] [fix][authentication] Store the original authentication data (#19519) Signed-off-by: Zixuan Liu --- .../pulsar/broker/service/ServerCnx.java | 31 +++---- .../MockAlwaysExpiredAuthenticationState.java | 58 +------------ .../MockMutableAuthenticationProvider.java | 32 ++++++++ .../auth/MockMutableAuthenticationState.java | 81 +++++++++++++++++++ .../pulsar/broker/service/ServerCnxTest.java | 49 +++++++++++ 5 files changed, 182 insertions(+), 69 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockMutableAuthenticationProvider.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockMutableAuthenticationState.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 71b31eeeeda2b..d6a6dda402eca 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -737,31 +737,32 @@ public void authChallengeSuccessCallback(AuthData authChallenge, // 2. an authentication refresh, in which case we need to refresh authenticationData AuthenticationState authState = useOriginalAuthState ? originalAuthState : this.authState; String newAuthRole = authState.getAuthRole(); - - // Refresh the auth data. - this.authenticationData = authState.getAuthDataSource(); - if (log.isDebugEnabled()) { - log.debug("[{}] Auth data refreshed for role={}", remoteAddress, this.authRole); - } - - if (!useOriginalAuthState) { - this.authRole = newAuthRole; - } - - if (log.isDebugEnabled()) { - log.debug("[{}] Client successfully authenticated with {} role {} and originalPrincipal {}", - remoteAddress, authMethod, this.authRole, originalPrincipal); - } + AuthenticationDataSource newAuthDataSource = authState.getAuthDataSource(); if (state != State.Connected) { + // Set the auth data and auth role + if (!useOriginalAuthState) { + this.authRole = newAuthRole; + this.authenticationData = newAuthDataSource; + } // First time authentication is done if (originalAuthState != null) { // We only set originalAuthState when we are going to use it. authenticateOriginalData(clientProtocolVersion, clientVersion); } else { completeConnect(clientProtocolVersion, clientVersion); + if (log.isDebugEnabled()) { + log.debug("[{}] Client successfully authenticated with {} role {} and originalPrincipal {}", + remoteAddress, authMethod, this.authRole, originalPrincipal); + } } } else { + // Refresh the auth data + if (!useOriginalAuthState) { + this.authenticationData = newAuthDataSource; + } else { + this.originalAuthData = newAuthDataSource; + } // If the connection was already ready, it means we're doing a refresh if (!StringUtils.isEmpty(authRole)) { if (!authRole.equals(newAuthRole)) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAlwaysExpiredAuthenticationState.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAlwaysExpiredAuthenticationState.java index 95751ed0b85b3..7828eed553205 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAlwaysExpiredAuthenticationState.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAlwaysExpiredAuthenticationState.java @@ -18,65 +18,15 @@ */ package org.apache.pulsar.broker.auth; -import org.apache.pulsar.broker.authentication.AuthenticationDataCommand; -import org.apache.pulsar.broker.authentication.AuthenticationDataSource; -import org.apache.pulsar.broker.authentication.AuthenticationState; -import org.apache.pulsar.common.api.AuthData; - -import javax.naming.AuthenticationException; -import java.util.concurrent.CompletableFuture; - -import static java.nio.charset.StandardCharsets.UTF_8; +import org.apache.pulsar.broker.authentication.AuthenticationProvider; /** * Class to use when verifying the behavior around expired authentication data because it will always return * true when isExpired is called. */ -public class MockAlwaysExpiredAuthenticationState implements AuthenticationState { - final MockAlwaysExpiredAuthenticationProvider provider; - AuthenticationDataSource authenticationDataSource; - volatile String authRole; - - MockAlwaysExpiredAuthenticationState(MockAlwaysExpiredAuthenticationProvider provider) { - this.provider = provider; - } - - - @Override - public String getAuthRole() throws AuthenticationException { - if (authRole == null) { - throw new AuthenticationException("Must authenticate first."); - } - return authRole; - } - - @Override - public AuthData authenticate(AuthData authData) throws AuthenticationException { - return null; - } - - /** - * This authentication is always single stage, so it returns immediately - */ - @Override - public CompletableFuture authenticateAsync(AuthData authData) { - authenticationDataSource = new AuthenticationDataCommand(new String(authData.getBytes(), UTF_8)); - return provider - .authenticateAsync(authenticationDataSource) - .thenApply(role -> { - authRole = role; - return null; - }); - } - - @Override - public AuthenticationDataSource getAuthDataSource() { - return authenticationDataSource; - } - - @Override - public boolean isComplete() { - return true; +public class MockAlwaysExpiredAuthenticationState extends MockMutableAuthenticationState { + MockAlwaysExpiredAuthenticationState(AuthenticationProvider provider) { + super(provider); } @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockMutableAuthenticationProvider.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockMutableAuthenticationProvider.java new file mode 100644 index 0000000000000..2390346c8ff66 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockMutableAuthenticationProvider.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.auth; + +import org.apache.pulsar.broker.authentication.AuthenticationState; +import org.apache.pulsar.common.api.AuthData; +import javax.net.ssl.SSLSession; +import java.net.SocketAddress; + +public class MockMutableAuthenticationProvider extends MockAuthenticationProvider { + public AuthenticationState newAuthState(AuthData authData, + SocketAddress remoteAddress, + SSLSession sslSession) { + return new MockMutableAuthenticationState(this); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockMutableAuthenticationState.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockMutableAuthenticationState.java new file mode 100644 index 0000000000000..4579deb7d05bd --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockMutableAuthenticationState.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.auth; + +import static java.nio.charset.StandardCharsets.UTF_8; +import javax.naming.AuthenticationException; +import java.util.concurrent.CompletableFuture; +import org.apache.pulsar.broker.authentication.AuthenticationDataCommand; +import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.authentication.AuthenticationProvider; +import org.apache.pulsar.broker.authentication.AuthenticationState; +import org.apache.pulsar.common.api.AuthData; + +// MockMutableAuthenticationState always update the authentication data source and auth role. +public class MockMutableAuthenticationState implements AuthenticationState { + final AuthenticationProvider provider; + AuthenticationDataSource authenticationDataSource; + volatile String authRole; + + MockMutableAuthenticationState(AuthenticationProvider provider) { + this.provider = provider; + } + + @Override + public String getAuthRole() throws AuthenticationException { + if (authRole == null) { + throw new AuthenticationException("Must authenticate first."); + } + return authRole; + } + + @Override + public AuthData authenticate(AuthData authData) throws AuthenticationException { + return null; + } + + /** + * This authentication is always single stage, so it returns immediately + */ + @Override + public CompletableFuture authenticateAsync(AuthData authData) { + authenticationDataSource = new AuthenticationDataCommand(new String(authData.getBytes(), UTF_8)); + return provider + .authenticateAsync(authenticationDataSource) + .thenApply(role -> { + authRole = role; + return null; + }); + } + + @Override + public AuthenticationDataSource getAuthDataSource() { + return authenticationDataSource; + } + + @Override + public boolean isComplete() { + return true; + } + + @Override + public boolean isExpired() { + return false; + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java index 7e7c2110533ae..ab13b8aa3c7e1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java @@ -75,6 +75,7 @@ import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.auth.MockAlwaysExpiredAuthenticationProvider; +import org.apache.pulsar.broker.auth.MockMutableAuthenticationProvider; import org.apache.pulsar.broker.authentication.AuthenticationDataSubscription; import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.broker.TransactionMetadataStoreService; @@ -1030,6 +1031,54 @@ public void testVerifyAuthRoleAndAuthDataFromDirectConnectionBroker() throws Exc })); } + @Test + public void testRefreshOriginalPrincipalWithAuthDataForwardedFromProxy() throws Exception { + AuthenticationService authenticationService = mock(AuthenticationService.class); + AuthenticationProvider authenticationProvider = new MockMutableAuthenticationProvider(); + String authMethodName = authenticationProvider.getAuthMethodName(); + when(brokerService.getAuthenticationService()).thenReturn(authenticationService); + when(authenticationService.getAuthenticationProvider(authMethodName)).thenReturn(authenticationProvider); + svcConfig.setAuthenticationEnabled(true); + svcConfig.setAuthenticateOriginalAuthData(true); + svcConfig.setProxyRoles(Collections.singleton("pass.proxy")); + + resetChannel(); + assertTrue(channel.isActive()); + assertEquals(serverCnx.getState(), State.Start); + + String proxyRole = "pass.proxy"; + String clientRole = "pass.client"; + ByteBuf connect = Commands.newConnect(authMethodName, proxyRole, "test", "localhost", + clientRole, clientRole, authMethodName); + channel.writeInbound(connect); + Object connectResponse = getResponse(); + assertTrue(connectResponse instanceof CommandConnected); + assertEquals(serverCnx.getOriginalAuthData().getCommandData(), clientRole); + assertEquals(serverCnx.getOriginalAuthState().getAuthRole(), clientRole); + assertEquals(serverCnx.getOriginalPrincipal(), clientRole); + assertEquals(serverCnx.getAuthData().getCommandData(), proxyRole); + assertEquals(serverCnx.getAuthRole(), proxyRole); + assertEquals(serverCnx.getAuthState().getAuthRole(), proxyRole); + + // Request refreshing the original auth. + // Expected: + // 1. Original role and original data equals to "pass.RefreshOriginAuthData". + // 2. The broker disconnects the client, because the new role doesn't equal the old role. + String newClientRole = "pass.RefreshOriginAuthData"; + ByteBuf refreshAuth = Commands.newAuthResponse(authMethodName, + AuthData.of(newClientRole.getBytes(StandardCharsets.UTF_8)), 0, "test"); + channel.writeInbound(refreshAuth); + + assertEquals(serverCnx.getOriginalAuthData().getCommandData(), newClientRole); + assertEquals(serverCnx.getOriginalAuthState().getAuthRole(), newClientRole); + assertEquals(serverCnx.getAuthData().getCommandData(), proxyRole); + assertEquals(serverCnx.getAuthRole(), proxyRole); + assertEquals(serverCnx.getAuthState().getAuthRole(), proxyRole); + + assertFalse(channel.isOpen()); + assertFalse(channel.isActive()); + } + @Test(timeOut = 30000) public void testProducerCommand() throws Exception { resetChannel(); From 66fda61d3f6fed2a8057e0a7a259307f7bc4aa06 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 16 Feb 2023 21:47:57 +0200 Subject: [PATCH 030/174] [fix][broker] Terminate the async call chain when the condition isn't met for resetCursor (#19541) --- .../broker/admin/impl/PersistentTopicsBase.java | 2 +- .../pulsar/broker/admin/AdminApi2Test.java | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index 0214079335bb3..633c4747ee068 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -2672,7 +2672,7 @@ protected void internalResetCursorOnPosition(AsyncResponse asyncResponse, String if (topicMetadata.partitions > 0) { log.warn("[{}] Not supported operation on partitioned-topic {} {}", clientAppId(), topicName, subName); - asyncResponse.resume(new RestException(Status.METHOD_NOT_ALLOWED, + throw new CompletionException(new RestException(Status.METHOD_NOT_ALLOWED, "Reset-cursor at position is not allowed for partitioned-topic")); } return CompletableFuture.completedFuture(null); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java index f7e7bcb4ea129..66b2b1d1470ea 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java @@ -661,6 +661,22 @@ public void testResetCursorOnPosition(String namespaceName) throws Exception { setup(); } + @Test + public void shouldNotSupportResetOnPartitionedTopic() throws PulsarAdminException, PulsarClientException { + final String partitionedTopicName = "persistent://prop-xyz/ns1/" + BrokerTestUtil.newUniqueName("parttopic"); + admin.topics().createPartitionedTopic(partitionedTopicName, 4); + @Cleanup + Consumer consumer = pulsarClient.newConsumer().topic(partitionedTopicName).subscriptionName("my-sub") + .subscriptionType(SubscriptionType.Shared).subscribe(); + try { + admin.topics().resetCursor(partitionedTopicName, "my-sub", MessageId.earliest); + fail(); + } catch (PulsarAdminException.NotAllowedException e) { + assertTrue(e.getMessage().contains("Reset-cursor at position is not allowed for partitioned-topic"), + "Condition doesn't match. Actual message:" + e.getMessage()); + } + } + private void publishMessagesOnPersistentTopic(String topicName, int messages, int startIdx) throws Exception { Producer producer = pulsarClient.newProducer() .topic(topicName) From e0c0d5e8785ae8933af1bcbb4ddea59f35644c05 Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Fri, 17 Feb 2023 09:18:42 +0800 Subject: [PATCH 031/174] [feature][txn] Fix individual ack batch message with transaction abort redevlier duplicate messages (#14327) ### Motivation If individual ack batch message with transaction and abort this transaction, we will redeliver this message. but this batch message some bit sit are acked by another transaction and re consume this bit sit will produce `TransactionConflictException`, we don't need to redeliver this bit sit witch is acked by another transaction. if batch have batch size 5 1. txn1 ack 0, 1 the ackSet is 00111 2. txn2 ack 2 3 4 the ack Set is 11000 3. abort txn2 redeliver this position is 00111 4. but now we don't filter txn1 ackSet so redeliver this position bitSet is 111111 ### Modifications When filter the message we should filter the bit sit witch is real ack or in pendingAck state ### Verifying this change add the test --- .../mledger/util/PositionAckSetUtil.java | 12 +++- .../service/AbstractBaseDispatcher.java | 25 +++++++- .../persistent/PersistentSubscription.java | 4 +- .../pendingack/PendingAckHandle.java | 11 ++++ .../impl/PendingAckHandleDisabled.java | 3 + .../pendingack/impl/PendingAckHandleImpl.java | 11 ++++ .../client/impl/TransactionEndToEndTest.java | 58 ++++++++++++++++++- 7 files changed, 118 insertions(+), 6 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/PositionAckSetUtil.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/PositionAckSetUtil.java index 336c3a69e45c1..8173b30c4fea9 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/PositionAckSetUtil.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/PositionAckSetUtil.java @@ -45,12 +45,18 @@ public static void andAckSet(PositionImpl currentPosition, PositionImpl otherPos if (currentPosition == null || otherPosition == null) { return; } - BitSetRecyclable thisAckSet = BitSetRecyclable.valueOf(currentPosition.getAckSet()); - BitSetRecyclable otherAckSet = BitSetRecyclable.valueOf(otherPosition.getAckSet()); + currentPosition.setAckSet(andAckSet(currentPosition.getAckSet(), otherPosition.getAckSet())); + } + + //This method is do `and` operation for ack set + public static long[] andAckSet(long[] firstAckSet, long[] secondAckSet) { + BitSetRecyclable thisAckSet = BitSetRecyclable.valueOf(firstAckSet); + BitSetRecyclable otherAckSet = BitSetRecyclable.valueOf(secondAckSet); thisAckSet.and(otherAckSet); - currentPosition.setAckSet(thisAckSet.toLongArray()); + long[] ackSet = thisAckSet.toLongArray(); thisAckSet.recycle(); otherAckSet.recycle(); + return ackSet; } //This method is compare two position which position is bigger than another one. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java index a1437efb8a4d3..ef2fd80302a98 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.service; +import static org.apache.bookkeeper.mledger.util.PositionAckSetUtil.andAckSet; import io.netty.buffer.ByteBuf; import io.prometheus.client.Gauge; import java.util.ArrayList; @@ -37,8 +38,10 @@ import org.apache.pulsar.broker.intercept.BrokerInterceptor; import org.apache.pulsar.broker.service.persistent.CompactorSubscription; import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter; +import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.service.plugin.EntryFilter; +import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.common.api.proto.CommandAck.AckType; import org.apache.pulsar.common.api.proto.MessageMetadata; @@ -217,8 +220,28 @@ public int filterEntriesForConsumer(@Nullable MessageMetadata[] metadataArray, i batchSizes.setBatchSize(i, batchSize); long[] ackSet = null; if (indexesAcks != null && cursor != null) { + PositionImpl position = PositionImpl.get(entry.getLedgerId(), entry.getEntryId()); ackSet = cursor - .getDeletedBatchIndexesAsLongArray(PositionImpl.get(entry.getLedgerId(), entry.getEntryId())); + .getDeletedBatchIndexesAsLongArray(position); + // some batch messages ack bit sit will be in pendingAck state, so don't send all bit sit to consumer + if (subscription instanceof PersistentSubscription + && ((PersistentSubscription) subscription) + .getPendingAckHandle() instanceof PendingAckHandleImpl) { + PositionImpl positionInPendingAck = + ((PersistentSubscription) subscription).getPositionInPendingAck(position); + // if this position not in pendingAck state, don't need to do any op + if (positionInPendingAck != null) { + if (positionInPendingAck.hasAckSet()) { + // need to or ackSet in pendingAck state and cursor ackSet which bit sit has been acked + if (ackSet != null) { + ackSet = andAckSet(ackSet, positionInPendingAck.getAckSet()); + } else { + // if actSet is null, use pendingAck ackSet + ackSet = positionInPendingAck.getAckSet(); + } + } + } + } if (ackSet != null) { indexesAcks.setIndexesAcks(i, Pair.of(batchSize, ackSet)); } else { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index 2012aa06b3006..a166f1789257c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -1227,6 +1227,9 @@ public Map getSubscriptionProperties() { return subscriptionProperties; } + public PositionImpl getPositionInPendingAck(PositionImpl position) { + return pendingAckHandle.getPositionInPendingAck(position); + } @Override public CompletableFuture updateSubscriptionProperties(Map subscriptionProperties) { Map newSubscriptionProperties; @@ -1240,7 +1243,6 @@ public CompletableFuture updateSubscriptionProperties(Map this.subscriptionProperties = newSubscriptionProperties; }); } - /** * Return a merged map that contains the cursor properties specified by used * (eg. when using compaction subscription) and the subscription properties. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java index c06e4ebcc1229..a7892a56f0bd5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java @@ -159,6 +159,17 @@ CompletableFuture individualAcknowledgeMessage(TxnID txnID, List + * If it does not return null, it means this Position is in pendingAck and if it is batch Position, + * it will return the corresponding ackSet in pendingAck + * + * @param position {@link Position} witch need to get in pendingAck + * @return {@link Position} return the position in pendingAck + */ + PositionImpl getPositionInPendingAck(PositionImpl position); + /** * Get the stats of this message position is in pending ack. * @param position message position. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java index 3c428106e3eb3..0fc528f880070 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java @@ -103,6 +103,9 @@ public boolean checkIfPendingAckStoreInit() { } @Override + public PositionImpl getPositionInPendingAck(PositionImpl position) { + return null; + } public PositionInPendingAckStats checkPositionInPendingAckState(PositionImpl position, Integer batchIndex) { return null; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java index ed78feb453d1d..7dbe0385fd7e9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java @@ -1061,6 +1061,17 @@ public boolean checkIfPendingAckStoreInit() { return this.pendingAckStoreFuture != null && this.pendingAckStoreFuture.isDone(); } + @Override + public PositionImpl getPositionInPendingAck(PositionImpl position) { + if (individualAckPositions != null) { + MutablePair positionPair = this.individualAckPositions.get(position); + if (positionPair != null) { + return positionPair.getLeft(); + } + } + return null; + } + protected void handleCacheRequest() { while (true) { Runnable runnable = acceptQueue.poll(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java index 696a0a7957c47..527b8532e0452 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java @@ -120,6 +120,62 @@ public Object[][] enableBatch() { return new Object[][] { { Boolean.TRUE }, { Boolean.FALSE } }; } + @Test + private void testIndividualAckAbortFilterAckSetInPendingAckState() throws Exception { + final String topicName = NAMESPACE1 + "/testIndividualAckAbortFilterAckSetInPendingAckState"; + final int count = 9; + Producer producer = pulsarClient + .newProducer(Schema.INT32) + .topic(topicName) + .enableBatching(true) + .batchingMaxPublishDelay(1, TimeUnit.HOURS) + .batchingMaxMessages(count).create(); + + @Cleanup + Consumer consumer = pulsarClient + .newConsumer(Schema.INT32) + .topic(topicName) + .isAckReceiptEnabled(true) + .subscriptionName("test") + .subscriptionType(SubscriptionType.Shared) + .enableBatchIndexAcknowledgment(true) + .subscribe(); + + for (int i = 0; i < count; i++) { + producer.sendAsync(i); + } + + Transaction firstTransaction = getTxn(); + + Transaction secondTransaction = getTxn(); + + // firstTransaction ack the first three messages and don't end the firstTransaction + for (int i = 0; i < count / 3; i++) { + consumer.acknowledgeAsync(consumer.receive().getMessageId(), firstTransaction).get(); + } + + // if secondTransaction abort we only can receive the middle three messages + for (int i = 0; i < count / 3; i++) { + consumer.acknowledgeAsync(consumer.receive().getMessageId(), secondTransaction).get(); + } + + // consumer normal ack the last three messages + for (int i = 0; i < count / 3; i++) { + consumer.acknowledgeAsync(consumer.receive()).get(); + } + + // if secondTransaction abort we only can receive the middle three messages + secondTransaction.abort().get(); + + // can receive 3 4 5 bit sit message + for (int i = 0; i < count / 3; i++) { + assertEquals(consumer.receive().getValue().intValue(), i + 3); + } + + // can't receive message anymore + assertNull(consumer.receive(2, TimeUnit.SECONDS)); + } + @Test(dataProvider="enableBatch") private void produceCommitTest(boolean enableBatch) throws Exception { @Cleanup @@ -674,7 +730,7 @@ private void txnCumulativeAckTest(boolean batchEnable, int maxBatchSize, Subscri admin.topics().delete(normalTopic, true); } - private Transaction getTxn() throws Exception { + public Transaction getTxn() throws Exception { return pulsarClient .newTransaction() .withTransactionTimeout(10, TimeUnit.SECONDS) From 5b129924ec13afa22ee342c7f9c2fa1bd80fe835 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Fri, 17 Feb 2023 18:43:51 +0800 Subject: [PATCH 032/174] [refactor][admin] Refactor namespace bundle transfer admin api (#19525) --- .../broker/admin/impl/NamespacesBase.java | 104 ++++++++++-------- .../pulsar/broker/admin/v1/Namespaces.java | 36 ++---- .../pulsar/broker/admin/v2/Namespaces.java | 37 ++----- 3 files changed, 74 insertions(+), 103 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 5446060ac6502..6746d29af732b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -38,9 +38,7 @@ import java.util.SortedSet; import java.util.TreeSet; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.stream.Collectors; @@ -59,7 +57,7 @@ import org.apache.pulsar.broker.admin.AdminResource; import org.apache.pulsar.broker.authorization.AuthorizationService; import org.apache.pulsar.broker.loadbalance.LeaderBroker; -import org.apache.pulsar.broker.lookup.LookupResult; +import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException; import org.apache.pulsar.broker.service.Subscription; import org.apache.pulsar.broker.service.Topic; @@ -886,55 +884,73 @@ protected BookieAffinityGroupData internalGetBookieAffinityGroup() { } } - private void validateLeaderBroker() { - if (!this.isLeaderBroker()) { - LeaderBroker leaderBroker = pulsar().getLeaderElectionService().getCurrentLeader().get(); - String leaderBrokerUrl = leaderBroker.getServiceUrl(); - CompletableFuture result = pulsar().getNamespaceService() - .createLookupResult(leaderBrokerUrl, false, null); - try { - LookupResult lookupResult = result.get(2L, TimeUnit.SECONDS); - String redirectUrl = isRequestHttps() ? lookupResult.getLookupData().getHttpUrlTls() - : lookupResult.getLookupData().getHttpUrl(); - if (redirectUrl == null) { - log.error("Redirected broker's service url is not configured"); - throw new RestException(Response.Status.PRECONDITION_FAILED, - "Redirected broker's service url is not configured."); - } - URL url = new URL(redirectUrl); - URI redirect = UriBuilder.fromUri(uri.getRequestUri()).host(url.getHost()) - .port(url.getPort()) - .replaceQueryParam("authoritative", - false).build(); - - // Redirect - if (log.isDebugEnabled()) { - log.debug("Redirecting the request call to leader - {}", redirect); - } - throw new WebApplicationException(Response.temporaryRedirect(redirect).build()); - } catch (MalformedURLException exception) { - log.error("The leader broker url is malformed - {}", leaderBrokerUrl); - throw new RestException(exception); - } catch (ExecutionException | InterruptedException exception) { - log.error("Leader broker not found - {}", leaderBrokerUrl); - throw new RestException(exception.getCause()); - } catch (TimeoutException exception) { - log.error("Leader broker not found within timeout - {}", leaderBrokerUrl); - throw new RestException(exception); - } + private CompletableFuture validateLeaderBrokerAsync() { + if (this.isLeaderBroker()) { + return CompletableFuture.completedFuture(null); } + Optional currentLeaderOpt = pulsar().getLeaderElectionService().getCurrentLeader(); + if (currentLeaderOpt.isEmpty()) { + String errorStr = "The current leader is empty."; + log.error(errorStr); + return FutureUtil.failedFuture(new RestException(Response.Status.PRECONDITION_FAILED, errorStr)); + } + LeaderBroker leaderBroker = pulsar().getLeaderElectionService().getCurrentLeader().get(); + String leaderBrokerUrl = leaderBroker.getServiceUrl(); + return pulsar().getNamespaceService() + .createLookupResult(leaderBrokerUrl, false, null) + .thenCompose(lookupResult -> { + String redirectUrl = isRequestHttps() ? lookupResult.getLookupData().getHttpUrlTls() + : lookupResult.getLookupData().getHttpUrl(); + if (redirectUrl == null) { + log.error("Redirected broker's service url is not configured"); + return FutureUtil.failedFuture(new RestException(Response.Status.PRECONDITION_FAILED, + "Redirected broker's service url is not configured.")); + } + + try { + URL url = new URL(redirectUrl); + URI redirect = UriBuilder.fromUri(uri.getRequestUri()).host(url.getHost()) + .port(url.getPort()) + .replaceQueryParam("authoritative", + false).build(); + // Redirect + if (log.isDebugEnabled()) { + log.debug("Redirecting the request call to leader - {}", redirect); + } + return FutureUtil.failedFuture(( + new WebApplicationException(Response.temporaryRedirect(redirect).build()))); + } catch (MalformedURLException exception) { + log.error("The leader broker url is malformed - {}", leaderBrokerUrl); + return FutureUtil.failedFuture(new RestException(exception)); + } + }); } - public void setNamespaceBundleAffinity (String bundleRange, String destinationBroker) { + public CompletableFuture setNamespaceBundleAffinityAsync(String bundleRange, String destinationBroker) { if (StringUtils.isBlank(destinationBroker)) { - return; + return CompletableFuture.completedFuture(null); } - validateLeaderBroker(); - pulsar().getLoadManager().get().setNamespaceBundleAffinity(bundleRange, destinationBroker); + return pulsar().getLoadManager().get().getAvailableBrokersAsync() + .thenCompose(brokers -> { + if (!brokers.contains(destinationBroker)) { + log.warn("[{}] Failed to unload namespace bundle {}/{} to inactive broker {}.", + clientAppId(), namespaceName, bundleRange, destinationBroker); + return FutureUtil.failedFuture(new BrokerServiceException.NotAllowedException( + "Not allowed unload namespace bundle to inactive destination broker")); + } + return CompletableFuture.completedFuture(null); + }) + .thenCompose(__ -> validateLeaderBrokerAsync()) + .thenAccept(__ -> { + pulsar().getLoadManager().get().setNamespaceBundleAffinity(bundleRange, destinationBroker); + }); } - public CompletableFuture internalUnloadNamespaceBundleAsync(String bundleRange, boolean authoritative) { + public CompletableFuture internalUnloadNamespaceBundleAsync(String bundleRange, + String destinationBroker, + boolean authoritative) { return validateSuperUserAccessAsync() + .thenCompose(__ -> setNamespaceBundleAffinityAsync(bundleRange, destinationBroker)) .thenAccept(__ -> { checkNotNull(bundleRange, "BundleRange should not be null"); log.info("[{}] Unloading namespace bundle {}/{}", clientAppId(), namespaceName, bundleRange); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java index c13441db3dfdb..59613558eb863 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java @@ -46,7 +46,6 @@ import javax.ws.rs.core.Response.Status; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.admin.impl.NamespacesBase; -import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.web.RestException; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace.Mode; @@ -891,34 +890,13 @@ public void unloadNamespaceBundle(@Suspended final AsyncResponse asyncResponse, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @QueryParam("destinationBroker") String destinationBroker) { validateNamespaceName(property, cluster, namespace); - pulsar().getLoadManager().get().getAvailableBrokersAsync() - .thenApply(brokers -> - StringUtils.isNotBlank(destinationBroker) ? brokers.contains(destinationBroker) : true) - .thenAccept(isActiveDestination -> { - if (isActiveDestination) { - setNamespaceBundleAffinity(bundleRange, destinationBroker); - internalUnloadNamespaceBundleAsync(bundleRange, authoritative) - .thenAccept(__ -> { - log.info("[{}] Successfully unloaded namespace bundle {}", - clientAppId(), bundleRange); - asyncResponse.resume(Response.noContent().build()); - }) - .exceptionally(ex -> { - if (!isRedirectException(ex)) { - log.error("[{}] Failed to unload namespace bundle {}/{}", - clientAppId(), namespaceName, bundleRange, ex); - } - resumeAsyncResponseExceptionally(asyncResponse, ex); - return null; - }); - } else { - log.warn("[{}] Failed to unload namespace bundle {}/{} to inactive broker {}.", - clientAppId(), namespaceName, bundleRange, destinationBroker); - resumeAsyncResponseExceptionally(asyncResponse, - new BrokerServiceException.NotAllowedException( - "Not allowed unload namespace bundle to inactive destination broker")); - } - }).exceptionally(ex -> { + internalUnloadNamespaceBundleAsync(bundleRange, destinationBroker, authoritative) + .thenAccept(__ -> { + log.info("[{}] Successfully unloaded namespace bundle {}", + clientAppId(), bundleRange); + asyncResponse.resume(Response.noContent().build()); + }) + .exceptionally(ex -> { if (!isRedirectException(ex)) { log.error("[{}] Failed to unload namespace bundle {}/{}", clientAppId(), namespaceName, bundleRange, ex); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java index 80af5f4ad45b7..f5e23db79b92f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java @@ -46,10 +46,8 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.StreamingOutput; -import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.admin.impl.NamespacesBase; import org.apache.pulsar.broker.admin.impl.OffloaderObjectsScannerUtils; -import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.web.RestException; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.SubscriptionType; @@ -817,34 +815,13 @@ public void unloadNamespaceBundle(@Suspended final AsyncResponse asyncResponse, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @QueryParam("destinationBroker") String destinationBroker) { validateNamespaceName(tenant, namespace); - pulsar().getLoadManager().get().getAvailableBrokersAsync() - .thenApply(brokers -> - StringUtils.isNotBlank(destinationBroker) ? brokers.contains(destinationBroker) : true) - .thenAccept(isActiveDestination -> { - if (isActiveDestination) { - setNamespaceBundleAffinity(bundleRange, destinationBroker); - internalUnloadNamespaceBundleAsync(bundleRange, authoritative) - .thenAccept(__ -> { - log.info("[{}] Successfully unloaded namespace bundle {}", - clientAppId(), bundleRange); - asyncResponse.resume(Response.noContent().build()); - }) - .exceptionally(ex -> { - if (!isRedirectException(ex)) { - log.error("[{}] Failed to unload namespace bundle {}/{}", - clientAppId(), namespaceName, bundleRange, ex); - } - resumeAsyncResponseExceptionally(asyncResponse, ex); - return null; - }); - } else { - log.warn("[{}] Failed to unload namespace bundle {}/{} to inactive broker {}.", - clientAppId(), namespaceName, bundleRange, destinationBroker); - resumeAsyncResponseExceptionally(asyncResponse, - new BrokerServiceException.NotAllowedException( - "Not allowed unload namespace bundle to inactive destination broker")); - } - }).exceptionally(ex -> { + internalUnloadNamespaceBundleAsync(bundleRange, destinationBroker, authoritative) + .thenAccept(__ -> { + log.info("[{}] Successfully unloaded namespace bundle {}", + clientAppId(), bundleRange); + asyncResponse.resume(Response.noContent().build()); + }) + .exceptionally(ex -> { if (!isRedirectException(ex)) { log.error("[{}] Failed to unload namespace bundle {}/{}", clientAppId(), namespaceName, bundleRange, ex); From f1765be36f050bb9771a2aa7f4404d1d4824e1bf Mon Sep 17 00:00:00 2001 From: Zike Yang Date: Fri, 17 Feb 2023 18:47:57 +0800 Subject: [PATCH 033/174] [fix][proxy] Fix using wrong client version in pulsar proxy (#19540) ### Motivations Currently, if we connect the client to the proxy, the `clientVersion` won't be send to the broker and we can't get the client version using the PulsarAdmin. For example, there is no `clientVersion` field shown in the output of topic stats: ``` "publishers" : [ { "accessMode" : "Shared", "msgRateIn" : 0.0, "msgThroughputIn" : 0.0, "averageMsgSize" : 0.0, "chunkedMessageRate" : 0.0, "producerId" : 0, "metadata" : { }, "address" : "/127.0.0.1:65385", "producerName" : "AlvaroProducer", "connectedSince" : "2023-02-16T11:34:30.384548+08:00" } ], ``` It works fine when directly connecting to the broker. The root cause is that the pulsar proxy doesn't pass the clientVersion from the client to the broker. It set it to `Pulsar proxy`. And thus it will be ignored due to here : https://github.com/apache/pulsar/blob/master/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java#L693-L695 ### Modifications * Use the correct clientVersion from the client Signed-off-by: Zike Yang --- .../proxy/server/DirectProxyHandler.java | 2 +- .../pulsar/proxy/server/ProxyClientCnx.java | 3 +-- .../pulsar/proxy/server/ProxyConnection.java | 5 ++-- .../apache/pulsar/proxy/server/ProxyTest.java | 23 +++++++++++++++++++ 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java index 4b5fef3a994bd..1e9fd676573fd 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java @@ -325,7 +325,7 @@ public void channelActive(ChannelHandlerContext ctx) throws Exception { AuthData authData = authenticationDataProvider.authenticate(AuthData.INIT_AUTH_DATA); ByteBuf command = Commands.newConnect( authentication.getAuthMethodName(), authData, protocolVersion, - "Pulsar proxy", null /* target broker */, + proxyConnection.clientVersion, null /* target broker */, originalPrincipal, clientAuthData, clientAuthMethod); writeAndFlush(command); isTlsOutboundChannel = ProxyConnection.isTlsChannel(inboundChannel); diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyClientCnx.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyClientCnx.java index 6985e1f96e070..a1994fb5af4b0 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyClientCnx.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyClientCnx.java @@ -23,7 +23,6 @@ import io.netty.channel.EventLoopGroup; import java.util.Arrays; import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.PulsarVersion; import org.apache.pulsar.client.impl.ClientCnx; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.common.api.AuthData; @@ -66,7 +65,7 @@ protected ByteBuf newConnectCommand() throws Exception { authenticationDataProvider = authentication.getAuthData(remoteHostName); AuthData authData = authenticationDataProvider.authenticate(AuthData.INIT_AUTH_DATA); return Commands.newConnect(authentication.getAuthMethodName(), authData, protocolVersion, - PulsarVersion.getVersion(), proxyToTargetBrokerAddress, clientAuthRole, clientAuthData, + proxyConnection.clientVersion, proxyToTargetBrokerAddress, clientAuthRole, clientAuthData, clientAuthMethod); } diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java index 5ee79f4ad23a1..5a53f6ec014a2 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java @@ -45,7 +45,6 @@ import javax.naming.AuthenticationException; import javax.net.ssl.SSLSession; import lombok.Getter; -import org.apache.pulsar.PulsarVersion; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.authentication.AuthenticationProvider; import org.apache.pulsar.broker.authentication.AuthenticationState; @@ -99,6 +98,7 @@ public class ProxyConnection extends PulsarHandler { String clientAuthRole; AuthData clientAuthData; String clientAuthMethod; + String clientVersion; private String authMethod = "none"; AuthenticationProvider authenticationProvider; @@ -475,6 +475,7 @@ protected void handleConnect(CommandConnect connect) { this.hasProxyToBrokerUrl = connect.hasProxyToBrokerUrl(); this.protocolVersionToAdvertise = getProtocolVersionToAdvertise(connect); this.proxyToBrokerUrl = connect.hasProxyToBrokerUrl() ? connect.getProxyToBrokerUrl() : "null"; + this.clientVersion = connect.getClientVersion(); if (LOG.isDebugEnabled()) { LOG.debug("Received CONNECT from {} proxyToBroker={}", remoteAddress, proxyToBrokerUrl); @@ -568,7 +569,7 @@ protected void handleAuthResponse(CommandAuthResponse authResponse) { if (authResponse.hasClientVersion()) { clientVersion = authResponse.getClientVersion(); } else { - clientVersion = PulsarVersion.getVersion(); + clientVersion = this.clientVersion; } int protocolVersion; if (authResponse.hasProtocolVersion()) { diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTest.java index 6c9a834bb042b..af128ce036f53 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTest.java @@ -35,6 +35,7 @@ import lombok.EqualsAndHashCode; import lombok.ToString; import org.apache.avro.reflect.Nullable; +import org.apache.pulsar.PulsarVersion; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.authentication.AuthenticationService; import org.apache.pulsar.client.api.Consumer; @@ -311,6 +312,28 @@ public void testProtocolVersionAdvertisement() throws Exception { } } + @Test + public void testGetClientVersion() throws Exception { + @Cleanup + PulsarClient client = PulsarClient.builder().serviceUrl(proxyService.getServiceUrl()) + .build(); + + String topic = "persistent://sample/test/local/testGetClientVersion"; + String subName = "test-sub"; + + @Cleanup + Consumer consumer = client.newConsumer() + .topic(topic) + .subscriptionName(subName) + .subscribe(); + + consumer.receiveAsync(); + + + Assert.assertEquals(admin.topics().getStats(topic).getSubscriptions().get(subName).getConsumers() + .get(0).getClientVersion(), PulsarVersion.getVersion()); + } + private static PulsarClient getClientActiveConsumerChangeNotSupported(ClientConfigurationData conf) throws Exception { ThreadFactory threadFactory = new DefaultThreadFactory("pulsar-client-io", Thread.currentThread().isDaemon()); From 38555851359f9cfc172650c387a58c5a03809e97 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Fri, 17 Feb 2023 21:39:11 +0800 Subject: [PATCH 034/174] [fix][broker] Fix delete namespace fail by a In-flight topic (#19374) --- .../broker/admin/impl/NamespacesBase.java | 183 +++++++++++------- .../pulsar/broker/admin/v1/Namespaces.java | 7 - .../pulsar/broker/admin/v2/Namespaces.java | 7 - .../pulsar/broker/service/AbstractTopic.java | 3 - .../pulsar/broker/admin/AdminApi2Test.java | 6 + 5 files changed, 119 insertions(+), 87 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 6746d29af732b..30f01ece7de11 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -42,6 +42,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.stream.Collectors; +import javax.annotation.Nonnull; import javax.ws.rs.WebApplicationException; import javax.ws.rs.container.AsyncResponse; import javax.ws.rs.core.Response; @@ -113,6 +114,7 @@ import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.api.MetadataStoreException.BadVersionException; import org.apache.pulsar.metadata.api.MetadataStoreException.NotFoundException; +import org.apache.zookeeper.KeeperException; @Slf4j public abstract class NamespacesBase extends AdminResource { @@ -202,78 +204,94 @@ protected CompletableFuture> internalGetNonPersistentTopics(Policie }); } - @SuppressWarnings("unchecked") - protected CompletableFuture internalDeleteNamespaceAsync(boolean force) { - CompletableFuture preconditionCheck = precheckWhenDeleteNamespace(namespaceName, force); - return preconditionCheck + /** + * Delete the namespace and retry to resolve some topics that were not created successfully(in metadata) + * during the deletion. + */ + protected @Nonnull CompletableFuture internalDeleteNamespaceAsync(boolean force) { + final CompletableFuture future = new CompletableFuture<>(); + internalRetryableDeleteNamespaceAsync0(force, 5, future); + return future; + } + private void internalRetryableDeleteNamespaceAsync0(boolean force, int retryTimes, + @Nonnull CompletableFuture callback) { + precheckWhenDeleteNamespace(namespaceName, force) .thenCompose(policies -> { + final CompletableFuture> topicsFuture; if (policies == null || CollectionUtils.isEmpty(policies.replication_clusters)){ - return pulsar().getNamespaceService().getListOfPersistentTopics(namespaceName); - } - return pulsar().getNamespaceService().getFullListOfTopics(namespaceName); - }) - .thenCompose(allTopics -> pulsar().getNamespaceService().getFullListOfPartitionedTopic(namespaceName) - .thenCompose(allPartitionedTopics -> { - List> topicsSum = new ArrayList<>(2); - topicsSum.add(allTopics); - topicsSum.add(allPartitionedTopics); - return CompletableFuture.completedFuture(topicsSum); - })) - .thenCompose(topics -> { - List allTopics = topics.get(0); - ArrayList allUserCreatedTopics = new ArrayList<>(); - List allPartitionedTopics = topics.get(1); - ArrayList allUserCreatedPartitionTopics = new ArrayList<>(); - boolean hasNonSystemTopic = false; - List allSystemTopics = new ArrayList<>(); - List allPartitionedSystemTopics = new ArrayList<>(); - List topicPolicy = new ArrayList<>(); - List partitionedTopicPolicy = new ArrayList<>(); - for (String topic : allTopics) { - if (!pulsar().getBrokerService().isSystemTopic(TopicName.get(topic))) { - hasNonSystemTopic = true; - allUserCreatedTopics.add(topic); - } else { - if (SystemTopicNames.isTopicPoliciesSystemTopic(topic)) { - topicPolicy.add(topic); - } else { - allSystemTopics.add(topic); - } - } - } - for (String topic : allPartitionedTopics) { - if (!pulsar().getBrokerService().isSystemTopic(TopicName.get(topic))) { - hasNonSystemTopic = true; - allUserCreatedPartitionTopics.add(topic); - } else { - if (SystemTopicNames.isTopicPoliciesSystemTopic(topic)) { - partitionedTopicPolicy.add(topic); - } else { - allPartitionedSystemTopics.add(topic); - } - } - } - if (!force) { - if (hasNonSystemTopic) { - throw new RestException(Status.CONFLICT, "Cannot delete non empty namespace"); - } + topicsFuture = pulsar().getNamespaceService().getListOfPersistentTopics(namespaceName); + } else { + topicsFuture = pulsar().getNamespaceService().getFullListOfTopics(namespaceName); } - return namespaceResources().setPoliciesAsync(namespaceName, old -> { - old.deleted = true; - return old; - }).thenCompose(ignore -> { - return internalDeleteTopicsAsync(allUserCreatedTopics); - }).thenCompose(ignore -> { - return internalDeletePartitionedTopicsAsync(allUserCreatedPartitionTopics); - }).thenCompose(ignore -> { - return internalDeleteTopicsAsync(allSystemTopics); - }).thenCompose(ignore__ -> { - return internalDeletePartitionedTopicsAsync(allPartitionedSystemTopics); - }).thenCompose(ignore -> { - return internalDeleteTopicsAsync(topicPolicy); - }).thenCompose(ignore__ -> { - return internalDeletePartitionedTopicsAsync(partitionedTopicPolicy); - }); + return topicsFuture.thenCompose(allTopics -> + pulsar().getNamespaceService().getFullListOfPartitionedTopic(namespaceName) + .thenCompose(allPartitionedTopics -> { + List> topicsSum = new ArrayList<>(2); + topicsSum.add(allTopics); + topicsSum.add(allPartitionedTopics); + return CompletableFuture.completedFuture(topicsSum); + })) + .thenCompose(topics -> { + List allTopics = topics.get(0); + ArrayList allUserCreatedTopics = new ArrayList<>(); + List allPartitionedTopics = topics.get(1); + ArrayList allUserCreatedPartitionTopics = new ArrayList<>(); + boolean hasNonSystemTopic = false; + List allSystemTopics = new ArrayList<>(); + List allPartitionedSystemTopics = new ArrayList<>(); + List topicPolicy = new ArrayList<>(); + List partitionedTopicPolicy = new ArrayList<>(); + for (String topic : allTopics) { + if (!pulsar().getBrokerService().isSystemTopic(TopicName.get(topic))) { + hasNonSystemTopic = true; + allUserCreatedTopics.add(topic); + } else { + if (SystemTopicNames.isTopicPoliciesSystemTopic(topic)) { + topicPolicy.add(topic); + } else { + allSystemTopics.add(topic); + } + } + } + for (String topic : allPartitionedTopics) { + if (!pulsar().getBrokerService().isSystemTopic(TopicName.get(topic))) { + hasNonSystemTopic = true; + allUserCreatedPartitionTopics.add(topic); + } else { + if (SystemTopicNames.isTopicPoliciesSystemTopic(topic)) { + partitionedTopicPolicy.add(topic); + } else { + allPartitionedSystemTopics.add(topic); + } + } + } + if (!force) { + if (hasNonSystemTopic) { + throw new RestException(Status.CONFLICT, "Cannot delete non empty namespace"); + } + } + final CompletableFuture markDeleteFuture; + if (policies != null && policies.deleted) { + markDeleteFuture = CompletableFuture.completedFuture(null); + } else { + markDeleteFuture = namespaceResources().setPoliciesAsync(namespaceName, old -> { + old.deleted = true; + return old; + }); + } + return markDeleteFuture.thenCompose(__ -> + internalDeleteTopicsAsync(allUserCreatedTopics)) + .thenCompose(ignore -> + internalDeletePartitionedTopicsAsync(allUserCreatedPartitionTopics)) + .thenCompose(ignore -> + internalDeleteTopicsAsync(allSystemTopics)) + .thenCompose(ignore -> + internalDeletePartitionedTopicsAsync(allPartitionedSystemTopics)) + .thenCompose(ignore -> + internalDeleteTopicsAsync(topicPolicy)) + .thenCompose(ignore -> + internalDeletePartitionedTopicsAsync(partitionedTopicPolicy)); + }); }) .thenCompose(ignore -> pulsar().getNamespaceService() .getNamespaceBundleFactory().getBundlesAsync(namespaceName)) @@ -297,7 +315,32 @@ protected CompletableFuture internalDeleteNamespaceAsync(boolean force) { return CompletableFuture.completedFuture(null); }) ).collect(Collectors.toList()))) - .thenCompose(ignore -> internalClearZkSources()); + .thenCompose(ignore -> internalClearZkSources()) + .whenComplete((result, error) -> { + if (error != null) { + final Throwable rc = FutureUtil.unwrapCompletionException(error); + if (rc instanceof MetadataStoreException) { + if (rc.getCause() != null && rc.getCause() instanceof KeeperException.NotEmptyException) { + log.info("[{}] There are in-flight topics created during the namespace deletion, " + + "retry to delete the namespace again.", namespaceName); + final int next = retryTimes - 1; + if (next > 0) { + // async recursive + internalRetryableDeleteNamespaceAsync0(force, next, callback); + } else { + callback.completeExceptionally( + new RestException(Status.CONFLICT, "The broker still have in-flight topics" + + " created during namespace deletion, please try again.")); + // drop out recursive + } + return; + } + } + callback.completeExceptionally(error); + return; + } + callback.complete(result); + }); } private CompletableFuture internalDeletePartitionedTopicsAsync(List topicNames) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java index 59613558eb863..153e29506c3d0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java @@ -47,7 +47,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.admin.impl.NamespacesBase; import org.apache.pulsar.broker.web.RestException; -import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace.Mode; import org.apache.pulsar.common.naming.NamespaceBundleSplitAlgorithm; import org.apache.pulsar.common.naming.NamespaceName; @@ -249,12 +248,6 @@ public void deleteNamespace(@Suspended final AsyncResponse asyncResponse, @PathP asyncResponse.resume(Response.noContent().build()); }) .exceptionally(ex -> { - Throwable cause = FutureUtil.unwrapCompletionException(ex); - if (cause instanceof PulsarAdminException.ConflictException) { - log.info("[{}] There are new topics created during the namespace deletion, " - + "retry to delete the namespace again.", namespaceName); - pulsar().getExecutor().execute(() -> internalDeleteNamespaceAsync(force)); - } if (!isRedirectException(ex)) { log.error("[{}] Failed to delete namespace {}", clientAppId(), namespaceName, ex); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java index f5e23db79b92f..12ff229c2f0ed 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java @@ -49,7 +49,6 @@ import org.apache.pulsar.broker.admin.impl.NamespacesBase; import org.apache.pulsar.broker.admin.impl.OffloaderObjectsScannerUtils; import org.apache.pulsar.broker.web.RestException; -import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace.Mode; import org.apache.pulsar.common.naming.NamespaceName; @@ -197,12 +196,6 @@ public void deleteNamespace(@Suspended final AsyncResponse asyncResponse, @PathP asyncResponse.resume(Response.noContent().build()); }) .exceptionally(ex -> { - Throwable cause = FutureUtil.unwrapCompletionException(ex); - if (cause instanceof PulsarAdminException.ConflictException) { - log.info("[{}] There are new topics created during the namespace deletion, " - + "retry to delete the namespace again.", namespaceName); - pulsar().getExecutor().execute(() -> internalDeleteNamespaceAsync(force)); - } if (!isRedirectException(ex)) { log.error("[{}] Failed to delete namespace {}", clientAppId(), namespaceName, ex); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 4e095cd66ba08..6245ce19eebc6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -249,9 +249,6 @@ protected void updateTopicPolicyByNamespacePolicy(Policies namespacePolicies) { if (log.isDebugEnabled()) { log.debug("[{}]updateTopicPolicyByNamespacePolicy,data={}", topic, namespacePolicies); } - if (namespacePolicies.deleted) { - return; - } topicPolicies.getRetentionPolicies().updateNamespaceValue(namespacePolicies.retention_policies); topicPolicies.getCompactionThreshold().updateNamespaceValue(namespacePolicies.compaction_threshold); topicPolicies.getReplicationClusters().updateNamespaceValue( diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java index 66b2b1d1470ea..90be220cfd8f7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java @@ -46,6 +46,7 @@ import java.util.Set; import java.util.TreeSet; import java.util.UUID; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import javax.ws.rs.NotAcceptableException; import javax.ws.rs.core.Response.Status; @@ -1681,6 +1682,11 @@ public void testDeleteNamespaceWithTopicPolicies() throws Exception { // verify namespace can be deleted even without topic policy events admin.namespaces().deleteNamespace(namespace, true); + Awaitility.await().untilAsserted(() -> { + final CompletableFuture> eventTopicFuture = + pulsar.getBrokerService().getTopics().get("persistent://test-tenant/test-ns2/__change_events"); + assertNull(eventTopicFuture); + }); admin.namespaces().createNamespace(namespace, Set.of("test")); // create topic String topic = namespace + "/test-topic2"; From e0b50c9ec5f12d0fb8275f235d8ac00e87a9099e Mon Sep 17 00:00:00 2001 From: StevenLuMT Date: Mon, 20 Feb 2023 10:24:53 +0800 Subject: [PATCH 035/174] [fix][client] Logger.warn usage bug fix: the number is inconsistent for format and arguments (#19562) Co-authored-by: lushiji --- .../java/org/apache/pulsar/client/impl/ConsumerImpl.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index eecd51c788f72..08a6bb15807c9 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -2028,10 +2028,12 @@ private CompletableFuture processPossibleToDLQ(MessageIdImpl messageId) }).exceptionally(ex -> { if (ex instanceof PulsarClientException.ProducerQueueIsFullError) { log.warn("[{}] [{}] [{}] Failed to send DLQ message to {} for message id {}: {}", - topicName, subscription, consumerName, finalMessageId, ex.getMessage()); + topicName, subscription, consumerName, + deadLetterPolicy.getDeadLetterTopic(), finalMessageId, ex.getMessage()); } else { log.warn("[{}] [{}] [{}] Failed to send DLQ message to {} for message id {}", - topicName, subscription, consumerName, finalMessageId, ex); + topicName, subscription, consumerName, + deadLetterPolicy.getDeadLetterTopic(), finalMessageId, ex); } result.complete(false); return null; From 954f406fab03ce8858883335ce6d5a676bb45c92 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Tue, 21 Feb 2023 11:21:12 +0800 Subject: [PATCH 036/174] [fix] [test] Wrong mock-fail of the test ManagedLedgerErrorsTest.recoverLongTimeAfterMultipleWriteErrors (#19545) --- .../mledger/impl/ManagedLedgerErrorsTest.java | 20 +++++++++++++++++-- .../client/PulsarMockBookKeeper.java | 18 ++++++++++++++++- .../client/PulsarMockLedgerHandle.java | 2 +- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerErrorsTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerErrorsTest.java index eac22d2469b17..512e90d17f5e8 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerErrorsTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerErrorsTest.java @@ -34,6 +34,8 @@ import java.util.concurrent.atomic.AtomicReference; import lombok.Cleanup; import org.apache.bookkeeper.client.BKException; +import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.client.api.DigestType; import org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback; @@ -48,6 +50,7 @@ import org.apache.bookkeeper.test.MockedBookKeeperTestCase; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.impl.FaultInjectionMetadataStore; +import org.awaitility.Awaitility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.annotations.Test; @@ -511,9 +514,10 @@ public void recoverAfterWriteError() throws Exception { public void recoverLongTimeAfterMultipleWriteErrors() throws Exception { ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("recoverLongTimeAfterMultipleWriteErrors"); ManagedCursor cursor = ledger.openCursor("c1"); + LedgerHandle firstLedger = ledger.currentLedger; - bkc.failAfter(0, BKException.Code.BookieHandleNotAvailableException); - bkc.failAfter(1, BKException.Code.BookieHandleNotAvailableException); + bkc.addEntryFailAfter(0, BKException.Code.BookieHandleNotAvailableException); + bkc.addEntryFailAfter(1, BKException.Code.BookieHandleNotAvailableException); CountDownLatch counter = new CountDownLatch(2); AtomicReference ex = new AtomicReference<>(); @@ -540,6 +544,18 @@ public void addFailed(ManagedLedgerException exception, Object ctx) { counter.await(); assertNull(ex.get()); + Awaitility.await().untilAsserted(() -> { + try { + bkc.openLedger(firstLedger.getId(), + BookKeeper.DigestType.fromApiDigestType(ledger.getConfig().getDigestType()), + ledger.getConfig().getPassword()); + fail("The expected behavior is that the first ledger will be deleted, but it still exists."); + } catch (Exception ledgerDeletedEx){ + // Expected LedgerNotExistsEx: the first ledger will be deleted after add entry fail. + assertTrue(ledgerDeletedEx instanceof BKException.BKNoSuchLedgerExistsException); + } + }); + assertEquals(cursor.getNumberOfEntriesInBacklog(false), 2); // Ensure that we are only creating one new ledger diff --git a/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java b/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java index 8d6c39d9a82a1..f0d279ef25050 100644 --- a/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java +++ b/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java @@ -90,6 +90,7 @@ public static Collection getMockEnsemble() { final Queue addEntryDelaysMillis = new ConcurrentLinkedQueue<>(); final List> failures = new ArrayList<>(); + final List> addEntryFailures = new ArrayList<>(); public PulsarMockBookKeeper(OrderedExecutor orderedExecutor) throws Exception { this.orderedExecutor = orderedExecutor; @@ -317,6 +318,13 @@ synchronized boolean checkReturnEmptyLedger() { return shouldFailNow; } + synchronized CompletableFuture getAddEntryFailure() { + if (!addEntryFailures.isEmpty()){ + return addEntryFailures.remove(0); + } + return failures.isEmpty() ? defaultResponse : failures.remove(0); + } + synchronized CompletableFuture getProgrammedFailure() { return failures.isEmpty() ? defaultResponse : failures.remove(0); } @@ -326,7 +334,11 @@ public void failNow(int rc) { } public void failAfter(int steps, int rc) { - promiseAfter(steps).completeExceptionally(BKException.create(rc)); + promiseAfter(steps, failures).completeExceptionally(BKException.create(rc)); + } + + public void addEntryFailAfter(int steps, int rc) { + promiseAfter(steps, addEntryFailures).completeExceptionally(BKException.create(rc)); } private int emptyLedgerAfter = -1; @@ -339,6 +351,10 @@ public synchronized void returnEmptyLedgerAfter(int steps) { } public synchronized CompletableFuture promiseAfter(int steps) { + return promiseAfter(steps, failures); + } + + public synchronized CompletableFuture promiseAfter(int steps, List> failures) { while (failures.size() <= steps) { failures.add(defaultResponse); } diff --git a/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockLedgerHandle.java b/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockLedgerHandle.java index 2790160a72141..dea33a0e67662 100644 --- a/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockLedgerHandle.java +++ b/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockLedgerHandle.java @@ -168,7 +168,7 @@ public void asyncAddEntry(final byte[] data, final int offset, final int length, @Override public void asyncAddEntry(final ByteBuf data, final AddCallback cb, final Object ctx) { - bk.getProgrammedFailure().thenComposeAsync((res) -> { + bk.getAddEntryFailure().thenComposeAsync((res) -> { Long delayMillis = bk.addEntryDelaysMillis.poll(); if (delayMillis == null) { delayMillis = 1L; From 25beb9724bc915d04e4179081dc93f43eacaa0dd Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Tue, 21 Feb 2023 12:29:39 +0800 Subject: [PATCH 037/174] [improve] Bump project version to 3.0.0-SNAPSHOT (#19573) --- bouncy-castle/bc/pom.xml | 2 +- bouncy-castle/bcfips-include-test/pom.xml | 2 +- bouncy-castle/bcfips/pom.xml | 2 +- bouncy-castle/pom.xml | 2 +- buildtools/pom.xml | 2 +- distribution/io/pom.xml | 2 +- distribution/offloaders/pom.xml | 2 +- distribution/pom.xml | 2 +- distribution/server/pom.xml | 2 +- distribution/shell/pom.xml | 2 +- docker/pom.xml | 2 +- docker/pulsar-all/pom.xml | 2 +- docker/pulsar/pom.xml | 2 +- jclouds-shaded/pom.xml | 2 +- managed-ledger/pom.xml | 2 +- pom.xml | 2 +- pulsar-broker-auth-athenz/pom.xml | 2 +- pulsar-broker-auth-sasl/pom.xml | 2 +- pulsar-broker-common/pom.xml | 2 +- pulsar-broker/pom.xml | 2 +- pulsar-client-1x-base/pom.xml | 2 +- pulsar-client-1x-base/pulsar-client-1x/pom.xml | 2 +- pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml | 2 +- pulsar-client-admin-api/pom.xml | 2 +- pulsar-client-admin-shaded/pom.xml | 2 +- pulsar-client-admin/pom.xml | 2 +- pulsar-client-all/pom.xml | 2 +- pulsar-client-api/pom.xml | 2 +- pulsar-client-auth-athenz/pom.xml | 2 +- pulsar-client-auth-sasl/pom.xml | 2 +- pulsar-client-messagecrypto-bc/pom.xml | 2 +- pulsar-client-shaded/pom.xml | 2 +- pulsar-client-tools-api/pom.xml | 2 +- pulsar-client-tools-customcommand-example/pom.xml | 2 +- pulsar-client-tools-test/pom.xml | 2 +- pulsar-client-tools/pom.xml | 2 +- pulsar-client/pom.xml | 2 +- pulsar-common/pom.xml | 2 +- pulsar-config-validation/pom.xml | 2 +- pulsar-functions/api-java/pom.xml | 2 +- pulsar-functions/instance/pom.xml | 2 +- pulsar-functions/java-examples-builtin/pom.xml | 2 +- pulsar-functions/java-examples/pom.xml | 2 +- pulsar-functions/localrun-shaded/pom.xml | 2 +- pulsar-functions/localrun/pom.xml | 2 +- pulsar-functions/pom.xml | 2 +- pulsar-functions/proto/pom.xml | 2 +- pulsar-functions/runtime-all/pom.xml | 2 +- pulsar-functions/runtime/pom.xml | 2 +- pulsar-functions/secrets/pom.xml | 2 +- pulsar-functions/utils/pom.xml | 2 +- pulsar-functions/worker/pom.xml | 2 +- pulsar-io/aerospike/pom.xml | 2 +- pulsar-io/alluxio/pom.xml | 2 +- pulsar-io/aws/pom.xml | 2 +- pulsar-io/batch-data-generator/pom.xml | 2 +- pulsar-io/batch-discovery-triggerers/pom.xml | 2 +- pulsar-io/canal/pom.xml | 2 +- pulsar-io/cassandra/pom.xml | 2 +- pulsar-io/common/pom.xml | 2 +- pulsar-io/core/pom.xml | 2 +- pulsar-io/data-generator/pom.xml | 2 +- pulsar-io/debezium/core/pom.xml | 2 +- pulsar-io/debezium/mongodb/pom.xml | 2 +- pulsar-io/debezium/mssql/pom.xml | 2 +- pulsar-io/debezium/mysql/pom.xml | 2 +- pulsar-io/debezium/oracle/pom.xml | 2 +- pulsar-io/debezium/pom.xml | 2 +- pulsar-io/debezium/postgres/pom.xml | 2 +- pulsar-io/docs/pom.xml | 2 +- pulsar-io/dynamodb/pom.xml | 2 +- pulsar-io/elastic-search/pom.xml | 2 +- pulsar-io/file/pom.xml | 2 +- pulsar-io/flume/pom.xml | 2 +- pulsar-io/hbase/pom.xml | 2 +- pulsar-io/hdfs2/pom.xml | 2 +- pulsar-io/hdfs3/pom.xml | 2 +- pulsar-io/http/pom.xml | 2 +- pulsar-io/influxdb/pom.xml | 2 +- pulsar-io/jdbc/clickhouse/pom.xml | 2 +- pulsar-io/jdbc/core/pom.xml | 2 +- pulsar-io/jdbc/mariadb/pom.xml | 2 +- pulsar-io/jdbc/openmldb/pom.xml | 2 +- pulsar-io/jdbc/pom.xml | 2 +- pulsar-io/jdbc/postgres/pom.xml | 2 +- pulsar-io/jdbc/sqlite/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor-nar/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor/pom.xml | 2 +- pulsar-io/kafka/pom.xml | 2 +- pulsar-io/kinesis/pom.xml | 2 +- pulsar-io/mongo/pom.xml | 2 +- pulsar-io/netty/pom.xml | 2 +- pulsar-io/nsq/pom.xml | 2 +- pulsar-io/pom.xml | 2 +- pulsar-io/rabbitmq/pom.xml | 2 +- pulsar-io/redis/pom.xml | 2 +- pulsar-io/solr/pom.xml | 2 +- pulsar-io/twitter/pom.xml | 2 +- pulsar-metadata/pom.xml | 2 +- pulsar-package-management/bookkeeper-storage/pom.xml | 2 +- pulsar-package-management/core/pom.xml | 2 +- pulsar-package-management/filesystem-storage/pom.xml | 2 +- pulsar-package-management/pom.xml | 2 +- pulsar-proxy/pom.xml | 2 +- pulsar-sql/pom.xml | 2 +- pulsar-sql/presto-distribution/pom.xml | 2 +- pulsar-sql/presto-pulsar-plugin/pom.xml | 2 +- pulsar-sql/presto-pulsar/pom.xml | 2 +- pulsar-testclient/pom.xml | 2 +- pulsar-transaction/common/pom.xml | 2 +- pulsar-transaction/coordinator/pom.xml | 2 +- pulsar-transaction/pom.xml | 2 +- pulsar-websocket/pom.xml | 2 +- structured-event-log/pom.xml | 2 +- testmocks/pom.xml | 2 +- tests/bc_2_0_0/pom.xml | 2 +- tests/bc_2_0_1/pom.xml | 2 +- tests/bc_2_6_0/pom.xml | 2 +- tests/docker-images/java-test-functions/pom.xml | 2 +- tests/docker-images/java-test-image/pom.xml | 2 +- tests/docker-images/java-test-plugins/pom.xml | 2 +- tests/docker-images/latest-version-image/pom.xml | 2 +- tests/docker-images/pom.xml | 2 +- tests/integration/pom.xml | 2 +- tests/pom.xml | 2 +- tests/pulsar-client-admin-shade-test/pom.xml | 2 +- tests/pulsar-client-all-shade-test/pom.xml | 2 +- tests/pulsar-client-shade-test/pom.xml | 2 +- tiered-storage/file-system/pom.xml | 2 +- tiered-storage/jcloud/pom.xml | 2 +- tiered-storage/pom.xml | 2 +- 131 files changed, 131 insertions(+), 131 deletions(-) diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml index 0e2f8db659002..cc7e7952b69f0 100644 --- a/bouncy-castle/bc/pom.xml +++ b/bouncy-castle/bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml index 41ea4590165fe..44f0ada4630d7 100644 --- a/bouncy-castle/bcfips-include-test/pom.xml +++ b/bouncy-castle/bcfips-include-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar bouncy-castle-parent - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml index 7feee8c27afd0..bd5d64bd84191 100644 --- a/bouncy-castle/bcfips/pom.xml +++ b/bouncy-castle/bcfips/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml index 46fb1db1bc807..9f8ace79b77c3 100644 --- a/bouncy-castle/pom.xml +++ b/bouncy-castle/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/buildtools/pom.xml b/buildtools/pom.xml index e6591aedbb045..83867fbb46c71 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -31,7 +31,7 @@ org.apache.pulsar buildtools - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT jar Pulsar Build Tools diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml index cb8591f0d066a..99105bef950d5 100644 --- a/distribution/io/pom.xml +++ b/distribution/io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml index 94395208d4969..1e86758ed5a6a 100644 --- a/distribution/offloaders/pom.xml +++ b/distribution/offloaders/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/distribution/pom.xml b/distribution/pom.xml index da1e002dddd3c..9782f269284bf 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index 7bf8c8b32e3a0..2043da516cf40 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/distribution/shell/pom.xml b/distribution/shell/pom.xml index 5c6c332120c96..b38baee4257ba 100644 --- a/distribution/shell/pom.xml +++ b/distribution/shell/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/docker/pom.xml b/docker/pom.xml index 77c427b05aef2..4d3b05fe33a76 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT docker-images Apache Pulsar :: Docker Images diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index fba5b1df5070c..c63ff6d656957 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT 4.0.0 pulsar-all-docker-image diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index 293d1daf8e208..647f68bf1672c 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT 4.0.0 pulsar-docker-image diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml index 41c713c4f5d8f..d4138ea041317 100644 --- a/jclouds-shaded/pom.xml +++ b/jclouds-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index b0eab9a36bb6a..2a7b9c576d318 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/pom.xml b/pom.xml index 07e7ad634dfb8..2879c122f0062 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT Pulsar Pulsar is a distributed pub-sub messaging platform with a very diff --git a/pulsar-broker-auth-athenz/pom.xml b/pulsar-broker-auth-athenz/pom.xml index 6d2bc54a25049..419346c7adb90 100644 --- a/pulsar-broker-auth-athenz/pom.xml +++ b/pulsar-broker-auth-athenz/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-broker-auth-athenz diff --git a/pulsar-broker-auth-sasl/pom.xml b/pulsar-broker-auth-sasl/pom.xml index 935f5e541dca9..8cde4a2fc4552 100644 --- a/pulsar-broker-auth-sasl/pom.xml +++ b/pulsar-broker-auth-sasl/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-broker-auth-sasl diff --git a/pulsar-broker-common/pom.xml b/pulsar-broker-common/pom.xml index c2064122271a2..3e024d14b92a0 100644 --- a/pulsar-broker-common/pom.xml +++ b/pulsar-broker-common/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-broker-common diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index 074c6fc8a9bc0..8fc1527a6e3e8 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/pulsar-client-1x-base/pom.xml b/pulsar-client-1x-base/pom.xml index 0c531fbc32ebd..f039c23f92cb3 100644 --- a/pulsar-client-1x-base/pom.xml +++ b/pulsar-client-1x-base/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/pulsar-client-1x-base/pulsar-client-1x/pom.xml b/pulsar-client-1x-base/pulsar-client-1x/pom.xml index 5be477dde092d..a0a254317bfc8 100644 --- a/pulsar-client-1x-base/pulsar-client-1x/pom.xml +++ b/pulsar-client-1x-base/pulsar-client-1x/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-client-1x-base - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml b/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml index 9c652d5557342..bc5e003b108d7 100644 --- a/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml +++ b/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-client-1x-base - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/pulsar-client-admin-api/pom.xml b/pulsar-client-admin-api/pom.xml index 1df9edf4f8cc6..69c423e86e246 100644 --- a/pulsar-client-admin-api/pom.xml +++ b/pulsar-client-admin-api/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/pulsar-client-admin-shaded/pom.xml b/pulsar-client-admin-shaded/pom.xml index 4fca4b1bb25ad..cf669b5869afb 100644 --- a/pulsar-client-admin-shaded/pom.xml +++ b/pulsar-client-admin-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/pulsar-client-admin/pom.xml b/pulsar-client-admin/pom.xml index 1f5541ba12664..9436f9c1ce245 100644 --- a/pulsar-client-admin/pom.xml +++ b/pulsar-client-admin/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml index 62f98b8316eec..b06c71616b6a5 100644 --- a/pulsar-client-all/pom.xml +++ b/pulsar-client-all/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/pulsar-client-api/pom.xml b/pulsar-client-api/pom.xml index f502d9ed97ba0..242cedc594e4e 100644 --- a/pulsar-client-api/pom.xml +++ b/pulsar-client-api/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/pulsar-client-auth-athenz/pom.xml b/pulsar-client-auth-athenz/pom.xml index b0606aad17f9a..24d7e259f790d 100644 --- a/pulsar-client-auth-athenz/pom.xml +++ b/pulsar-client-auth-athenz/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/pulsar-client-auth-sasl/pom.xml b/pulsar-client-auth-sasl/pom.xml index 4c9381219d000..9744a230cf3af 100644 --- a/pulsar-client-auth-sasl/pom.xml +++ b/pulsar-client-auth-sasl/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/pulsar-client-messagecrypto-bc/pom.xml b/pulsar-client-messagecrypto-bc/pom.xml index 8e59ed131dbff..6292d99003dd6 100644 --- a/pulsar-client-messagecrypto-bc/pom.xml +++ b/pulsar-client-messagecrypto-bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/pulsar-client-shaded/pom.xml b/pulsar-client-shaded/pom.xml index ce5e6acf152d8..bf3d8a802dff2 100644 --- a/pulsar-client-shaded/pom.xml +++ b/pulsar-client-shaded/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/pulsar-client-tools-api/pom.xml b/pulsar-client-tools-api/pom.xml index 21899ff033e04..d6420a69e3a2a 100644 --- a/pulsar-client-tools-api/pom.xml +++ b/pulsar-client-tools-api/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/pulsar-client-tools-customcommand-example/pom.xml b/pulsar-client-tools-customcommand-example/pom.xml index a25b2ac40647c..f1f9180e2d550 100644 --- a/pulsar-client-tools-customcommand-example/pom.xml +++ b/pulsar-client-tools-customcommand-example/pom.xml @@ -22,7 +22,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. 4.0.0 diff --git a/pulsar-client-tools-test/pom.xml b/pulsar-client-tools-test/pom.xml index 53610ccf73593..9edb3c1129ed5 100644 --- a/pulsar-client-tools-test/pom.xml +++ b/pulsar-client-tools-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/pulsar-client-tools/pom.xml b/pulsar-client-tools/pom.xml index cb7a411b426da..3a0b03d2c616e 100644 --- a/pulsar-client-tools/pom.xml +++ b/pulsar-client-tools/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/pulsar-client/pom.xml b/pulsar-client/pom.xml index bf599d2f3eb66..9d0dd08eecf10 100644 --- a/pulsar-client/pom.xml +++ b/pulsar-client/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml index 72a2ce358b25b..643b03a59c0f7 100644 --- a/pulsar-common/pom.xml +++ b/pulsar-common/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/pulsar-config-validation/pom.xml b/pulsar-config-validation/pom.xml index e67730ce84913..5ce8f74d6ebad 100644 --- a/pulsar-config-validation/pom.xml +++ b/pulsar-config-validation/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/pulsar-functions/api-java/pom.xml b/pulsar-functions/api-java/pom.xml index 07d00aaff3008..0f6e33bc05de5 100644 --- a/pulsar-functions/api-java/pom.xml +++ b/pulsar-functions/api-java/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-functions-api diff --git a/pulsar-functions/instance/pom.xml b/pulsar-functions/instance/pom.xml index 78d4a07286659..15b3eff0b6339 100644 --- a/pulsar-functions/instance/pom.xml +++ b/pulsar-functions/instance/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-functions-instance diff --git a/pulsar-functions/java-examples-builtin/pom.xml b/pulsar-functions/java-examples-builtin/pom.xml index 9a9ad67e2c4ae..fe5449372f199 100644 --- a/pulsar-functions/java-examples-builtin/pom.xml +++ b/pulsar-functions/java-examples-builtin/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-functions-api-examples-builtin diff --git a/pulsar-functions/java-examples/pom.xml b/pulsar-functions/java-examples/pom.xml index 8b2ff5736d7df..14e1e17d282ca 100644 --- a/pulsar-functions/java-examples/pom.xml +++ b/pulsar-functions/java-examples/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-functions-api-examples diff --git a/pulsar-functions/localrun-shaded/pom.xml b/pulsar-functions/localrun-shaded/pom.xml index f1c281f81e4a2..2626732457b00 100644 --- a/pulsar-functions/localrun-shaded/pom.xml +++ b/pulsar-functions/localrun-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/pulsar-functions/localrun/pom.xml b/pulsar-functions/localrun/pom.xml index 889df7ac05cff..6dcac5c4ac2be 100644 --- a/pulsar-functions/localrun/pom.xml +++ b/pulsar-functions/localrun/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/pulsar-functions/pom.xml b/pulsar-functions/pom.xml index 34de5a925c026..3176aba9c9188 100644 --- a/pulsar-functions/pom.xml +++ b/pulsar-functions/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-functions diff --git a/pulsar-functions/proto/pom.xml b/pulsar-functions/proto/pom.xml index f571b6e8c50e2..f188a147b2cee 100644 --- a/pulsar-functions/proto/pom.xml +++ b/pulsar-functions/proto/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-functions - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-functions-proto diff --git a/pulsar-functions/runtime-all/pom.xml b/pulsar-functions/runtime-all/pom.xml index ec6a364e0cf25..bf41c9b400be5 100644 --- a/pulsar-functions/runtime-all/pom.xml +++ b/pulsar-functions/runtime-all/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/pulsar-functions/runtime/pom.xml b/pulsar-functions/runtime/pom.xml index c1dd340e584f7..5558ced4c0749 100644 --- a/pulsar-functions/runtime/pom.xml +++ b/pulsar-functions/runtime/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-functions-runtime diff --git a/pulsar-functions/secrets/pom.xml b/pulsar-functions/secrets/pom.xml index cb302bbc3eca7..08d4e9bf63ab8 100644 --- a/pulsar-functions/secrets/pom.xml +++ b/pulsar-functions/secrets/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-functions-secrets diff --git a/pulsar-functions/utils/pom.xml b/pulsar-functions/utils/pom.xml index b18d29578fe26..a0dd90d5577af 100644 --- a/pulsar-functions/utils/pom.xml +++ b/pulsar-functions/utils/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-functions-utils diff --git a/pulsar-functions/worker/pom.xml b/pulsar-functions/worker/pom.xml index f4a30891d2e20..1201c5aea5d22 100644 --- a/pulsar-functions/worker/pom.xml +++ b/pulsar-functions/worker/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/pulsar-io/aerospike/pom.xml b/pulsar-io/aerospike/pom.xml index d70b6669ddbd1..9f058741cafbd 100644 --- a/pulsar-io/aerospike/pom.xml +++ b/pulsar-io/aerospike/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-aerospike diff --git a/pulsar-io/alluxio/pom.xml b/pulsar-io/alluxio/pom.xml index 6ab75c44d8d68..b0f6a51b79cf4 100644 --- a/pulsar-io/alluxio/pom.xml +++ b/pulsar-io/alluxio/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT diff --git a/pulsar-io/aws/pom.xml b/pulsar-io/aws/pom.xml index 8a7618da7b272..2c5ba9450d258 100644 --- a/pulsar-io/aws/pom.xml +++ b/pulsar-io/aws/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-aws diff --git a/pulsar-io/batch-data-generator/pom.xml b/pulsar-io/batch-data-generator/pom.xml index 7ff835b240809..621dd226f0eef 100644 --- a/pulsar-io/batch-data-generator/pom.xml +++ b/pulsar-io/batch-data-generator/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-batch-data-generator diff --git a/pulsar-io/batch-discovery-triggerers/pom.xml b/pulsar-io/batch-discovery-triggerers/pom.xml index 8820a546d7586..6b07bf4ecc8db 100644 --- a/pulsar-io/batch-discovery-triggerers/pom.xml +++ b/pulsar-io/batch-discovery-triggerers/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-batch-discovery-triggerers diff --git a/pulsar-io/canal/pom.xml b/pulsar-io/canal/pom.xml index 4feee749369e3..76eb484c8a56f 100644 --- a/pulsar-io/canal/pom.xml +++ b/pulsar-io/canal/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT 4.0.0 diff --git a/pulsar-io/cassandra/pom.xml b/pulsar-io/cassandra/pom.xml index 592b626101235..4d9296ea2b35c 100644 --- a/pulsar-io/cassandra/pom.xml +++ b/pulsar-io/cassandra/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-cassandra diff --git a/pulsar-io/common/pom.xml b/pulsar-io/common/pom.xml index 77dcba284b41f..b975d9fef07f6 100644 --- a/pulsar-io/common/pom.xml +++ b/pulsar-io/common/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-io - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-common diff --git a/pulsar-io/core/pom.xml b/pulsar-io/core/pom.xml index 22917ebaafd10..0d253bbbfc5ca 100644 --- a/pulsar-io/core/pom.xml +++ b/pulsar-io/core/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-core diff --git a/pulsar-io/data-generator/pom.xml b/pulsar-io/data-generator/pom.xml index 985e267d220bd..2ee0120c98836 100644 --- a/pulsar-io/data-generator/pom.xml +++ b/pulsar-io/data-generator/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-data-generator diff --git a/pulsar-io/debezium/core/pom.xml b/pulsar-io/debezium/core/pom.xml index 4bf6b31119011..4588019a9aa97 100644 --- a/pulsar-io/debezium/core/pom.xml +++ b/pulsar-io/debezium/core/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-debezium-core diff --git a/pulsar-io/debezium/mongodb/pom.xml b/pulsar-io/debezium/mongodb/pom.xml index 792550c0a8d57..b672d64e6147f 100644 --- a/pulsar-io/debezium/mongodb/pom.xml +++ b/pulsar-io/debezium/mongodb/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-debezium-mongodb diff --git a/pulsar-io/debezium/mssql/pom.xml b/pulsar-io/debezium/mssql/pom.xml index af3df377c1aa4..1151940d770f4 100644 --- a/pulsar-io/debezium/mssql/pom.xml +++ b/pulsar-io/debezium/mssql/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-debezium-mssql diff --git a/pulsar-io/debezium/mysql/pom.xml b/pulsar-io/debezium/mysql/pom.xml index 41520a835b164..47bb9506cb851 100644 --- a/pulsar-io/debezium/mysql/pom.xml +++ b/pulsar-io/debezium/mysql/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-debezium-mysql diff --git a/pulsar-io/debezium/oracle/pom.xml b/pulsar-io/debezium/oracle/pom.xml index 719b0a6afa2ad..ee158d6fb8414 100644 --- a/pulsar-io/debezium/oracle/pom.xml +++ b/pulsar-io/debezium/oracle/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-debezium-oracle diff --git a/pulsar-io/debezium/pom.xml b/pulsar-io/debezium/pom.xml index b18dba876d277..050da1af45330 100644 --- a/pulsar-io/debezium/pom.xml +++ b/pulsar-io/debezium/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-debezium diff --git a/pulsar-io/debezium/postgres/pom.xml b/pulsar-io/debezium/postgres/pom.xml index 68a902a1167c5..0bab38c71ef10 100644 --- a/pulsar-io/debezium/postgres/pom.xml +++ b/pulsar-io/debezium/postgres/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-debezium-postgres diff --git a/pulsar-io/docs/pom.xml b/pulsar-io/docs/pom.xml index ba64da6efc661..305c7f1473077 100644 --- a/pulsar-io/docs/pom.xml +++ b/pulsar-io/docs/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-docs diff --git a/pulsar-io/dynamodb/pom.xml b/pulsar-io/dynamodb/pom.xml index 1583108a6b88f..4233601180caa 100644 --- a/pulsar-io/dynamodb/pom.xml +++ b/pulsar-io/dynamodb/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-dynamodb diff --git a/pulsar-io/elastic-search/pom.xml b/pulsar-io/elastic-search/pom.xml index e55a12d2273fd..f9f87e5bd02b2 100644 --- a/pulsar-io/elastic-search/pom.xml +++ b/pulsar-io/elastic-search/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-elastic-search Pulsar IO :: ElasticSearch diff --git a/pulsar-io/file/pom.xml b/pulsar-io/file/pom.xml index 8ab9db2b6971d..b6e5a13679663 100644 --- a/pulsar-io/file/pom.xml +++ b/pulsar-io/file/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-file diff --git a/pulsar-io/flume/pom.xml b/pulsar-io/flume/pom.xml index 05f8053430479..a27540683a3d0 100644 --- a/pulsar-io/flume/pom.xml +++ b/pulsar-io/flume/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-flume diff --git a/pulsar-io/hbase/pom.xml b/pulsar-io/hbase/pom.xml index abd8383e226f2..33396394de4a4 100644 --- a/pulsar-io/hbase/pom.xml +++ b/pulsar-io/hbase/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-hbase Pulsar IO :: Hbase diff --git a/pulsar-io/hdfs2/pom.xml b/pulsar-io/hdfs2/pom.xml index 24a92e59fe8f4..1f43b31a0aa6a 100644 --- a/pulsar-io/hdfs2/pom.xml +++ b/pulsar-io/hdfs2/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-hdfs2 Pulsar IO :: Hdfs2 diff --git a/pulsar-io/hdfs3/pom.xml b/pulsar-io/hdfs3/pom.xml index 35bb2753ef19d..3fbc030184c5c 100644 --- a/pulsar-io/hdfs3/pom.xml +++ b/pulsar-io/hdfs3/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-hdfs3 Pulsar IO :: Hdfs3 diff --git a/pulsar-io/http/pom.xml b/pulsar-io/http/pom.xml index 0a64891963bf8..d86201a67b30f 100644 --- a/pulsar-io/http/pom.xml +++ b/pulsar-io/http/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-http diff --git a/pulsar-io/influxdb/pom.xml b/pulsar-io/influxdb/pom.xml index a2104112a4cf7..91571cf66e803 100644 --- a/pulsar-io/influxdb/pom.xml +++ b/pulsar-io/influxdb/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-influxdb diff --git a/pulsar-io/jdbc/clickhouse/pom.xml b/pulsar-io/jdbc/clickhouse/pom.xml index 8e8908a7f7aad..89ac29d72653a 100644 --- a/pulsar-io/jdbc/clickhouse/pom.xml +++ b/pulsar-io/jdbc/clickhouse/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT 4.0.0 diff --git a/pulsar-io/jdbc/core/pom.xml b/pulsar-io/jdbc/core/pom.xml index 53b2599afeb1d..f9dd1e0e96346 100644 --- a/pulsar-io/jdbc/core/pom.xml +++ b/pulsar-io/jdbc/core/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT 4.0.0 diff --git a/pulsar-io/jdbc/mariadb/pom.xml b/pulsar-io/jdbc/mariadb/pom.xml index 298addcf35f60..7e96a2901456f 100644 --- a/pulsar-io/jdbc/mariadb/pom.xml +++ b/pulsar-io/jdbc/mariadb/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT 4.0.0 diff --git a/pulsar-io/jdbc/openmldb/pom.xml b/pulsar-io/jdbc/openmldb/pom.xml index 3462971cbd39d..1b2bab67d04ab 100644 --- a/pulsar-io/jdbc/openmldb/pom.xml +++ b/pulsar-io/jdbc/openmldb/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT 4.0.0 diff --git a/pulsar-io/jdbc/pom.xml b/pulsar-io/jdbc/pom.xml index d6e168df2aecf..3f841ab183399 100644 --- a/pulsar-io/jdbc/pom.xml +++ b/pulsar-io/jdbc/pom.xml @@ -33,7 +33,7 @@ org.apache.pulsar pulsar-io - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-jdbc diff --git a/pulsar-io/jdbc/postgres/pom.xml b/pulsar-io/jdbc/postgres/pom.xml index 25216d9028302..3ae7eaf93bb7c 100644 --- a/pulsar-io/jdbc/postgres/pom.xml +++ b/pulsar-io/jdbc/postgres/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT 4.0.0 diff --git a/pulsar-io/jdbc/sqlite/pom.xml b/pulsar-io/jdbc/sqlite/pom.xml index 0cbc22543dda5..6c688af0ae15e 100644 --- a/pulsar-io/jdbc/sqlite/pom.xml +++ b/pulsar-io/jdbc/sqlite/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT 4.0.0 pulsar-io-jdbc-sqlite diff --git a/pulsar-io/kafka-connect-adaptor-nar/pom.xml b/pulsar-io/kafka-connect-adaptor-nar/pom.xml index 8c01d37a12b47..bd742f0f20332 100644 --- a/pulsar-io/kafka-connect-adaptor-nar/pom.xml +++ b/pulsar-io/kafka-connect-adaptor-nar/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-kafka-connect-adaptor-nar diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml index aee1b800b489c..5e192be95ecf4 100644 --- a/pulsar-io/kafka-connect-adaptor/pom.xml +++ b/pulsar-io/kafka-connect-adaptor/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-kafka-connect-adaptor diff --git a/pulsar-io/kafka/pom.xml b/pulsar-io/kafka/pom.xml index ec8de4c2e2e12..9e322961889ed 100644 --- a/pulsar-io/kafka/pom.xml +++ b/pulsar-io/kafka/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-kafka diff --git a/pulsar-io/kinesis/pom.xml b/pulsar-io/kinesis/pom.xml index bfa9fccdc8f72..b957a770dfcb0 100644 --- a/pulsar-io/kinesis/pom.xml +++ b/pulsar-io/kinesis/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-kinesis diff --git a/pulsar-io/mongo/pom.xml b/pulsar-io/mongo/pom.xml index 0a42972aa7f3b..63cdd397f2548 100644 --- a/pulsar-io/mongo/pom.xml +++ b/pulsar-io/mongo/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-io - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-mongo diff --git a/pulsar-io/netty/pom.xml b/pulsar-io/netty/pom.xml index 8dc26302282e7..b6959199053e3 100644 --- a/pulsar-io/netty/pom.xml +++ b/pulsar-io/netty/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-netty diff --git a/pulsar-io/nsq/pom.xml b/pulsar-io/nsq/pom.xml index 032b6ad6c5faa..8f7307843627f 100644 --- a/pulsar-io/nsq/pom.xml +++ b/pulsar-io/nsq/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-nsq diff --git a/pulsar-io/pom.xml b/pulsar-io/pom.xml index 6cab06ee00b2c..2e62509c08651 100644 --- a/pulsar-io/pom.xml +++ b/pulsar-io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io diff --git a/pulsar-io/rabbitmq/pom.xml b/pulsar-io/rabbitmq/pom.xml index 48ba68537e45c..8fc0535c16089 100644 --- a/pulsar-io/rabbitmq/pom.xml +++ b/pulsar-io/rabbitmq/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-rabbitmq diff --git a/pulsar-io/redis/pom.xml b/pulsar-io/redis/pom.xml index 165fe34c4d674..3f04f6645697f 100644 --- a/pulsar-io/redis/pom.xml +++ b/pulsar-io/redis/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-redis diff --git a/pulsar-io/solr/pom.xml b/pulsar-io/solr/pom.xml index 66d4e66348d69..4ef67a31ec33c 100644 --- a/pulsar-io/solr/pom.xml +++ b/pulsar-io/solr/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT diff --git a/pulsar-io/twitter/pom.xml b/pulsar-io/twitter/pom.xml index 15cca2cb9d2f2..901f8639a4326 100644 --- a/pulsar-io/twitter/pom.xml +++ b/pulsar-io/twitter/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-io-twitter diff --git a/pulsar-metadata/pom.xml b/pulsar-metadata/pom.xml index 71ae6f7c4ffdb..4096a1ee0f994 100644 --- a/pulsar-metadata/pom.xml +++ b/pulsar-metadata/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/pulsar-package-management/bookkeeper-storage/pom.xml b/pulsar-package-management/bookkeeper-storage/pom.xml index a728481f139eb..274e5abb4abec 100644 --- a/pulsar-package-management/bookkeeper-storage/pom.xml +++ b/pulsar-package-management/bookkeeper-storage/pom.xml @@ -25,7 +25,7 @@ pulsar-package-management org.apache.pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT 4.0.0 diff --git a/pulsar-package-management/core/pom.xml b/pulsar-package-management/core/pom.xml index d17079298e088..a54d2b1d8ad54 100644 --- a/pulsar-package-management/core/pom.xml +++ b/pulsar-package-management/core/pom.xml @@ -25,7 +25,7 @@ pulsar-package-management org.apache.pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT 4.0.0 diff --git a/pulsar-package-management/filesystem-storage/pom.xml b/pulsar-package-management/filesystem-storage/pom.xml index f88c36399948f..94e075fea1973 100644 --- a/pulsar-package-management/filesystem-storage/pom.xml +++ b/pulsar-package-management/filesystem-storage/pom.xml @@ -25,7 +25,7 @@ pulsar-package-management org.apache.pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT 4.0.0 diff --git a/pulsar-package-management/pom.xml b/pulsar-package-management/pom.xml index d55c3b28256e5..f05798ecb4bb3 100644 --- a/pulsar-package-management/pom.xml +++ b/pulsar-package-management/pom.xml @@ -25,7 +25,7 @@ pulsar org.apache.pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. 4.0.0 diff --git a/pulsar-proxy/pom.xml b/pulsar-proxy/pom.xml index 41443ee30ce13..03ec0aed8b56d 100644 --- a/pulsar-proxy/pom.xml +++ b/pulsar-proxy/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-proxy diff --git a/pulsar-sql/pom.xml b/pulsar-sql/pom.xml index e616c9dca005d..4dbf7d5737335 100644 --- a/pulsar-sql/pom.xml +++ b/pulsar-sql/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-sql diff --git a/pulsar-sql/presto-distribution/pom.xml b/pulsar-sql/presto-distribution/pom.xml index fffe47e34e855..6e39bd50ad8be 100644 --- a/pulsar-sql/presto-distribution/pom.xml +++ b/pulsar-sql/presto-distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-presto-distribution diff --git a/pulsar-sql/presto-pulsar-plugin/pom.xml b/pulsar-sql/presto-pulsar-plugin/pom.xml index 5f8f9fd939b9b..11d2cbdfa9925 100644 --- a/pulsar-sql/presto-pulsar-plugin/pom.xml +++ b/pulsar-sql/presto-pulsar-plugin/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-presto-connector diff --git a/pulsar-sql/presto-pulsar/pom.xml b/pulsar-sql/presto-pulsar/pom.xml index 407c9f5fcb7a7..622c66e6d76ba 100644 --- a/pulsar-sql/presto-pulsar/pom.xml +++ b/pulsar-sql/presto-pulsar/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-presto-connector-original diff --git a/pulsar-testclient/pom.xml b/pulsar-testclient/pom.xml index b61f5d047484a..ceb64ecbadd90 100644 --- a/pulsar-testclient/pom.xml +++ b/pulsar-testclient/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/pulsar-transaction/common/pom.xml b/pulsar-transaction/common/pom.xml index 4ba07f53a5e06..2b1cf90a2cda7 100644 --- a/pulsar-transaction/common/pom.xml +++ b/pulsar-transaction/common/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-transaction-parent - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-transaction-common diff --git a/pulsar-transaction/coordinator/pom.xml b/pulsar-transaction/coordinator/pom.xml index fac56f750cdc0..25390007b7056 100644 --- a/pulsar-transaction/coordinator/pom.xml +++ b/pulsar-transaction/coordinator/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-transaction-parent - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-transaction-coordinator diff --git a/pulsar-transaction/pom.xml b/pulsar-transaction/pom.xml index 01df05aca36eb..133f1fe826e2b 100644 --- a/pulsar-transaction/pom.xml +++ b/pulsar-transaction/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-transaction-parent diff --git a/pulsar-websocket/pom.xml b/pulsar-websocket/pom.xml index 3ac6403db56d6..3f8f91d1416ba 100644 --- a/pulsar-websocket/pom.xml +++ b/pulsar-websocket/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/structured-event-log/pom.xml b/structured-event-log/pom.xml index 3122688bd0239..6daaa21991515 100644 --- a/structured-event-log/pom.xml +++ b/structured-event-log/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/testmocks/pom.xml b/testmocks/pom.xml index add9e64454ff3..1cdcd7906079f 100644 --- a/testmocks/pom.xml +++ b/testmocks/pom.xml @@ -25,7 +25,7 @@ pulsar org.apache.pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT testmocks diff --git a/tests/bc_2_0_0/pom.xml b/tests/bc_2_0_0/pom.xml index 5d190690e0767..93adaffe83459 100644 --- a/tests/bc_2_0_0/pom.xml +++ b/tests/bc_2_0_0/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT bc_2_0_0 diff --git a/tests/bc_2_0_1/pom.xml b/tests/bc_2_0_1/pom.xml index 95b01fc2ac141..41da961f3bb8e 100644 --- a/tests/bc_2_0_1/pom.xml +++ b/tests/bc_2_0_1/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT bc_2_0_1 diff --git a/tests/bc_2_6_0/pom.xml b/tests/bc_2_6_0/pom.xml index e9411f7e4e180..f1e75c41ae6d3 100644 --- a/tests/bc_2_6_0/pom.xml +++ b/tests/bc_2_6_0/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT 4.0.0 diff --git a/tests/docker-images/java-test-functions/pom.xml b/tests/docker-images/java-test-functions/pom.xml index 2cef72eb3f251..db685d0394253 100644 --- a/tests/docker-images/java-test-functions/pom.xml +++ b/tests/docker-images/java-test-functions/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT 4.0.0 java-test-functions diff --git a/tests/docker-images/java-test-image/pom.xml b/tests/docker-images/java-test-image/pom.xml index 522ee3c713185..5154c4e5599e5 100644 --- a/tests/docker-images/java-test-image/pom.xml +++ b/tests/docker-images/java-test-image/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT 4.0.0 java-test-image diff --git a/tests/docker-images/java-test-plugins/pom.xml b/tests/docker-images/java-test-plugins/pom.xml index 40fa3b666bc9a..b214c84bb6c36 100644 --- a/tests/docker-images/java-test-plugins/pom.xml +++ b/tests/docker-images/java-test-plugins/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT 4.0.0 java-test-plugins diff --git a/tests/docker-images/latest-version-image/pom.xml b/tests/docker-images/latest-version-image/pom.xml index 0d96a3729bf01..8474f820c9566 100644 --- a/tests/docker-images/latest-version-image/pom.xml +++ b/tests/docker-images/latest-version-image/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT 4.0.0 latest-version-image diff --git a/tests/docker-images/pom.xml b/tests/docker-images/pom.xml index 9864fe0518677..ade69e0778b04 100644 --- a/tests/docker-images/pom.xml +++ b/tests/docker-images/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar.tests tests-parent - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT docker-images Apache Pulsar :: Tests :: Docker Images diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml index 49901437622af..d9b817ca2e2b5 100644 --- a/tests/integration/pom.xml +++ b/tests/integration/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT integration diff --git a/tests/pom.xml b/tests/pom.xml index 80590c12cd216..8a4aec9504ff3 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT org.apache.pulsar.tests tests-parent diff --git a/tests/pulsar-client-admin-shade-test/pom.xml b/tests/pulsar-client-admin-shade-test/pom.xml index 496537ea397a8..0e7bed47db060 100644 --- a/tests/pulsar-client-admin-shade-test/pom.xml +++ b/tests/pulsar-client-admin-shade-test/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar.tests tests-parent - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-client-admin-shade-test diff --git a/tests/pulsar-client-all-shade-test/pom.xml b/tests/pulsar-client-all-shade-test/pom.xml index 955137109100a..2b65f49a360ae 100644 --- a/tests/pulsar-client-all-shade-test/pom.xml +++ b/tests/pulsar-client-all-shade-test/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar.tests tests-parent - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-client-all-shade-test diff --git a/tests/pulsar-client-shade-test/pom.xml b/tests/pulsar-client-shade-test/pom.xml index 4848847547538..6efd88f4ac762 100644 --- a/tests/pulsar-client-shade-test/pom.xml +++ b/tests/pulsar-client-shade-test/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar.tests tests-parent - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT pulsar-client-shade-test diff --git a/tiered-storage/file-system/pom.xml b/tiered-storage/file-system/pom.xml index 117267305e4b5..1d8aab48cbff8 100644 --- a/tiered-storage/file-system/pom.xml +++ b/tiered-storage/file-system/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar tiered-storage-parent - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/tiered-storage/jcloud/pom.xml b/tiered-storage/jcloud/pom.xml index c1c1ca76f1051..56ec203b47690 100644 --- a/tiered-storage/jcloud/pom.xml +++ b/tiered-storage/jcloud/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar tiered-storage-parent - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. diff --git a/tiered-storage/pom.xml b/tiered-storage/pom.xml index cc55203a23444..6057e082fca01 100644 --- a/tiered-storage/pom.xml +++ b/tiered-storage/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 2.12.0-SNAPSHOT + 3.0.0-SNAPSHOT .. From 8ec65113b49af4459b4d8628b18ed8760e540a7e Mon Sep 17 00:00:00 2001 From: labuladong Date: Tue, 21 Feb 2023 15:05:36 +0800 Subject: [PATCH 038/174] [fix][test] flaky test: testOnlyCloseActiveConsumerForSingleActiveConsumerDispatcherWhenSeek (#19572) --- .../apache/pulsar/broker/service/SubscriptionSeekTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java index 2c2f62529d20a..93f2a42bcda35 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java @@ -598,7 +598,7 @@ public void testOnlyCloseActiveConsumerForSingleActiveConsumerDispatcherWhenSeek .subscriptionName("my-subscription") .subscribe(); - pulsarClient.newConsumer() + org.apache.pulsar.client.api.Consumer consumer2 = pulsarClient.newConsumer() .topic(topicName) .subscriptionType(SubscriptionType.Failover) .subscriptionName("my-subscription") @@ -615,8 +615,8 @@ public void testOnlyCloseActiveConsumerForSingleActiveConsumerDispatcherWhenSeek } assertEquals(connectedSinceSet.size(), 2); consumer1.seek(MessageId.earliest); - // Wait for consumer to reconnect - Awaitility.await().until(consumer1::isConnected); + // Wait for consumers to reconnect + Awaitility.await().until(() -> consumer1.isConnected() && consumer2.isConnected()); consumers = topicRef.getSubscriptions().get("my-subscription").getConsumers(); assertEquals(consumers.size(), 2); From 02b25a3b2461b975b46afdca4130cdba876721c0 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Tue, 21 Feb 2023 17:10:35 +0800 Subject: [PATCH 039/174] [fix] [test] fix flaky test PersistentFailoverE2ETest.testSimpleConsumerEventsWithPartition (#19574) --- ...sistentDispatcherFailoverConsumerTest.java | 80 +++++++++---- .../service/PersistentFailoverE2ETest.java | 106 ++++++++++++------ 2 files changed, 131 insertions(+), 55 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java index 29a3227c92bef..0dafe4bca3e2c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java @@ -37,6 +37,8 @@ import static org.testng.AssertJUnit.assertTrue; import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.EventLoopGroup; +import io.netty.util.concurrent.DefaultThreadFactory; import java.lang.reflect.Field; import java.net.InetSocketAddress; import java.util.ArrayList; @@ -47,6 +49,7 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; import org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCursorCallback; @@ -74,6 +77,8 @@ import org.apache.pulsar.common.api.proto.ProtocolVersion; import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.util.netty.EventLoopUtil; +import org.awaitility.Awaitility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; @@ -399,9 +404,54 @@ public void testAddRemoveConsumer() throws Exception { assertTrue(pdfc.canUnsubscribe(consumer1)); } - @Test + private String[] sortConsumerNameByHashSelector(String...consumerNames) throws Exception { + String[] result = new String[consumerNames.length]; + PersistentTopic topic = + new PersistentTopic(successTopicName, ledgerMock, pulsarTestContext.getBrokerService()); + PersistentSubscription sub = new PersistentSubscription(topic, "sub-1", cursorMock, false); + int partitionIndex = -1; + PersistentDispatcherSingleActiveConsumer dispatcher = new PersistentDispatcherSingleActiveConsumer(cursorMock, + SubType.Failover, partitionIndex, topic, sub); + for (String consumerName : consumerNames){ + Consumer consumer = spy(new Consumer(sub, SubType.Failover, topic.getName(), 999 /* consumer id */, 1, + consumerName/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), + false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH)); + dispatcher.addConsumer(consumer); + } + for (int i = 0; i < consumerNames.length; i++) { + result[i] = dispatcher.getActiveConsumer().consumerName(); + dispatcher.removeConsumer(dispatcher.getActiveConsumer()); + } + consumerChanges.clear(); + return result; + } + + private CommandActiveConsumerChange waitActiveChangeEvent(int consumerId) + throws Exception { + AtomicReference res = new AtomicReference<>(); + Awaitility.await().until(() -> { + while (!consumerChanges.isEmpty()){ + CommandActiveConsumerChange change = consumerChanges.take(); + if (change.getConsumerId() == consumerId){ + res.set(change); + return true; + } + } + return false; + }); + consumerChanges.clear(); + return res.get(); + } + + @Test(invocationCount = 100) public void testAddRemoveConsumerNonPartitionedTopic() throws Exception { log.info("--- Starting PersistentDispatcherFailoverConsumerTest::testAddConsumer ---"); + String[] sortedConsumerNameByHashSelector = sortConsumerNameByHashSelector("Cons1", "Cons2"); + BrokerService spyBrokerService = spy(pulsarTestContext.getBrokerService()); + final EventLoopGroup singleEventLoopGroup = EventLoopUtil.newEventLoopGroup(1, + pulsarTestContext.getBrokerService().getPulsar().getConfig().isEnableBusyWait(), + new DefaultThreadFactory("pulsar-io")); + doAnswer(invocation -> singleEventLoopGroup).when(spyBrokerService).executor(); PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, pulsarTestContext.getBrokerService()); @@ -417,23 +467,26 @@ public void testAddRemoveConsumerNonPartitionedTopic() throws Exception { // 2. Add a consumer Consumer consumer1 = spy(new Consumer(sub, SubType.Failover, topic.getName(), 1 /* consumer id */, 1, - "Cons1"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), + sortedConsumerNameByHashSelector[0]/* consumer name */, + true, serverCnx, "myrole-1", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH)); pdfc.addConsumer(consumer1); List consumers = pdfc.getConsumers(); assertEquals(1, consumers.size()); assertSame(pdfc.getActiveConsumer().consumerName(), consumer1.consumerName()); + waitActiveChangeEvent(1); // 3. Add a consumer with same priority level and consumer name is smaller in lexicographic order. Consumer consumer2 = spy(new Consumer(sub, SubType.Failover, topic.getName(), 2 /* consumer id */, 1, - "Cons2"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), + sortedConsumerNameByHashSelector[1]/* consumer name */, + true, serverCnx, "myrole-1", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH)); pdfc.addConsumer(consumer2); // 4. Verify active consumer doesn't change consumers = pdfc.getConsumers(); assertEquals(2, consumers.size()); - CommandActiveConsumerChange change = consumerChanges.take(); + CommandActiveConsumerChange change = waitActiveChangeEvent(2); verifyActiveConsumerChange(change, 2, false); assertSame(pdfc.getActiveConsumer().consumerName(), consumer1.consumerName()); verify(consumer2, times(1)).notifyActiveConsumerChange(same(consumer1)); @@ -444,21 +497,10 @@ public void testAddRemoveConsumerNonPartitionedTopic() throws Exception { pdfc.addConsumer(consumer3); consumers = pdfc.getConsumers(); assertEquals(3, consumers.size()); - change = consumerChanges.take(); - verifyActiveConsumerChange(change, 3, false); - assertSame(pdfc.getActiveConsumer().consumerName(), consumer1.consumerName()); - verify(consumer3, times(1)).notifyActiveConsumerChange(same(consumer1)); - - // 7. Remove first consumer and active consumer should change to consumer2 since it's added before consumer3 - // though consumer 3 has higher priority level - pdfc.removeConsumer(consumer1); - consumers = pdfc.getConsumers(); - assertEquals(2, consumers.size()); - change = consumerChanges.take(); - verifyActiveConsumerChange(change, 2, true); - assertSame(pdfc.getActiveConsumer().consumerName(), consumer2.consumerName()); - verify(consumer2, times(1)).notifyActiveConsumerChange(same(consumer2)); - verify(consumer3, times(1)).notifyActiveConsumerChange(same(consumer2)); + change = waitActiveChangeEvent(3); + verifyActiveConsumerChange(change, 3, true); + assertSame(pdfc.getActiveConsumer().consumerName(), consumer3.consumerName()); + verify(consumer3, times(1)).notifyActiveConsumerChange(same(consumer3)); } @Test diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentFailoverE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentFailoverE2ETest.java index ffc1444676b23..b263d4448d87a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentFailoverE2ETest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentFailoverE2ETest.java @@ -31,6 +31,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; +import lombok.AllArgsConstructor; import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherSingleActiveConsumer; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; @@ -107,14 +108,17 @@ private void verifyConsumerActive(TestConsumerStateEventListener listener, int p Integer pid = listener.activeQueue.take(); assertNotNull(pid); assertEquals(partitionId, pid.intValue()); - assertNull(listener.inActiveQueue.poll()); } private void verifyConsumerInactive(TestConsumerStateEventListener listener, int partitionId) throws Exception { Integer pid = listener.inActiveQueue.take(); assertNotNull(pid); assertEquals(partitionId, pid.intValue()); - assertNull(listener.activeQueue.poll()); + } + + private void clearEventQueue(TestConsumerStateEventListener listener) { + listener.inActiveQueue.clear(); + listener.activeQueue.clear(); } private static class ActiveInactiveListenerEvent implements ConsumerEventListener { @@ -135,27 +139,57 @@ public synchronized void becameInactive(Consumer consumer, int partitionId) { } } + @AllArgsConstructor + static class FailoverConsumer { + private String consumerName; + private Consumer consumer; + private TestConsumerStateEventListener listener; + private PersistentDispatcherSingleActiveConsumer dispatcher; + private boolean isActiveConsumer(){ + return dispatcher.getActiveConsumer().consumerName().equals(consumerName); + } + } + + FailoverConsumer createConsumer(String topicName, String subName, String listenerName, String consumerName) + throws Exception { + TestConsumerStateEventListener listener = new TestConsumerStateEventListener(listenerName); + Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName) + .acknowledgmentGroupTime(0, TimeUnit.SECONDS) + .subscriptionType(SubscriptionType.Failover) + .consumerName(consumerName) + .consumerEventListener(listener) + .subscribe(); + PersistentDispatcherSingleActiveConsumer dispatcher = + (PersistentDispatcherSingleActiveConsumer) pulsar.getBrokerService() + .getTopic(topicName, false).get().get() + .getSubscription(subName) + .getDispatcher(); + return new FailoverConsumer(consumerName, consumer, listener, dispatcher); + } + @Test public void testSimpleConsumerEventsWithoutPartition() throws Exception { final String topicName = "persistent://prop/use/ns-abc/failover-topic1-" + System.currentTimeMillis(); final String subName = "sub1"; final int numMsgs = 100; - TestConsumerStateEventListener listener1 = new TestConsumerStateEventListener("listener-1"); - TestConsumerStateEventListener listener2 = new TestConsumerStateEventListener("listener-2"); - ConsumerBuilder consumerBuilder = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName) - .acknowledgmentGroupTime(0, TimeUnit.SECONDS).subscriptionType(SubscriptionType.Failover); - + // 1. Registry two consumers. + FailoverConsumer failoverConsumer1 = createConsumer(topicName, subName, "l1", "c1"); + FailoverConsumer failoverConsumer2 = createConsumer(topicName, subName, "l2", "c2"); + FailoverConsumer firstConsumer; + FailoverConsumer secondConsumer; + if (failoverConsumer1.isActiveConsumer()){ + firstConsumer = failoverConsumer1; + secondConsumer = failoverConsumer2; + } else { + firstConsumer = failoverConsumer2; + secondConsumer = failoverConsumer1; + } - // 1. two consumers on the same subscription - ConsumerBuilder consumerBulder1 = consumerBuilder.clone().consumerName("1") - .consumerEventListener(listener1); - Consumer consumer1 = consumerBulder1.subscribe(); - Consumer consumer2 = consumerBuilder.clone().consumerName("2").consumerEventListener(listener2) - .subscribe(); - verifyConsumerActive(listener1, -1); - verifyConsumerInactive(listener2, -1); - listener2.inActiveQueue.clear(); + verifyConsumerActive(firstConsumer.listener, -1); + verifyConsumerInactive(secondConsumer.listener, -1); + clearEventQueue(firstConsumer.listener); + clearEventQueue(secondConsumer.listener); PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get(); PersistentSubscription subRef = topicRef.getSubscription(subName); @@ -185,14 +219,14 @@ public void testSimpleConsumerEventsWithoutPartition() throws Exception { assertEquals(subRef.getNumberOfEntriesInBacklog(false), numMsgs); }); - // 3. consumer1 should have all the messages while consumer2 should have no messages + // 3. firstConsumer should have all the messages while secondConsumer should have no messages Message msg = null; - Assert.assertNull(consumer2.receive(100, TimeUnit.MILLISECONDS)); + Assert.assertNull(secondConsumer.consumer.receive(100, TimeUnit.MILLISECONDS)); for (int i = 0; i < numMsgs; i++) { - msg = consumer1.receive(1, TimeUnit.SECONDS); + msg = firstConsumer.consumer.receive(1, TimeUnit.SECONDS); Assert.assertNotNull(msg); Assert.assertEquals(new String(msg.getData()), "my-message-" + i); - consumer1.acknowledge(msg); + firstConsumer.consumer.acknowledge(msg); } rolloverPerIntervalStats(); @@ -211,51 +245,52 @@ public void testSimpleConsumerEventsWithoutPartition() throws Exception { // 5. master consumer failure should resend unacked messages and new messages to another consumer for (int i = 0; i < 5; i++) { - msg = consumer1.receive(1, TimeUnit.SECONDS); + msg = firstConsumer.consumer.receive(1, TimeUnit.SECONDS); Assert.assertNotNull(msg); Assert.assertEquals(new String(msg.getData()), "my-message-" + i); - consumer1.acknowledge(msg); + firstConsumer.consumer.acknowledge(msg); } for (int i = 5; i < 10; i++) { - msg = consumer1.receive(1, TimeUnit.SECONDS); + msg = firstConsumer.consumer.receive(1, TimeUnit.SECONDS); Assert.assertNotNull(msg); Assert.assertEquals(new String(msg.getData()), "my-message-" + i); // do not ack } - consumer1.close(); + firstConsumer.consumer.close(); Awaitility.await().untilAsserted(() -> { - verifyConsumerActive(listener2, -1); - verifyConsumerNotReceiveAnyStateChanges(listener1); + verifyConsumerActive(secondConsumer.listener, -1); + verifyConsumerNotReceiveAnyStateChanges(firstConsumer.listener); + clearEventQueue(firstConsumer.listener); + clearEventQueue(secondConsumer.listener); }); for (int i = 5; i < numMsgs; i++) { - msg = consumer2.receive(1, TimeUnit.SECONDS); + msg = secondConsumer.consumer.receive(1, TimeUnit.SECONDS); Assert.assertNotNull(msg); Assert.assertEquals(new String(msg.getData()), "my-message-" + i); - consumer2.acknowledge(msg); + secondConsumer.consumer.acknowledge(msg); } - Assert.assertNull(consumer2.receive(100, TimeUnit.MILLISECONDS)); + Assert.assertNull(secondConsumer.consumer.receive(100, TimeUnit.MILLISECONDS)); rolloverPerIntervalStats(); Awaitility.await().untilAsserted(() -> { assertEquals(subRef.getNumberOfEntriesInBacklog(false), 0); - }); // 8. unsubscribe not allowed if multiple consumers connected try { - consumer1.unsubscribe(); + firstConsumer.consumer.unsubscribe(); fail("should fail"); } catch (PulsarClientException e) { // ok } - // 9. unsubscribe allowed if there is a lone consumer - consumer1.close(); + // 9. unsubscribe allowed if there is alone consumer + firstConsumer.consumer.close(); Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME); try { - consumer2.unsubscribe(); + secondConsumer.consumer.unsubscribe(); } catch (PulsarClientException e) { fail("Should not fail", e); } @@ -265,8 +300,7 @@ public void testSimpleConsumerEventsWithoutPartition() throws Exception { }); producer.close(); - consumer2.close(); - + secondConsumer.consumer.close(); admin.topics().delete(topicName); } From 3314d70231cbed89b6eefa2073ef8c048d84ec16 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Tue, 21 Feb 2023 21:47:28 +0800 Subject: [PATCH 040/174] [fix] [ml] topic load fail by ledger lost (#19444) Makes only ledgers removed from the meta of ledger info can be deleted from the BK. --- .../mledger/impl/ManagedLedgerImpl.java | 27 +++-- .../mledger/impl/ShadowManagedLedgerImpl.java | 5 +- .../mledger/impl/ManagedLedgerTest.java | 114 ++++++++++++++++++ 3 files changed, 135 insertions(+), 11 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index e334adf078a9e..9c05fb7c1047e 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -68,6 +68,7 @@ import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; +import javax.annotation.Nullable; import lombok.Getter; import org.apache.bookkeeper.client.AsyncCallback; import org.apache.bookkeeper.client.AsyncCallback.CreateCallback; @@ -471,6 +472,7 @@ protected synchronized void initializeBookKeeper(final ManagedLedgerInitializeLe } // Calculate total entries and size + final List emptyLedgersToBeDeleted = Collections.synchronizedList(new ArrayList<>()); Iterator iterator = ledgers.values().iterator(); while (iterator.hasNext()) { LedgerInfo li = iterator.next(); @@ -479,9 +481,7 @@ protected synchronized void initializeBookKeeper(final ManagedLedgerInitializeLe TOTAL_SIZE_UPDATER.addAndGet(this, li.getSize()); } else { iterator.remove(); - bookKeeper.asyncDeleteLedger(li.getLedgerId(), (rc, ctx) -> { - log.info("[{}] Deleted empty ledger ledgerId={} rc={}", name, li.getLedgerId(), rc); - }, null); + emptyLedgersToBeDeleted.add(li.getLedgerId()); } } @@ -497,6 +497,11 @@ protected synchronized void initializeBookKeeper(final ManagedLedgerInitializeLe @Override public void operationComplete(Void v, Stat stat) { ledgersStat = stat; + emptyLedgersToBeDeleted.forEach(ledgerId -> { + bookKeeper.asyncDeleteLedger(ledgerId, (rc, ctx) -> { + log.info("[{}] Deleted empty ledger ledgerId={} rc={}", name, ledgerId, rc); + }, null); + }); initializeCursors(callback); } @@ -1551,11 +1556,12 @@ public void operationComplete(Void v, Stat stat) { } ledgersStat = stat; synchronized (ManagedLedgerImpl.this) { + LedgerHandle originalCurrentLedger = currentLedger; ledgers.put(lh.getId(), newLedger); currentLedger = lh; currentLedgerEntries = 0; currentLedgerSize = 0; - updateLedgersIdsComplete(); + updateLedgersIdsComplete(originalCurrentLedger); mbean.addLedgerSwitchLatencySample(System.currentTimeMillis() - lastLedgerCreationInitiationTimestamp, TimeUnit.MILLISECONDS); } @@ -1646,8 +1652,15 @@ void createNewOpAddEntryForNewLedger() { } while (existsOp != null && --pendingSize > 0); } - protected synchronized void updateLedgersIdsComplete() { + protected synchronized void updateLedgersIdsComplete(@Nullable LedgerHandle originalCurrentLedger) { STATE_UPDATER.set(this, State.LedgerOpened); + // Delete original "currentLedger" if it has been removed from "ledgers". + if (originalCurrentLedger != null && !ledgers.containsKey(originalCurrentLedger.getId())){ + bookKeeper.asyncDeleteLedger(originalCurrentLedger.getId(), (rc, ctx) -> { + mbean.endDataLedgerDeleteOp(); + log.info("[{}] Delete complete for empty ledger {}. rc={}", name, originalCurrentLedger.getId(), rc); + }, null); + } updateLastLedgerCreatedTimeAndScheduleRolloverTask(); if (log.isDebugEnabled()) { @@ -1710,10 +1723,6 @@ synchronized void ledgerClosed(final LedgerHandle lh) { // The last ledger was empty, so we can discard it ledgers.remove(lh.getId()); mbean.startDataLedgerDeleteOp(); - bookKeeper.asyncDeleteLedger(lh.getId(), (rc, ctx) -> { - mbean.endDataLedgerDeleteOp(); - log.info("[{}] Delete complete for empty ledger {}. rc={}", name, lh.getId(), rc); - }, null); } trimConsumedLedgersInBackground(); diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImpl.java index 9a029778fe01c..b1f239413472f 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImpl.java @@ -30,6 +30,7 @@ import org.apache.bookkeeper.client.AsyncCallback; import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.common.util.OrderedScheduler; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; @@ -332,7 +333,7 @@ private synchronized void processSourceManagedLedgerInfo(MLDataFormats.ManagedLe currentLedgerEntries = 0; currentLedgerSize = 0; initLastConfirmedEntry(); - updateLedgersIdsComplete(); + updateLedgersIdsComplete(null); maybeUpdateCursorBeforeTrimmingConsumedLedger(); } else if (isNoSuchLedgerExistsException(rc)) { log.warn("[{}] Source ledger not found: {}", name, lastLedgerId); @@ -365,7 +366,7 @@ public synchronized void asyncClose(AsyncCallbacks.CloseCallback callback, Objec } @Override - protected synchronized void updateLedgersIdsComplete() { + protected synchronized void updateLedgersIdsComplete(LedgerHandle originalCurrentLedger) { STATE_UPDATER.set(this, State.LedgerOpened); updateLastLedgerCreatedTimeAndScheduleRolloverTask(); diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java index 3b011fe8d56f8..a4d8b75d00c96 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java @@ -48,6 +48,7 @@ import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.security.GeneralSecurityException; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -75,7 +76,9 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Predicate; import lombok.Cleanup; +import lombok.Data; import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.client.AsyncCallback; import org.apache.bookkeeper.client.AsyncCallback.AddCallback; import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.BookKeeper; @@ -144,6 +147,117 @@ public Object[][] checkOwnershipFlagProvider() { return new Object[][] { { Boolean.TRUE }, { Boolean.FALSE } }; } + private void makeAddEntryTimeout(ManagedLedgerImpl ml, AtomicBoolean addEntryFinished) throws Exception { + LedgerHandle currentLedger = ml.currentLedger; + final LedgerHandle spyLedgerHandle = spy(currentLedger); + doAnswer(invocation -> { + ByteBuf bs = (ByteBuf) invocation.getArguments()[0]; + AddCallback addCallback = (AddCallback) invocation.getArguments()[1]; + Object originalContext = invocation.getArguments()[2]; + currentLedger.asyncAddEntry(bs, (rc, lh, entryId, ctx) -> { + addEntryFinished.set(true); + addCallback.addComplete(BKException.Code.TimeoutException, spyLedgerHandle, -1, ctx); + }, originalContext); + return null; + }).when(spyLedgerHandle).asyncAddEntry(any(ByteBuf.class), any(AddCallback.class), any()); + ml.currentLedger = spyLedgerHandle; + } + + @Data + private static class DeleteLedgerInfo{ + volatile boolean hasCalled; + volatile CompletableFuture future = new CompletableFuture<>(); + } + + private DeleteLedgerInfo makeDelayIfDoLedgerDelete(LedgerHandle ledger, final AtomicBoolean signal, + BookKeeper spyBookKeeper) { + DeleteLedgerInfo deleteLedgerInfo = new DeleteLedgerInfo(); + doAnswer(invocation -> { + long ledgerId = (long) invocation.getArguments()[0]; + AsyncCallback.DeleteCallback originalCb = (AsyncCallback.DeleteCallback) invocation.getArguments()[1]; + AsyncCallback.DeleteCallback cb = (rc, ctx) -> { + if (deleteLedgerInfo.hasCalled) { + deleteLedgerInfo.future.complete(null); + } + originalCb.deleteComplete(rc, ctx); + }; + Object ctx = invocation.getArguments()[2]; + if (ledgerId != ledger.getId()){ + bkc.asyncDeleteLedger(ledgerId, originalCb, ctx); + } else { + deleteLedgerInfo.hasCalled = true; + new Thread(() -> { + Awaitility.await().atMost(Duration.ofSeconds(60)).until(signal::get); + bkc.asyncDeleteLedger(ledgerId, cb, ctx); + }).start(); + } + return null; + }).when(spyBookKeeper).asyncDeleteLedger(any(long.class), any(AsyncCallback.DeleteCallback.class), any()); + return deleteLedgerInfo; + } + + /*** + * This test simulates the following problems that can occur when ZK connections are unstable: + * - add entry timeout + * - write ZK fail when update ledger info of ML + * and verifies that ledger info of ML is still correct when the above problems occur. + */ + @Test + public void testLedgerInfoMetaCorrectIfAddEntryTimeOut() throws Exception { + String mlName = "testLedgerInfoMetaCorrectIfAddEntryTimeOut"; + BookKeeper spyBookKeeper = spy(bkc); + ManagedLedgerFactoryImpl factory = new ManagedLedgerFactoryImpl(metadataStore, spyBookKeeper); + ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open(mlName); + + // Make add entry timeout(The data write was actually successful). + AtomicBoolean addEntryFinished = new AtomicBoolean(false); + makeAddEntryTimeout(ml, addEntryFinished); + + // Make the update operation of ledger info failure when switch ledger. + metadataStore.failConditional(new MetadataStoreException.BadVersionException(""), (opType, path) -> { + if (opType == FaultInjectionMetadataStore.OperationType.PUT && addEntryFinished.get() + && "/managed-ledgers/testLedgerInfoMetaCorrectIfAddEntryTimeOut".equals(path)) { + return true; + } + return false; + }); + + // Make delete ledger is delayed if delete is called. + AtomicBoolean deleteLedgerDelaySignal = new AtomicBoolean(false); + DeleteLedgerInfo deleteLedgerInfo = + makeDelayIfDoLedgerDelete(ml.currentLedger, deleteLedgerDelaySignal, spyBookKeeper); + + // Add one entry. + // - it will fail and trigger ledger switch(we mocked the error). + // - ledger switch will also fail(we mocked the error). + try { + ml.addEntry("1".getBytes(Charset.defaultCharset())); + fail("Expected the operation of add entry will fail by timeout or ledger fenced."); + } catch (Exception e){ + // expected ex. + } + + // Reopen ML. + try { + ml.close(); + fail("Expected the operation of ml close will fail by fenced state."); + } catch (Exception e){ + // expected ex. + } + ManagedLedgerImpl mlReopened = (ManagedLedgerImpl) factory.open(mlName); + deleteLedgerDelaySignal.set(true); + if (deleteLedgerInfo.hasCalled){ + deleteLedgerInfo.future.join(); + } + mlReopened.close(); + + // verify: all ledgers in ledger info is worked. + for (long ledgerId : mlReopened.getLedgersInfo().keySet()){ + LedgerHandle lh = bkc.openLedger(ledgerId, ml.digestType, ml.getConfig().getPassword()); + lh.close(); + } + } + @Test public void managedLedgerApi() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger"); From 660525e57ed35b74cb9204521d1fba02cc08c542 Mon Sep 17 00:00:00 2001 From: jiangpengcheng Date: Wed, 22 Feb 2023 07:58:51 +0800 Subject: [PATCH 041/174] [improve][fn] Support schema in python instance (#18432) --- .../worker/PulsarFunctionLocalRunTest.java | 2 +- .../common/functions/FunctionConfig.java | 2 + .../apache/pulsar/admin/cli/CmdFunctions.java | 12 +++ .../src/main/python/python_instance.py | 87 +++++++++++++-- .../api/examples/pojo/AvroTestObject.java | 2 +- .../avro_schema_test_function.py | 40 +++++++ .../functions/utils/FunctionConfigUtils.java | 4 + .../functions/PulsarFunctionsTest.java | 101 ++++++++++++------ .../functions/PulsarFunctionsTestBase.java | 2 + .../java/PulsarFunctionsJavaTest.java | 2 +- .../python/PulsarFunctionsPythonTest.java | 5 + .../functions/utils/CommandGenerator.java | 20 ++++ 12 files changed, 234 insertions(+), 45 deletions(-) create mode 100644 pulsar-functions/python-examples/avro_schema_test_function.py diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java index 1d98ec1e2515c..c832cba163d63 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionLocalRunTest.java @@ -670,7 +670,7 @@ private void testAvroFunctionLocalRun(String jarFilePathUrl) throws Exception { }, 50, 150); int totalMsgs = 5; - Method setBaseValueMethod = avroTestObjectClass.getMethod("setBaseValue", new Class[]{int.class}); + Method setBaseValueMethod = avroTestObjectClass.getMethod("setBaseValue", new Class[]{Integer.class}); for (int i = 0; i < totalMsgs; i++) { Object avroTestObject = avroTestObjectClass.getDeclaredConstructor().newInstance(); setBaseValueMethod.invoke(avroTestObject, i); diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/FunctionConfig.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/FunctionConfig.java index 2dddb67c9503c..0b26e7e93b5f0 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/FunctionConfig.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/FunctionConfig.java @@ -72,6 +72,7 @@ public enum Runtime { * A generalized way of specifying inputs. */ private Map inputSpecs; + private String inputTypeClassName; private String output; @@ -83,6 +84,7 @@ public enum Runtime { * implementation. */ private String outputSchemaType; + private String outputTypeClassName; private String outputSerdeClassName; private String logTopic; diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java index a9c2f91d2b2c2..bc2585bc67bc9 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java @@ -269,6 +269,9 @@ abstract class FunctionDetailsCommand extends BaseCommand { @Parameter(names = "--input-specs", description = "The map of inputs to custom configuration (as a JSON string) #Java, Python, Go") protected String inputSpecs; + @Parameter(names = "--input-type-class-name", + description = "The class name of input type class #Java, Python, Go") + protected String inputTypeClassName; // for backwards compatibility purposes @Parameter(names = "--outputSerdeClassName", description = "The SerDe class to be used for messages output by the function", hidden = true) @@ -276,6 +279,9 @@ abstract class FunctionDetailsCommand extends BaseCommand { @Parameter(names = "--output-serde-classname", description = "The SerDe class to be used for messages output by the function #Java, Python") protected String outputSerdeClassName; + @Parameter(names = "--output-type-class-name", + description = "The class name of output type class #Java, Python, Go") + protected String outputTypeClassName; // for backwards compatibility purposes @Parameter(names = "--functionConfigFile", description = "The path to a YAML config file that specifies " + "the configuration of a Pulsar Function", hidden = true) @@ -485,12 +491,18 @@ void processArguments() throws Exception { Type type = new TypeToken>() {}.getType(); functionConfig.setInputSpecs(new Gson().fromJson(inputSpecs, type)); } + if (null != inputTypeClassName) { + functionConfig.setInputTypeClassName(inputTypeClassName); + } if (null != topicsPattern) { functionConfig.setTopicsPattern(topicsPattern); } if (null != output) { functionConfig.setOutput(output); } + if (null != outputTypeClassName) { + functionConfig.setOutputTypeClassName(outputTypeClassName); + } if (null != producerConfig) { Type type = new TypeToken() {}.getType(); functionConfig.setProducerConfig(new Gson().fromJson(producerConfig, type)); diff --git a/pulsar-functions/instance/src/main/python/python_instance.py b/pulsar-functions/instance/src/main/python/python_instance.py index cf53e75d9d0cf..f77ef38f76e4d 100755 --- a/pulsar-functions/instance/src/main/python/python_instance.py +++ b/pulsar-functions/instance/src/main/python/python_instance.py @@ -32,6 +32,7 @@ import threading import sys import re +import inspect import pulsar import contextimpl import Function_pb2 @@ -50,9 +51,10 @@ # Equivalent of the InstanceConfig in Java InstanceConfig = namedtuple('InstanceConfig', 'instance_id function_id function_version function_details max_buffered_tuples') # This is the message that the consumers put on the queue for the function thread to process -InternalMessage = namedtuple('InternalMessage', 'message topic serde consumer') +InternalMessage = namedtuple('InternalMessage', 'message topic serde use_schema consumer') InternalQuitMessage = namedtuple('InternalQuitMessage', 'quit') DEFAULT_SERIALIZER = "serde.IdentitySerDe" +DEFAULT_SCHEMA = pulsar.schema.BytesSchema() PY3 = sys.version_info[0] >= 3 @@ -93,8 +95,10 @@ def __init__(self, self.pulsar_client = pulsar_client self.state_storage_serviceurl = state_storage_serviceurl self.input_serdes = {} + self.input_schema = {} self.consumers = {} self.output_serde = None + self.output_schema = DEFAULT_SCHEMA self.function_class = None self.function_purefunction = None self.producer = None @@ -166,7 +170,7 @@ def run(self): self.consumers[topic] = self.pulsar_client.subscribe( str(topic), subscription_name, consumer_type=mode, - message_listener=partial(self.message_listener, self.input_serdes[topic]), + message_listener=partial(self.message_listener, self.input_serdes[topic], DEFAULT_SCHEMA), unacked_messages_timeout_ms=int(self.timeout_ms) if self.timeout_ms else None, initial_position=position, properties=properties @@ -178,11 +182,16 @@ def run(self): else: serde_kclass = util.import_class(os.path.dirname(self.user_code), consumer_conf.serdeClassName) self.input_serdes[topic] = serde_kclass() + + self.input_schema[topic] = self.get_schema(consumer_conf.schemaType, + self.instance_config.function_details.source.typeClassName, + consumer_conf.schemaProperties) Log.debug("Setting up consumer for topic %s with subname %s" % (topic, subscription_name)) consumer_args = { "consumer_type": mode, - "message_listener": partial(self.message_listener, self.input_serdes[topic]), + "schema": self.input_schema[topic], + "message_listener": partial(self.message_listener, self.input_serdes[topic], self.input_schema[topic]), "unacked_messages_timeout_ms": int(self.timeout_ms) if self.timeout_ms else None, "initial_position": position, "properties": properties @@ -233,8 +242,10 @@ def actual_execution(self): if isinstance(msg, InternalQuitMessage): break Log.debug("Got a message from topic %s" % msg.topic) - # deserialize message - input_object = msg.serde.deserialize(msg.message.data()) + input_object = msg.message.value() + if not msg.use_schema: + # deserialize message + input_object = msg.serde.deserialize(msg.message.data()) # set current message in context self.contextimpl.set_current_message_context(msg.message, msg.topic) output_object = None @@ -296,12 +307,14 @@ def process_result(self, output, msg): if self.producer is None: self.setup_producer() - # serialize function output - output_bytes = self.output_serde.serialize(output) + # only serialize function output when output schema is not set + output_object = output + if self.output_schema == DEFAULT_SCHEMA: + output_object = self.output_serde.serialize(output) - if output_bytes is not None: + if output_object is not None: props = {"__pfn_input_topic__" : str(msg.topic), "__pfn_input_msg_id__" : base64ify(msg.message.message_id().serialize())} - self.producer.send_async(output_bytes, partial(self.done_producing, msg.consumer, msg.message, self.producer.topic()), properties=props) + self.producer.send_async(output_object, partial(self.done_producing, msg.consumer, msg.message, self.producer.topic()), properties=props) elif self.auto_ack and self.atleast_once: msg.consumer.acknowledge(msg.message) @@ -327,8 +340,13 @@ def setup_producer(self): if batch_builder == "KEY_BASED": batch_type = pulsar.BatchingType.KeyBased + self.output_schema = self.get_schema(self.instance_config.function_details.sink.schemaType, + self.instance_config.function_details.sink.typeClassName, + self.instance_config.function_details.sink.schemaProperties) + self.producer = self.pulsar_client.create_producer( str(self.instance_config.function_details.sink.topic), + schema=self.output_schema, block_if_queue_full=True, batching_enabled=True, batching_type=batch_type, @@ -351,10 +369,13 @@ def setup_state(self): table_name = str(self.instance_config.function_details.name) return state_context.create_state_context(self.state_storage_serviceurl, table_ns, table_name) - def message_listener(self, serde, consumer, message): + def message_listener(self, serde, schema, consumer, message): # increment number of received records from source self.stats.incr_total_received() - item = InternalMessage(message, message.topic_name(), serde, consumer) + use_schema = False + if schema != DEFAULT_SCHEMA: + use_schema = True + item = InternalMessage(message, message.topic_name(), serde, use_schema, consumer) self.queue.put(item, True) if self.atmost_once and self.auto_ack: consumer.acknowledge(message) @@ -462,3 +483,47 @@ def close(self): if self.pulsar_client: self.pulsar_client.close() + + + # TODO: support other schemas: PROTOBUF, PROTOBUF_NATIVE, and KeyValue + def get_schema(self, schema_type, type_class_name, schema_properties): + schema = DEFAULT_SCHEMA + if schema_type == "" or schema_type is None: + schema = DEFAULT_SCHEMA + elif schema_type.lower() == "string": + schema = pulsar.schema.StringSchema() + elif schema_type.lower() == "json": + record_kclass = self.get_record_class(type_class_name) + schema = pulsar.schema.JsonSchema(record_kclass) + elif schema_type.lower() == "avro": + record_kclass = self.get_record_class(type_class_name) + schema = pulsar.schema.AvroSchema(record_kclass) + else: # load custom schema + record_kclass = self.get_record_class(type_class_name) + schema_kclass = util.import_class(os.path.dirname(self.user_code), schema_type) + args_count = 0 + try: + args_count = len(inspect.signature(schema_kclass.__init__).parameters) + except: # for compatibility with python 2 + args_count = len(inspect.getargspec(schema_kclass.__init__).args) + if args_count == 1: # doesn't take any arguments + schema = schema_kclass() + elif args_count == 2: # take one argument, it can be either schema properties or record class + try: + schema = schema_kclass(record_kclass) + except TypeError: + schema = schema_kclass(schema_properties) + elif args_count >= 3: # take two or more arguments + schema = schema_kclass(record_kclass, schema_properties) + else: + raise Exception("Invalid schema class %s" % schema_type) + return schema + + def get_record_class(self, class_name): + record_kclass = None + if class_name != None and len(class_name) > 0: + try: + record_kclass = util.import_class(os.path.dirname(self.user_code), class_name) + except: + pass + return record_kclass \ No newline at end of file diff --git a/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/pojo/AvroTestObject.java b/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/pojo/AvroTestObject.java index 4967dbac9dae4..e7abb8e7d6ed0 100644 --- a/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/pojo/AvroTestObject.java +++ b/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/pojo/AvroTestObject.java @@ -27,7 +27,7 @@ @Data public class AvroTestObject { - private int baseValue; + private Integer baseValue; private String objectValue; } diff --git a/pulsar-functions/python-examples/avro_schema_test_function.py b/pulsar-functions/python-examples/avro_schema_test_function.py new file mode 100644 index 0000000000000..46704439a93ca --- /dev/null +++ b/pulsar-functions/python-examples/avro_schema_test_function.py @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +from pulsar import Function +from pulsar.schema import * + + +class AvroTestObject(Record): + def __init__(self, baseValue, objectValue): + self.baseValue = baseValue + self.objectValue = objectValue + + baseValue = Integer() + objectValue = String() + + +class AvroSchemaTestFunction(Function): + def __init__(self): + pass + + def process(self, input, context): + input.__setattr__("baseValue", input.baseValue + 10) + return input diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java index f6a5da1177fc4..647210de33acf 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java @@ -200,6 +200,8 @@ public static FunctionDetails convert(FunctionConfig functionConfig, ExtractedFu if (extractedDetails.getTypeArg0() != null) { sourceSpecBuilder.setTypeClassName(extractedDetails.getTypeArg0()); + } else if (StringUtils.isNotEmpty(functionConfig.getInputTypeClassName())) { + sourceSpecBuilder.setTypeClassName(functionConfig.getInputTypeClassName()); } if (functionConfig.getTimeoutMs() != null) { sourceSpecBuilder.setTimeoutMs(functionConfig.getTimeoutMs()); @@ -242,6 +244,8 @@ public static FunctionDetails convert(FunctionConfig functionConfig, ExtractedFu } if (extractedDetails.getTypeArg1() != null) { sinkSpecBuilder.setTypeClassName(extractedDetails.getTypeArg1()); + } else if (StringUtils.isNotEmpty(functionConfig.getOutputTypeClassName())) { + sinkSpecBuilder.setTypeClassName(functionConfig.getOutputTypeClassName()); } if (functionConfig.getProducerConfig() != null) { ProducerConfig producerConf = functionConfig.getProducerConfig(); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index 7a88d35a65ac3..eaf66974b982a 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -30,6 +30,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import java.time.Duration; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.Map; @@ -60,6 +61,7 @@ import org.apache.pulsar.client.api.schema.GenericRecord; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.schema.generic.GenericJsonRecord; +import org.apache.pulsar.common.functions.ConsumerConfig; import org.apache.pulsar.common.policies.data.FunctionStatsImpl; import org.apache.pulsar.common.policies.data.FunctionStatus; import org.apache.pulsar.common.policies.data.FunctionStatusUtil; @@ -174,8 +176,8 @@ protected void testFunctionLocalRun(Runtime runtime) throws Exception { assertEquals(admin.topics().getStats(inputTopicName).getSubscriptions().size(), 1); // publish and consume result - if (Runtime.JAVA == runtime) { - // java supports schema + if (Runtime.JAVA == runtime || Runtime.PYTHON == runtime) { + // java and python supports schema @Cleanup PulsarClient client = PulsarClient.builder() .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) .build(); @@ -208,7 +210,7 @@ protected void testFunctionLocalRun(Runtime runtime) throws Exception { assertEquals(expectedMessages.size(), 0); } else { - // python doesn't support schema + // golang doesn't support schema @Cleanup PulsarClient client = PulsarClient.builder() .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) @@ -355,7 +357,7 @@ protected void testFunctionNegAck(Runtime runtime) throws Exception { Schema schema; - if (Runtime.JAVA == runtime) { + if (Runtime.JAVA == runtime || Runtime.PYTHON == runtime) { schema = Schema.STRING; } else { schema = Schema.BYTES; @@ -388,8 +390,8 @@ protected void testFunctionNegAck(Runtime runtime) throws Exception { getFunctionStatsEmpty(functionName); // publish and consume result - if (Runtime.JAVA == runtime) { - // java supports schema + if (Runtime.JAVA == runtime || Runtime.PYTHON == runtime) { + // java and python supports schema @Cleanup PulsarClient client = PulsarClient.builder() .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) .build(); @@ -420,7 +422,7 @@ protected void testFunctionNegAck(Runtime runtime) throws Exception { assertEquals(expectedMessages.size(), 0); } else { - // python doesn't support schema + // golang doesn't support schema @Cleanup PulsarClient client = PulsarClient.builder() .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) @@ -530,7 +532,7 @@ protected void testPublishFunction(Runtime runtime) throws Exception { } Schema schema; - if (Runtime.JAVA == runtime) { + if (Runtime.JAVA == runtime || Runtime.PYTHON == runtime) { schema = Schema.STRING; } else { schema = Schema.BYTES; @@ -559,9 +561,14 @@ protected void testPublishFunction(Runtime runtime) throws Exception { PUBLISH_JAVA_CLASS, schema, Collections.singletonMap("publish-topic", outputTopicName), - null, null, null); + null, null, null, null, null); break; case PYTHON: + ConsumerConfig consumerConfig = new ConsumerConfig(); + consumerConfig.setSchemaType("string"); + Map inputSpecs = new HashMap<>() {{ + put(inputTopicName, objectMapper.writeValueAsString(consumerConfig)); + }}; submitFunction( runtime, inputTopicName, @@ -571,7 +578,7 @@ protected void testPublishFunction(Runtime runtime) throws Exception { PUBLISH_PYTHON_CLASS, schema, Collections.singletonMap("publish-topic", outputTopicName), - null, null, null); + objectMapper.writeValueAsString(inputSpecs), "string", null, null, null); break; case GO: submitFunction( @@ -583,7 +590,7 @@ protected void testPublishFunction(Runtime runtime) throws Exception { null, schema, Collections.singletonMap("publish-topic", outputTopicName), - null, null, null); + null, null, null, null, null); } // get function info @@ -594,11 +601,11 @@ protected void testPublishFunction(Runtime runtime) throws Exception { // publish and consume result - if (Runtime.JAVA == runtime) { - // java supports schema + if (Runtime.JAVA == runtime || Runtime.PYTHON == runtime) { + // java and python supports schema publishAndConsumeMessages(inputTopicName, outputTopicName, numMessages); } else { - // python doesn't support schema. Does Go? Maybe we need a switch instead for the Go case. + // Does Go support schema? Maybe we need a switch instead for the Go case. @Cleanup PulsarClient client = PulsarClient.builder() .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) @@ -663,7 +670,7 @@ protected void testExclamationFunction(Runtime runtime, Schema schema; - if (Runtime.JAVA == runtime) { + if (Runtime.JAVA == runtime || Runtime.PYTHON == runtime) { schema = Schema.STRING; } else { schema = Schema.BYTES; @@ -707,11 +714,11 @@ protected void testExclamationFunction(Runtime runtime, getFunctionStatsEmpty(functionName); // publish and consume result - if (Runtime.JAVA == runtime) { + if (Runtime.JAVA == runtime || Runtime.PYTHON == runtime) { // java supports schema publishAndConsumeMessages(inputTopicName, outputTopicName, numMessages); } else { - // python doesn't support schema + // golang doesn't support schema publishAndConsumeMessagesBytes(inputTopicName, outputTopicName, numMessages); } @@ -793,7 +800,7 @@ private void submitFunction(Runtime runtime, String functionClass, Schema inputTopicSchema) throws Exception { submitFunction(runtime, inputTopicName, outputTopicName, functionName, functionFile, functionClass, - inputTopicSchema, null, null, null, null); + inputTopicSchema, null, null, null, null, null, null); } private void submitFunction(Runtime runtime, @@ -806,7 +813,9 @@ private void submitFunction(Runtime runtime, Map userConfigs, String customSchemaInputs, String outputSchemaType, - SubscriptionInitialPosition subscriptionInitialPosition) throws Exception { + SubscriptionInitialPosition subscriptionInitialPosition, + String inputTypeClassName, + String outputTypeClassName) throws Exception { if (StringUtils.isNotEmpty(inputTopicName)) { ensureSubscriptionCreated( @@ -836,6 +845,12 @@ private void submitFunction(Runtime runtime, if (subscriptionInitialPosition != null) { generator.setSubscriptionInitialPosition(subscriptionInitialPosition); } + if (inputTypeClassName != null) { + generator.setInputTypeClassName(inputTypeClassName); + } + if (outputTypeClassName != null) { + generator.setOutputTypeClassName(outputTypeClassName); + } String command = ""; switch (runtime){ @@ -857,6 +872,8 @@ private void submitFunction(Runtime runtime, }; ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd( commands); + log.info("---------- stdout is: {}", result.getStdout()); + log.info("---------- stderr is: {}", result.getStderr()); assertTrue(result.getStdout().contains("Created successfully")); } @@ -1326,7 +1343,12 @@ private void publishAndConsumeAvroMessages(String inputTopic, } } - protected void testAvroSchemaFunction() throws Exception { + protected void testAvroSchemaFunction(Runtime runtime) throws Exception { + if (functionRuntimeType == FunctionRuntimeType.THREAD && runtime == Runtime.PYTHON) { + // python can only run on process mode + return; + } + log.info("testAvroSchemaFunction start ..."); final String inputTopic = "test-avroschema-input-" + randomName(8); final String outputTopic = "test-avroschema-output-" + randomName(8); @@ -1375,14 +1397,31 @@ protected void testAvroSchemaFunction() throws Exception { } }); - submitFunction( - Runtime.JAVA, - inputTopic, - outputTopic, - functionName, - null, - AvroSchemaTestFunction.class.getName(), - Schema.AVRO(AvroTestObject.class)); + if (runtime == Runtime.JAVA) { + submitFunction( + Runtime.JAVA, + inputTopic, + outputTopic, + functionName, + null, + AvroSchemaTestFunction.class.getName(), + Schema.AVRO(AvroTestObject.class)); + } else if (runtime == Runtime.PYTHON) { + ConsumerConfig consumerConfig = new ConsumerConfig(); + consumerConfig.setSchemaType("avro"); + Map inputSpecs = new HashMap<>() {{ + put(inputTopic, objectMapper.writeValueAsString(consumerConfig)); + }}; + submitFunction( + Runtime.PYTHON, + inputTopic, + outputTopic, + functionName, + AVRO_SCHEMA_FUNCTION_PYTHON_FILE, + AVRO_SCHEMA_PYTHON_CLASS, + Schema.AVRO(AvroTestObject.class), + null, objectMapper.writeValueAsString(inputSpecs), "avro", null, "avro_schema_test_function.AvroTestObject", "avro_schema_test_function.AvroTestObject"); + } log.info("pulsar submitFunction"); getFunctionInfoSuccess(functionName); @@ -1457,7 +1496,7 @@ protected void testInitFunction(Runtime runtime) throws Exception { // submit the exclamation function submitFunction(runtime, inputTopicName, outputTopicName, functionName, null, InitializableFunction.class.getName(), schema, - Collections.singletonMap("publish-topic", outputTopicName), null, null, null); + Collections.singletonMap("publish-topic", outputTopicName), null, null, null, null, null); // publish and consume result publishAndConsumeMessages(inputTopicName, outputTopicName, numMessages); @@ -1645,7 +1684,7 @@ protected void testGenericObjectFunction(String function, boolean removeAgeField null, null, SchemaType.NONE.name(), - SubscriptionInitialPosition.Earliest); + SubscriptionInitialPosition.Earliest, null, null); try { if (keyValue) { @Cleanup @@ -1830,7 +1869,7 @@ protected void testMergeFunction() throws Exception { null, inputSpecNode.toString(), SchemaType.AUTO_PUBLISH.name().toUpperCase(), - SubscriptionInitialPosition.Earliest); + SubscriptionInitialPosition.Earliest, null, null); getFunctionInfoSuccess(functionName); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java index 71f7a861cd8e2..033e590d6dd4e 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java @@ -68,11 +68,13 @@ public abstract class PulsarFunctionsTestBase extends PulsarTestSuite { public static final String PUBLISH_PYTHON_CLASS = "typed_message_builder_publish.TypedMessageBuilderPublish"; public static final String EXCEPTION_PYTHON_CLASS = "exception_function"; + public static final String AVRO_SCHEMA_PYTHON_CLASS = "avro_schema_test_function.AvroSchemaTestFunction"; public static final String EXCLAMATION_PYTHON_FILE = "exclamation_function.py"; public static final String EXCLAMATION_WITH_DEPS_PYTHON_FILE = "exclamation_with_extra_deps.py"; public static final String EXCLAMATION_PYTHON_ZIP_FILE = "exclamation.zip"; public static final String PUBLISH_FUNCTION_PYTHON_FILE = "typed_message_builder_publish.py"; public static final String EXCEPTION_FUNCTION_PYTHON_FILE = "exception_function.py"; + public static final String AVRO_SCHEMA_FUNCTION_PYTHON_FILE = "avro_schema_test_function.py"; public static final String EXCLAMATION_GO_FILE = "exclamationFunc"; public static final String PUBLISH_FUNCTION_GO_FILE = "exclamationFunc"; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java index 6828630940e2e..e6f3c67ebcdb7 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java @@ -210,7 +210,7 @@ public void testAutoSchemaFunctionTest() throws Exception { @Test(groups = {"java_function", "function"}) public void testAvroSchemaFunctionTest() throws Exception { - testAvroSchemaFunction(); + testAvroSchemaFunction(Runtime.JAVA); } } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonTest.java index 909da3d28583e..1c75d70498609 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonTest.java @@ -64,4 +64,9 @@ public void testPythonExclamationTopicPatternFunction() throws Exception { testExclamationFunction(Runtime.PYTHON, true, false, false); } + @Test(groups = {"python_function", "function"}) + public void testAvroSchemaFunctionTest() throws Exception { + testAvroSchemaFunction(Runtime.PYTHON); + } + } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/utils/CommandGenerator.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/utils/CommandGenerator.java index ab411e24dbe5f..90fac7a055268 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/utils/CommandGenerator.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/utils/CommandGenerator.java @@ -60,6 +60,8 @@ public enum Runtime { private Integer slidingIntervalCount; private Long slidingIntervalDurationMs; private String customSchemaInputs; + private String inputTypeClassName; + private String outputTypeClassName; private String schemaType; private SubscriptionInitialPosition subscriptionInitialPosition; @@ -111,6 +113,12 @@ public String generateLocalRunCommand(String codeFile) { if (customSchemaInputs != null) { commandBuilder.append(" --custom-schema-inputs \'" + customSchemaInputs + "\'"); } + if (inputTypeClassName != null) { + commandBuilder.append(" --input-type-class-name " + inputTypeClassName); + } + if (outputTypeClassName != null) { + commandBuilder.append(" --output-type-class-name " + outputTypeClassName); + } if (schemaType != null) { commandBuilder.append(" --schema-type " + schemaType); } @@ -207,6 +215,12 @@ public String generateCreateFunctionCommand(String codeFile) { if (customSchemaInputs != null) { commandBuilder.append(" --custom-schema-inputs \'" + customSchemaInputs + "\'"); } + if (inputTypeClassName != null) { + commandBuilder.append(" --input-type-class-name " + inputTypeClassName); + } + if (outputTypeClassName != null) { + commandBuilder.append(" --output-type-class-name " + outputTypeClassName); + } if (schemaType != null) { commandBuilder.append(" --schema-type " + schemaType); } @@ -305,6 +319,12 @@ public String generateUpdateFunctionCommand(String codeFile) { if (customSchemaInputs != null) { commandBuilder.append(" --custom-schema-inputs \'" + customSchemaInputs + "\'"); } + if (inputTypeClassName != null) { + commandBuilder.append(" --input-type-class-name " + inputTypeClassName); + } + if (outputTypeClassName != null) { + commandBuilder.append(" --output-type-class-name " + outputTypeClassName); + } if (schemaType != null) { commandBuilder.append(" --schema-type " + schemaType); } From b38556aa19a1b29accc3b2d64170d169e80ce135 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Wed, 22 Feb 2023 10:18:13 +0800 Subject: [PATCH 042/174] [fix][broker] Fix geo-replication admin (#19548) Signed-off-by: Zixuan Liu --- .../pulsar/broker/service/BrokerService.java | 92 +++++++++++++----- .../service/ReplicatorAdminTlsTest.java | 69 +++++++++++++ .../ReplicatorAdminTlsWithKeyStoreTest.java | 74 ++++++++++++++ .../broker/service/ReplicatorTestBase.java | 96 +++++++++++++++---- .../broker/service/ReplicatorTlsTest.java | 16 +++- 5 files changed, 303 insertions(+), 44 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorAdminTlsTest.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorAdminTlsWithKeyStoreTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index db7a3f16f97f2..b9b63427b370e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -1370,6 +1370,29 @@ private void configTlsSettings(ClientBuilder clientBuilder, String serviceUrl, } } + private void configAdminTlsSettings(PulsarAdminBuilder adminBuilder, boolean brokerClientTlsEnabledWithKeyStore, + boolean isTlsAllowInsecureConnection, + String brokerClientTlsTrustStoreType, String brokerClientTlsTrustStore, + String brokerClientTlsTrustStorePassword, String brokerClientTlsKeyStoreType, + String brokerClientTlsKeyStore, String brokerClientTlsKeyStorePassword, + String brokerClientTrustCertsFilePath, + String brokerClientKeyFilePath, String brokerClientCertificateFilePath) { + if (brokerClientTlsEnabledWithKeyStore) { + adminBuilder.useKeyStoreTls(true) + .tlsTrustStoreType(brokerClientTlsTrustStoreType) + .tlsTrustStorePath(brokerClientTlsTrustStore) + .tlsTrustStorePassword(brokerClientTlsTrustStorePassword) + .tlsKeyStoreType(brokerClientTlsKeyStoreType) + .tlsKeyStorePath(brokerClientTlsKeyStore) + .tlsKeyStorePassword(brokerClientTlsKeyStorePassword); + } else { + adminBuilder.tlsTrustCertsFilePath(brokerClientTrustCertsFilePath) + .tlsKeyFilePath(brokerClientKeyFilePath) + .tlsCertificateFilePath(brokerClientCertificateFilePath); + } + adminBuilder.allowTlsInsecureConnection(isTlsAllowInsecureConnection); + } + public PulsarAdmin getClusterPulsarAdmin(String cluster, Optional clusterDataOp) { PulsarAdmin admin = clusterAdmins.get(cluster); if (admin != null) { @@ -1379,37 +1402,58 @@ public PulsarAdmin getClusterPulsarAdmin(String cluster, Optional c try { ClusterData data = clusterDataOp .orElseThrow(() -> new MetadataStoreException.NotFoundException(cluster)); + PulsarAdminBuilder builder = PulsarAdmin.builder(); ServiceConfiguration conf = pulsar.getConfig(); - - boolean isTlsUrl = conf.isBrokerClientTlsEnabled() && isNotBlank(data.getServiceUrlTls()); - String adminApiUrl = isTlsUrl ? data.getServiceUrlTls() : data.getServiceUrl(); - PulsarAdminBuilder builder = PulsarAdmin.builder().serviceHttpUrl(adminApiUrl); - // Apply all arbitrary configuration. This must be called before setting any fields annotated as // @Secret on the ClientConfigurationData object because of the way they are serialized. // See https://github.com/apache/pulsar/issues/8509 for more information. builder.loadConf(PropertiesUtils.filterAndMapProperties(conf.getProperties(), "brokerClient_")); - builder.authentication( - conf.getBrokerClientAuthenticationPlugin(), - conf.getBrokerClientAuthenticationParameters()); - - if (isTlsUrl) { - builder.allowTlsInsecureConnection(conf.isTlsAllowInsecureConnection()); - if (conf.isBrokerClientTlsEnabledWithKeyStore()) { - builder.useKeyStoreTls(true) - .tlsTrustStoreType(conf.getBrokerClientTlsTrustStoreType()) - .tlsTrustStorePath(conf.getBrokerClientTlsTrustStore()) - .tlsTrustStorePassword(conf.getBrokerClientTlsTrustStorePassword()) - .tlsKeyStoreType(conf.getBrokerClientTlsKeyStoreType()) - .tlsKeyStorePath(conf.getBrokerClientTlsKeyStore()) - .tlsKeyStorePassword(conf.getBrokerClientTlsKeyStorePassword()); - } else { - builder.tlsTrustCertsFilePath(conf.getBrokerClientTrustCertsFilePath()) - .tlsKeyFilePath(conf.getBrokerClientKeyFilePath()) - .tlsCertificateFilePath(conf.getBrokerClientCertificateFilePath()); - } + if (data.getAuthenticationPlugin() != null && data.getAuthenticationParameters() != null) { + builder.authentication(data.getAuthenticationPlugin(), data.getAuthenticationParameters()); + } else { + builder.authentication(pulsar.getConfiguration().getBrokerClientAuthenticationPlugin(), + pulsar.getConfiguration().getBrokerClientAuthenticationParameters()); + } + + boolean isTlsEnabled = data.isBrokerClientTlsEnabled() || conf.isBrokerClientTlsEnabled(); + if (isTlsEnabled && StringUtils.isEmpty(data.getServiceUrlTls())) { + throw new IllegalArgumentException("serviceUrlTls is empty, brokerClientTlsEnabled: " + + isTlsEnabled); + } else if (StringUtils.isEmpty(data.getServiceUrl())) { + throw new IllegalArgumentException("serviceUrl is empty, brokerClientTlsEnabled: " + isTlsEnabled); + } + String adminApiUrl = isTlsEnabled ? data.getServiceUrlTls() : data.getServiceUrl(); + builder.serviceHttpUrl(adminApiUrl); + if (data.isBrokerClientTlsEnabled()) { + configAdminTlsSettings(builder, + data.isBrokerClientTlsEnabledWithKeyStore(), + data.isTlsAllowInsecureConnection(), + data.getBrokerClientTlsTrustStoreType(), + data.getBrokerClientTlsTrustStore(), + data.getBrokerClientTlsTrustStorePassword(), + data.getBrokerClientTlsKeyStoreType(), + data.getBrokerClientTlsKeyStore(), + data.getBrokerClientTlsKeyStorePassword(), + data.getBrokerClientTrustCertsFilePath(), + data.getBrokerClientKeyFilePath(), + data.getBrokerClientCertificateFilePath() + ); + } else if (conf.isBrokerClientTlsEnabled()) { + configAdminTlsSettings(builder, + conf.isBrokerClientTlsEnabledWithKeyStore(), + conf.isTlsAllowInsecureConnection(), + conf.getBrokerClientTlsTrustStoreType(), + conf.getBrokerClientTlsTrustStore(), + conf.getBrokerClientTlsTrustStorePassword(), + conf.getBrokerClientTlsKeyStoreType(), + conf.getBrokerClientTlsKeyStore(), + conf.getBrokerClientTlsKeyStorePassword(), + conf.getBrokerClientTrustCertsFilePath(), + conf.getBrokerClientKeyFilePath(), + conf.getBrokerClientCertificateFilePath() + ); } // most of the admin request requires to make zk-call so, keep the max read-timeout based on diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorAdminTlsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorAdminTlsTest.java new file mode 100644 index 0000000000000..a5d14ca0487dc --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorAdminTlsTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; +import java.util.List; +import java.util.Optional; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.internal.PulsarAdminImpl; +import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class ReplicatorAdminTlsTest extends ReplicatorTestBase { + + @Override + @BeforeClass(timeOut = 300000) + public void setup() throws Exception { + super.setup(); + } + + @Override + @AfterClass(alwaysRun = true, timeOut = 300000) + public void cleanup() throws Exception { + super.cleanup(); + } + + @Test + public void testReplicationAdmin() throws Exception { + for (BrokerService ns : List.of(ns1, ns2, ns3)) { + // load the admin + ns.getClusterPulsarAdmin(cluster1, Optional.of(admin1.clusters().getCluster(cluster1))); + ns.getClusterPulsarAdmin(cluster2, Optional.of(admin1.clusters().getCluster(cluster2))); + ns.getClusterPulsarAdmin(cluster3, Optional.of(admin1.clusters().getCluster(cluster3))); + + // verify the admin + ConcurrentOpenHashMap clusterAdmins = ns.getClusterAdmins(); + assertFalse(clusterAdmins.isEmpty()); + clusterAdmins.forEach((cluster, admin) -> { + ClientConfigurationData clientConfigData = ((PulsarAdminImpl) admin).getClientConfigData(); + assertEquals(clientConfigData.getTlsTrustCertsFilePath(), caCertFilePath); + assertEquals(clientConfigData.getTlsKeyFilePath(), clientKeyFilePath); + assertEquals(clientConfigData.getTlsCertificateFilePath(), clientCertFilePath); + assertTrue(clientConfigData.isUseTls()); + }); + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorAdminTlsWithKeyStoreTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorAdminTlsWithKeyStoreTest.java new file mode 100644 index 0000000000000..3d3eb3faa727f --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorAdminTlsWithKeyStoreTest.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; +import java.util.List; +import java.util.Optional; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.internal.PulsarAdminImpl; +import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class ReplicatorAdminTlsWithKeyStoreTest extends ReplicatorTestBase { + + @Override + @BeforeClass(timeOut = 300000) + public void setup() throws Exception { + tlsWithKeyStore = true; + super.setup(); + } + + @Override + @AfterClass(alwaysRun = true, timeOut = 300000) + public void cleanup() throws Exception { + super.cleanup(); + } + + @Test + public void testReplicationAdmin() throws Exception { + for (BrokerService ns : List.of(ns1, ns2, ns3)) { + // load the admin + ns.getClusterPulsarAdmin(cluster1, Optional.of(admin1.clusters().getCluster(cluster1))); + ns.getClusterPulsarAdmin(cluster2, Optional.of(admin1.clusters().getCluster(cluster2))); + ns.getClusterPulsarAdmin(cluster3, Optional.of(admin1.clusters().getCluster(cluster3))); + + // verify the admin + ConcurrentOpenHashMap clusterAdmins = ns.getClusterAdmins(); + assertFalse(clusterAdmins.isEmpty()); + clusterAdmins.forEach((cluster, admin) -> { + ClientConfigurationData clientConfigData = ((PulsarAdminImpl) admin).getClientConfigData(); + assertEquals(clientConfigData.getTlsKeyStorePath(), clientKeyStorePath); + assertEquals(clientConfigData.getTlsKeyStorePassword(), keyStorePassword); + assertEquals(clientConfigData.getTlsKeyStoreType(), keyStoreType); + assertEquals(clientConfigData.getTlsTrustStorePath(), clientTrustStorePath); + assertEquals(clientConfigData.getTlsTrustStorePassword(), keyStorePassword); + assertEquals(clientConfigData.getTlsTrustStoreType(), keyStoreType); + assertTrue(clientConfigData.isUseKeyStoreTls()); + assertTrue(clientConfigData.isUseTls()); + }); + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTestBase.java index a7752b4a63fe2..b83e8ac9d2dbf 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTestBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTestBase.java @@ -21,6 +21,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; +import com.google.common.io.Resources; import com.google.common.collect.Sets; import io.netty.util.concurrent.DefaultThreadFactory; @@ -87,8 +88,29 @@ public abstract class ReplicatorTestBase extends TestRetrySupport { static final int TIME_TO_CHECK_BACKLOG_QUOTA = 5; - protected static final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/certificate/server.crt"; - protected static final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/certificate/server.key"; + // PEM + protected final String brokerCertFilePath = Resources.getResource("certificate-authority/server-keys/broker.cert.pem").getPath(); + protected final String brokerFilePath = Resources.getResource("certificate-authority/server-keys/broker.key-pk8.pem").getPath(); + protected final String clientCertFilePath = Resources.getResource("certificate-authority/client-keys/admin.cert.pem").getPath(); + protected final String clientKeyFilePath = Resources.getResource("certificate-authority/client-keys/admin.key-pk8.pem").getPath(); + protected final String caCertFilePath = Resources.getResource("certificate-authority/certs/ca.cert.pem").getPath(); + + // KEYSTORE + protected boolean tlsWithKeyStore = false; + protected final static String brokerKeyStorePath = + Resources.getResource("certificate-authority/jks/broker.keystore.jks").getPath(); + protected final static String brokerTrustStorePath = + Resources.getResource("certificate-authority/jks/broker.truststore.jks").getPath(); + protected final static String clientKeyStorePath = + Resources.getResource("certificate-authority/jks/client.keystore.jks").getPath(); + protected final static String clientTrustStorePath = + Resources.getResource("certificate-authority/jks/client.truststore.jks").getPath(); + protected final static String keyStoreType = "JKS"; + protected final static String keyStorePassword = "111111"; + + protected final String cluster1 = "r1"; + protected final String cluster2 = "r2"; + protected final String cluster3 = "r3"; // Default frequency public int getBrokerServicePurgeInactiveFrequency() { @@ -157,23 +179,56 @@ protected void setup() throws Exception { admin3 = PulsarAdmin.builder().serviceHttpUrl(url3.toString()).build(); // Provision the global namespace - admin1.clusters().createCluster("r1", ClusterData.builder() + admin1.clusters().createCluster(cluster1, ClusterData.builder() .serviceUrl(url1.toString()) .serviceUrlTls(urlTls1.toString()) .brokerServiceUrl(pulsar1.getBrokerServiceUrl()) .brokerServiceUrlTls(pulsar1.getBrokerServiceUrlTls()) + .brokerClientTlsEnabled(true) + .brokerClientCertificateFilePath(clientCertFilePath) + .brokerClientKeyFilePath(clientKeyFilePath) + .brokerClientTrustCertsFilePath(caCertFilePath) + .brokerClientTlsEnabledWithKeyStore(tlsWithKeyStore) + .brokerClientTlsKeyStore(clientKeyStorePath) + .brokerClientTlsKeyStorePassword(keyStorePassword) + .brokerClientTlsKeyStoreType(keyStoreType) + .brokerClientTlsTrustStore(clientTrustStorePath) + .brokerClientTlsTrustStorePassword(keyStorePassword) + .brokerClientTlsTrustStoreType(keyStoreType) .build()); - admin1.clusters().createCluster("r2", ClusterData.builder() + admin1.clusters().createCluster(cluster2, ClusterData.builder() .serviceUrl(url2.toString()) .serviceUrlTls(urlTls2.toString()) .brokerServiceUrl(pulsar2.getBrokerServiceUrl()) .brokerServiceUrlTls(pulsar2.getBrokerServiceUrlTls()) + .brokerClientTlsEnabled(true) + .brokerClientCertificateFilePath(clientCertFilePath) + .brokerClientKeyFilePath(clientKeyFilePath) + .brokerClientTrustCertsFilePath(caCertFilePath) + .brokerClientTlsEnabledWithKeyStore(tlsWithKeyStore) + .brokerClientTlsKeyStore(clientKeyStorePath) + .brokerClientTlsKeyStorePassword(keyStorePassword) + .brokerClientTlsKeyStoreType(keyStoreType) + .brokerClientTlsTrustStore(clientTrustStorePath) + .brokerClientTlsTrustStorePassword(keyStorePassword) + .brokerClientTlsTrustStoreType(keyStoreType) .build()); - admin1.clusters().createCluster("r3", ClusterData.builder() + admin1.clusters().createCluster(cluster3, ClusterData.builder() .serviceUrl(url3.toString()) .serviceUrlTls(urlTls3.toString()) .brokerServiceUrl(pulsar3.getBrokerServiceUrl()) .brokerServiceUrlTls(pulsar3.getBrokerServiceUrlTls()) + .brokerClientTlsEnabled(true) + .brokerClientCertificateFilePath(clientCertFilePath) + .brokerClientKeyFilePath(clientKeyFilePath) + .brokerClientTrustCertsFilePath(caCertFilePath) + .brokerClientTlsEnabledWithKeyStore(tlsWithKeyStore) + .brokerClientTlsKeyStore(clientKeyStorePath) + .brokerClientTlsKeyStorePassword(keyStorePassword) + .brokerClientTlsKeyStoreType(keyStoreType) + .brokerClientTlsTrustStore(clientTrustStorePath) + .brokerClientTlsTrustStorePassword(keyStorePassword) + .brokerClientTlsTrustStoreType(keyStoreType) .build()); admin1.tenants().createTenant("pulsar", @@ -181,12 +236,12 @@ protected void setup() throws Exception { admin1.namespaces().createNamespace("pulsar/ns", Sets.newHashSet("r1", "r2", "r3")); admin1.namespaces().createNamespace("pulsar/ns1", Sets.newHashSet("r1", "r2")); - assertEquals(admin2.clusters().getCluster("r1").getServiceUrl(), url1.toString()); - assertEquals(admin2.clusters().getCluster("r2").getServiceUrl(), url2.toString()); - assertEquals(admin2.clusters().getCluster("r3").getServiceUrl(), url3.toString()); - assertEquals(admin2.clusters().getCluster("r1").getBrokerServiceUrl(), pulsar1.getBrokerServiceUrl()); - assertEquals(admin2.clusters().getCluster("r2").getBrokerServiceUrl(), pulsar2.getBrokerServiceUrl()); - assertEquals(admin2.clusters().getCluster("r3").getBrokerServiceUrl(), pulsar3.getBrokerServiceUrl()); + assertEquals(admin2.clusters().getCluster(cluster1).getServiceUrl(), url1.toString()); + assertEquals(admin2.clusters().getCluster(cluster2).getServiceUrl(), url2.toString()); + assertEquals(admin2.clusters().getCluster(cluster3).getServiceUrl(), url3.toString()); + assertEquals(admin2.clusters().getCluster(cluster1).getBrokerServiceUrl(), pulsar1.getBrokerServiceUrl()); + assertEquals(admin2.clusters().getCluster(cluster2).getBrokerServiceUrl(), pulsar2.getBrokerServiceUrl()); + assertEquals(admin2.clusters().getCluster(cluster3).getBrokerServiceUrl(), pulsar3.getBrokerServiceUrl()); // Also create V1 namespace for compatibility check admin1.clusters().createCluster("global", ClusterData.builder() @@ -194,7 +249,7 @@ protected void setup() throws Exception { .serviceUrlTls("https://global:8443") .build()); admin1.namespaces().createNamespace("pulsar/global/ns"); - admin1.namespaces().setNamespaceReplicationClusters("pulsar/global/ns", Sets.newHashSet("r1", "r2", "r3")); + admin1.namespaces().setNamespaceReplicationClusters("pulsar/global/ns", Sets.newHashSet(cluster1, cluster2, cluster3)); Thread.sleep(100); log.info("--- ReplicatorTestBase::setup completed ---"); @@ -207,11 +262,11 @@ public void setConfig3DefaultValue() { } public void setConfig1DefaultValue(){ - setConfigDefaults(config1, "r1", bkEnsemble1); + setConfigDefaults(config1, cluster1, bkEnsemble1); } public void setConfig2DefaultValue() { - setConfigDefaults(config2, "r2", bkEnsemble2); + setConfigDefaults(config2, cluster2, bkEnsemble2); } private void setConfigDefaults(ServiceConfiguration config, String clusterName, @@ -229,9 +284,16 @@ private void setConfigDefaults(ServiceConfiguration config, String clusterName, config.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); config.setBrokerServicePort(Optional.of(0)); config.setBrokerServicePortTls(Optional.of(0)); - config.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH); - config.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH); - config.setTlsTrustCertsFilePath(TLS_SERVER_CERT_FILE_PATH); + config.setTlsCertificateFilePath(brokerCertFilePath); + config.setTlsKeyFilePath(brokerFilePath); + config.setTlsTrustCertsFilePath(caCertFilePath); + config.setTlsEnabledWithKeyStore(tlsWithKeyStore); + config.setTlsKeyStore(brokerKeyStorePath); + config.setTlsKeyStoreType(keyStoreType); + config.setTlsKeyStorePassword(keyStorePassword); + config.setTlsTrustStore(brokerTrustStorePath); + config.setTlsTrustStoreType(keyStoreType); + config.setTlsTrustStorePassword(keyStorePassword); config.setBacklogQuotaCheckIntervalInSeconds(TIME_TO_CHECK_BACKLOG_QUOTA); config.setDefaultNumberOfNamespaceBundles(1); config.setAllowAutoTopicCreationType(TopicType.NON_PARTITIONED); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTlsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTlsTest.java index e1506d16188b0..49e4e79539499 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTlsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTlsTest.java @@ -21,6 +21,8 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; import java.util.List; +import java.util.Optional; +import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,10 +52,18 @@ public void cleanup() throws Exception { public void testReplicationClient() throws Exception { log.info("--- Starting ReplicatorTlsTest::testReplicationClient ---"); for (BrokerService ns : List.of(ns1, ns2, ns3)) { + // load the client + ns.getReplicationClient(cluster1, Optional.of(admin1.clusters().getCluster(cluster1))); + ns.getReplicationClient(cluster2, Optional.of(admin1.clusters().getCluster(cluster2))); + ns.getReplicationClient(cluster3, Optional.of(admin1.clusters().getCluster(cluster3))); + + // verify the client ns.getReplicationClients().forEach((cluster, client) -> { - assertTrue(((PulsarClientImpl) client).getConfiguration().isUseTls()); - assertEquals(((PulsarClientImpl) client).getConfiguration().getTlsTrustCertsFilePath(), - TLS_SERVER_CERT_FILE_PATH); + ClientConfigurationData configuration = ((PulsarClientImpl) client).getConfiguration(); + assertTrue(configuration.isUseTls()); + assertEquals(configuration.getTlsTrustCertsFilePath(), caCertFilePath); + assertEquals(configuration.getTlsKeyFilePath(), clientKeyFilePath); + assertEquals(configuration.getTlsCertificateFilePath(), clientCertFilePath); }); } } From fb7f14ceb04d612e456b2e5a834385ae3a97f68f Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Wed, 22 Feb 2023 00:59:08 -0600 Subject: [PATCH 043/174] [cleanup][broker] Update deprecation warnings to use 3.0.0 (#19586) ### Motivation With https://github.com/apache/pulsar/pull/19573, we should replace deprecation warnings that were set to 2.12.0 with 3.0.0. ### Modifications * Update 4 deprecation warnings. ### Verifying this change This change is a trivial rework / code cleanup without any test coverage. ### Does this pull request potentially affect one of the following parts: This is cosmetic. ### Documentation - [x] `doc` This is a documentation change. ### Matching PR in forked repository PR in forked repository: skipping forked PR since this is a trivial change --- .../pulsar/broker/authentication/AuthenticationProvider.java | 2 +- .../pulsar/broker/authentication/AuthenticationService.java | 2 +- .../broker/authentication/OneStageAuthenticationState.java | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProvider.java index dd1942e318a08..109259537a494 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProvider.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProvider.java @@ -114,7 +114,7 @@ default AuthenticationState newAuthState(AuthData authData, * an {@link AuthenticationDataSource} that was added as the {@link AuthenticatedDataAttributeName} attribute to * the http request. Removing this method removes an unnecessary step in the authentication flow.

*/ - @Deprecated(since = "2.12.0") + @Deprecated(since = "3.0.0") default AuthenticationState newHttpAuthState(HttpServletRequest request) throws AuthenticationException { return new OneStageAuthenticationState(request, this); diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationService.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationService.java index e758c2de7e308..d11bb6d76e82e 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationService.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationService.java @@ -158,7 +158,7 @@ public boolean authenticateHttpRequest(HttpServletRequest request, HttpServletRe /** * @deprecated use {@link #authenticateHttpRequest(HttpServletRequest, HttpServletResponse)} */ - @Deprecated(since = "2.12.0") + @Deprecated(since = "3.0.0") public String authenticateHttpRequest(HttpServletRequest request, AuthenticationDataSource authData) throws AuthenticationException { String authMethodName = getAuthMethodName(request); diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/OneStageAuthenticationState.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/OneStageAuthenticationState.java index 725f24489d5f8..242564a8d1f5a 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/OneStageAuthenticationState.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/OneStageAuthenticationState.java @@ -106,7 +106,7 @@ public CompletableFuture authenticateAsync(AuthData authData) { /** * @deprecated use {@link #authenticateAsync(AuthData)} */ - @Deprecated(since = "2.12.0") + @Deprecated(since = "3.0.0") @Override public AuthData authenticate(AuthData authData) throws AuthenticationException { try { @@ -120,7 +120,7 @@ public AuthData authenticate(AuthData authData) throws AuthenticationException { * @deprecated rely on result from {@link #authenticateAsync(AuthData)}. For more information, see the Javadoc * for {@link AuthenticationState#isComplete()}. */ - @Deprecated(since = "2.12.0") + @Deprecated(since = "3.0.0") @Override public boolean isComplete() { return authRole != null; From f292bad8ac92eaa0c9f6735b12742c0aa27be5e8 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Wed, 22 Feb 2023 11:09:57 -0600 Subject: [PATCH 044/174] [fix][test] ProxyWithAuthorizationTest remove SAN from test certs (#19594) --- .../authentication/tls/broker-cert.pem | 79 ++++++----- .../resources/authentication/tls/cacert.pem | 125 +++++++++--------- .../authentication/tls/client-cert.pem | 79 ++++++----- build/regenerate_certs_for_tests.sh | 16 ++- .../server/ProxyWithAuthorizationTest.java | 38 +++--- .../broker-cacert.pem | 125 +++++++++--------- .../broker-cert.pem | 79 ++++++----- .../client-cacert.pem | 125 +++++++++--------- .../client-cert.pem | 79 ++++++----- .../no-subject-alt-cert.pem | 67 ++++++++++ .../no-subject-alt-key.pem | 28 ++++ .../proxy-cacert.pem | 125 +++++++++--------- .../ProxyWithAuthorizationTest/proxy-cert.pem | 79 ++++++----- .../resources/authentication/tls/cacert.pem | 125 +++++++++--------- .../authentication/tls/client-cert.pem | 79 ++++++----- .../authentication/tls/server-cert.pem | 79 ++++++----- 16 files changed, 718 insertions(+), 609 deletions(-) create mode 100644 pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/no-subject-alt-cert.pem create mode 100644 pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/no-subject-alt-key.pem diff --git a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-cert.pem b/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-cert.pem index e9be840d3a083..e2b44e0bf0c42 100644 --- a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-cert.pem +++ b/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-cert.pem @@ -1,17 +1,16 @@ Certificate: Data: Version: 3 (0x2) - Serial Number: - 61:e6:1b:07:90:6a:4f:f7:cd:46:b9:59:1d:3e:1c:39:0d:f2:5e:05 - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN = CARoot + Serial Number: 15537474201172114493 (0xd7a0327703a8fc3d) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=CARoot Validity - Not Before: May 30 13:38:24 2022 GMT - Not After : May 27 13:38:24 2032 GMT - Subject: C = US, ST = CA, O = Apache, OU = Apache Pulsar, CN = localhost + Not Before: Feb 22 06:26:33 2023 GMT + Not After : Feb 19 06:26:33 2033 GMT + Subject: C=US, ST=CA, O=Apache, OU=Apache Pulsar, CN=localhost Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (2048 bit) + Public-Key: (2048 bit) Modulus: 00:af:bf:b7:2d:98:ad:9d:f6:da:a3:13:d4:62:0f: 98:be:1c:a2:89:22:ba:6f:d5:fd:1f:67:e3:91:03: @@ -36,37 +35,37 @@ Certificate: X509v3 Subject Alternative Name: DNS:localhost, IP Address:127.0.0.1 Signature Algorithm: sha256WithRSAEncryption - 88:1d:a7:42:a1:1c:87:45:4a:e6:5e:aa:9c:7b:71:2e:5c:9e: - 11:85:0f:a3:c5:b4:ea:73:9e:b7:61:9d:4a:e9:cd:1a:c5:2e: - 03:be:a3:2b:b6:12:6a:15:03:04:3f:fb:4a:09:0d:84:0e:dd: - c0:63:2b:0f:13:fb:1f:98:64:49:48:e7:96:d5:41:c4:ca:94: - bf:ab:c5:ea:80:2c:ee:1f:ab:12:54:74:f1:f1:56:ea:03:c0: - 1c:0d:8d:b9:6e:b0:d0:5f:21:c1:d3:e3:45:df:cf:64:69:13: - 6c:54:79:06:7d:53:46:77:3c:21:cc:c4:6a:5f:f9:9a:07:0f: - a5:95:20:f0:0e:93:07:48:96:a9:2c:28:50:21:d7:f8:13:4f: - b8:ca:aa:1f:a6:41:7c:71:1f:ad:11:3f:3d:1e:e9:81:3c:86: - c1:af:2d:39:a0:13:9f:99:ec:9a:47:44:df:28:02:a7:1d:6a: - 8d:c0:1e:24:e8:19:fc:1d:dc:67:29:04:be:0a:d6:c5:81:59: - 27:2c:f5:e5:df:ba:0b:c6:50:e5:b3:bd:73:12:3e:2c:ef:a6: - 8a:ed:eb:86:9a:45:45:52:a3:44:78:12:60:17:e2:3a:32:92: - 03:6e:89:89:16:c5:e0:bc:be:a7:cb:93:4b:d8:56:33:a0:a0: - 53:b2:0d:a5 + 5f:e0:73:7b:5e:db:c0:8b:5e:4c:43:5f:80:94:ca:0b:f8:e9: + 9b:93:91:3d:b1:3a:99:ce:1c:fb:15:32:68:3e:b9:9c:52:d0: + 4b:7f:17:09:ec:af:6b:05:3e:e2:a3:e6:cc:bb:53:d7:ea:4a: + 82:3c:4e:a5:37:ca:f4:1e:38:e2:d6:a5:98:4d:ee:b9:e2:9a: + 48:d2:9f:0a:bc:61:42:70:22:b9:fb:cd:73:72:fb:94:13:ac: + 6e:c5:b6:4b:24:ef:0f:df:2d:e6:56:da:b2:76:e8:16:be:7f: + 3f:1b:99:6e:32:3e:b9:f4:2b:35:72:c7:e4:c6:a5:92:68:c0: + 1f:a0:f7:17:fd:a3:b6:73:98:d3:ea:1c:af:ea:7d:f8:a0:27: + 40:dc:4e:8b:13:28:ba:65:60:c5:90:57:e8:54:c1:83:b4:9d: + f0:ae:2a:de:27:57:e5:a2:e5:f4:87:1c:df:6b:dc:7b:43:ff: + b6:be:0b:3b:b2:8b:1a:36:dc:e3:57:aa:52:ef:23:d6:50:d7: + e4:72:8f:a0:0a:43:de:3d:f2:42:5b:fa:ed:1f:8d:0e:cf:c5: + 6a:ce:3b:8e:fd:6b:68:01:a9:f9:d2:0e:0d:ac:39:8d:f5:6c: + 80:f8:49:af:bb:b9:d4:81:b9:f3:b2:b6:ce:75:1c:20:e8:6a: + 53:dc:26:86 -----BEGIN CERTIFICATE----- -MIIDFDCCAfygAwIBAgIUYeYbB5BqT/fNRrlZHT4cOQ3yXgUwDQYJKoZIhvcNAQEL -BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz -MzgyNFowVzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMQ8wDQYDVQQKEwZBcGFj -aGUxFjAUBgNVBAsTDUFwYWNoZSBQdWxzYXIxEjAQBgNVBAMTCWxvY2FsaG9zdDCC -ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK+/ty2YrZ322qMT1GIPmL4c -ookium/V/R9n45EDmICBDu3Y9nB/LDZoPVPqWDqm1YlmS70eV3ETbUsR5UCldoQk -kkBYgJbJHyzEVeujeXNwXDeaie0vumvjgnxpSgJUi4FePL9MisvqLF6D57cQCF+C -WKOJ0dqSuioo7jAoP1uuEHGWx+ESxbAarURvRDoRSpo8D40GgHs07z9s9F7FRFQe -yN3HgIWA2WjmxlMDd+H+GGEHdwVM7Vm8XUE4au9dobJgmNRIKJUCig79z3sb0hHM -EAxQc9fMOGyD3XkmqpDIm4SGvFnpYmn0mBvEgHh+oBqBndLhZt3EzPxjBKzspzUC -AwEAAaMeMBwwGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEB -CwUAA4IBAQCIHadCoRyHRUrmXqqce3EuXJ4RhQ+jxbTqc563YZ1K6c0axS4DvqMr -thJqFQMEP/tKCQ2EDt3AYysPE/sfmGRJSOeW1UHEypS/q8XqgCzuH6sSVHTx8Vbq -A8AcDY25brDQXyHB0+NF389kaRNsVHkGfVNGdzwhzMRqX/maBw+llSDwDpMHSJap -LChQIdf4E0+4yqofpkF8cR+tET89HumBPIbBry05oBOfmeyaR0TfKAKnHWqNwB4k -6Bn8HdxnKQS+CtbFgVknLPXl37oLxlDls71zEj4s76aK7euGmkVFUqNEeBJgF+I6 -MpIDbomJFsXgvL6ny5NL2FYzoKBTsg2l +MIIDCTCCAfGgAwIBAgIJANegMncDqPw9MA0GCSqGSIb3DQEBCwUAMBExDzANBgNV +BAMMBkNBUm9vdDAeFw0yMzAyMjIwNjI2MzNaFw0zMzAyMTkwNjI2MzNaMFcxCzAJ +BgNVBAYTAlVTMQswCQYDVQQIEwJDQTEPMA0GA1UEChMGQXBhY2hlMRYwFAYDVQQL +Ew1BcGFjaGUgUHVsc2FyMRIwEAYDVQQDEwlsb2NhbGhvc3QwggEiMA0GCSqGSIb3 +DQEBAQUAA4IBDwAwggEKAoIBAQCvv7ctmK2d9tqjE9RiD5i+HKKJIrpv1f0fZ+OR +A5iAgQ7t2PZwfyw2aD1T6lg6ptWJZku9HldxE21LEeVApXaEJJJAWICWyR8sxFXr +o3lzcFw3montL7pr44J8aUoCVIuBXjy/TIrL6ixeg+e3EAhfglijidHakroqKO4w +KD9brhBxlsfhEsWwGq1Eb0Q6EUqaPA+NBoB7NO8/bPRexURUHsjdx4CFgNlo5sZT +A3fh/hhhB3cFTO1ZvF1BOGrvXaGyYJjUSCiVAooO/c97G9IRzBAMUHPXzDhsg915 +JqqQyJuEhrxZ6WJp9JgbxIB4fqAagZ3S4WbdxMz8YwSs7Kc1AgMBAAGjHjAcMBoG +A1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEAX+Bz +e17bwIteTENfgJTKC/jpm5ORPbE6mc4c+xUyaD65nFLQS38XCeyvawU+4qPmzLtT +1+pKgjxOpTfK9B444talmE3uueKaSNKfCrxhQnAiufvNc3L7lBOsbsW2SyTvD98t +5lbasnboFr5/PxuZbjI+ufQrNXLH5MalkmjAH6D3F/2jtnOY0+ocr+p9+KAnQNxO +ixMoumVgxZBX6FTBg7Sd8K4q3idX5aLl9Icc32vce0P/tr4LO7KLGjbc41eqUu8j +1lDX5HKPoApD3j3yQlv67R+NDs/Fas47jv1raAGp+dIODaw5jfVsgPhJr7u51IG5 +87K2znUcIOhqU9wmhg== -----END CERTIFICATE----- diff --git a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/cacert.pem b/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/cacert.pem index 21bbaba213f69..4ed454ec52a52 100644 --- a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/cacert.pem +++ b/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/cacert.pem @@ -1,77 +1,78 @@ Certificate: Data: Version: 3 (0x2) - Serial Number: - 70:4c:6b:e0:aa:cc:01:77:f2:1f:04:8c:d4:72:03:a5:32:5f:c7:be - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN = CARoot + Serial Number: 15358526754272834781 (0xd52472b5c5c3f4dd) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=CARoot Validity - Not Before: May 30 13:38:24 2022 GMT - Not After : May 27 13:38:24 2032 GMT - Subject: CN = CARoot + Not Before: Feb 22 06:26:32 2023 GMT + Not After : Feb 19 06:26:32 2033 GMT + Subject: CN=CARoot Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (2048 bit) + Public-Key: (2048 bit) Modulus: - 00:dc:9c:01:30:5f:c5:42:48:10:78:30:5d:66:20: - 0e:74:61:f6:82:74:9f:6f:b2:ed:00:9e:6c:21:b6: - 83:21:6b:54:34:e8:a9:dc:81:83:7a:0e:9f:cc:3d: - eb:97:ee:cf:ca:0e:5f:96:81:dc:e7:75:88:91:2f: - d5:65:74:c2:d8:67:58:d8:41:6a:5f:a9:79:dc:29: - 36:4a:b8:39:20:d2:f8:a8:59:9f:e3:be:f9:61:80: - 1b:ce:63:bb:12:56:06:b9:77:4e:6a:40:65:9b:bf: - 5b:f8:27:88:f5:ff:40:ee:47:bc:2d:8e:c3:a6:62: - 0d:18:76:d1:f5:af:1a:6b:25:4e:d4:55:15:f0:e3: - 97:1b:68:eb:75:b8:80:ea:64:ef:7e:e2:f0:5c:da: - 6d:d6:16:7b:0f:5e:ae:72:47:5a:df:0b:8a:e0:74: - c1:b7:82:0d:97:41:d7:84:16:51:40:37:15:a1:eb: - 70:0c:f1:5a:26:39:11:1e:97:b9:36:32:ce:16:b9: - 42:ad:31:5b:1e:89:f5:3e:07:0e:d6:fc:9a:46:8e: - 87:89:90:5c:f3:00:e4:9b:ce:7b:93:fe:9a:d8:65: - ec:49:5c:e8:eb:41:3d:53:bc:ce:e8:6d:44:ec:76: - 3f:e6:9b:13:e4:f8:d0:1c:00:e6:4f:73:e1:b0:27: - 6f:99 + 00:d0:87:45:0b:b4:83:11:ab:5a:b4:b6:1c:15:d4: + 92:6a:0c:ac:3b:76:da:ff:8d:61:1b:bd:96:bd:d7: + b0:70:23:87:d4:00:19:b2:e5:63:b7:80:58:4a:a4: + d8:a8:a6:4f:eb:c8:8c:54:07:f5:56:52:23:64:fc: + 66:54:39:f1:33:d0:e5:cc:b6:40:c8:d7:9a:9f:0e: + c4:aa:57:b0:b3:e2:41:61:54:ca:1f:90:3b:18:ef: + 60:d2:dc:ee:34:29:33:08:1b:37:4b:c4:ca:7e:cb: + 94:7f:50:c4:8d:16:2f:90:03:94:07:bf:cf:52:ff: + 24:54:56:ac:74:6c:d3:31:8c:ce:ef:b3:14:5a:5b: + 8a:0c:83:2d:e1:f7:4d:60:2f:a1:4d:85:38:96:7f: + 01:2f:9a:99:c7:2e:3d:09:4d:5e:53:df:fd:29:9f: + ff:6b:e4:c2:a1:e3:67:85:db:e2:02:4d:6f:29:d4: + e1:b3:a2:34:71:e0:90:dd:3f:b3:3f:86:41:8c:97: + 09:e6:c3:de:a0:0e:d3:d4:3e:ce:ea:58:70:e6:9f: + 24:a8:19:ca:df:61:b8:9c:c3:4e:53:d0:69:96:44: + 84:76:2b:99:65:08:06:42:d4:b2:76:a7:2f:69:12: + d5:c2:65:a6:ff:2c:77:73:00:e7:97:a5:77:6b:8a: + 9c:3f Exponent: 65537 (0x10001) X509v3 extensions: + X509v3 Basic Constraints: critical + CA:TRUE X509v3 Subject Key Identifier: - 8B:30:D2:81:7C:BE:AB:4D:76:37:19:2B:69:5E:DB:F7:81:95:73:F5 + A7:55:6B:51:10:75:CE:4E:5B:0B:64:FF:A9:6D:23:FB:57:88:59:69 X509v3 Authority Key Identifier: - keyid:8B:30:D2:81:7C:BE:AB:4D:76:37:19:2B:69:5E:DB:F7:81:95:73:F5 + keyid:A7:55:6B:51:10:75:CE:4E:5B:0B:64:FF:A9:6D:23:FB:57:88:59:69 + DirName:/CN=CARoot + serial:D5:24:72:B5:C5:C3:F4:DD - X509v3 Basic Constraints: critical - CA:TRUE Signature Algorithm: sha256WithRSAEncryption - 02:4c:80:4f:a4:b5:f4:70:be:82:cf:3a:ed:40:f9:97:17:22: - 07:5d:e0:9b:4e:54:f8:4b:64:99:f5:07:7f:87:5b:9c:60:ec: - 9f:69:e6:00:97:5a:cd:14:59:31:45:be:b7:bd:c4:ce:57:82: - 1a:4a:62:ce:8e:c8:59:d5:62:43:8b:94:c0:ab:c2:cc:3a:a0: - 69:d3:65:15:82:35:de:85:64:e6:7b:d9:3a:22:12:77:f7:71: - 82:86:d7:6c:e5:69:d5:3a:f2:a7:25:f7:dc:f3:6f:cb:eb:85: - 48:44:63:e2:6d:3c:82:eb:3a:c0:e1:bd:9d:3a:12:11:66:1f: - 05:8f:49:65:31:d6:cf:26:06:46:ba:73:c7:ad:61:fc:14:5f: - 68:d1:ee:02:5f:4b:98:b6:5b:0c:98:4e:61:7b:cb:35:ee:44: - a1:ce:e1:00:a2:56:f0:0d:72:3b:58:66:e8:9a:dc:62:d5:95: - 3e:5a:48:21:a8:7c:f8:1f:5a:13:db:53:33:11:3e:e6:14:39: - cd:2b:3f:77:5b:ee:f7:0c:59:69:2f:46:9a:34:56:89:05:8e: - 40:94:94:3f:95:f6:fa:f9:1a:e8:1a:80:7b:1d:f7:0c:a1:be: - e2:38:98:fd:0f:e7:68:4d:7d:fe:ae:5f:e3:32:c6:5d:37:77: - 7a:28:ce:cc + 21:b1:4d:2b:14:1e:5a:91:5d:28:9e:ba:cb:ed:f1:96:da:c3: + fa:8d:b5:74:e4:c5:fb:2f:3e:39:b4:a6:59:69:dd:84:64:a8: + f0:e0:39:d2:ef:87:cc:8b:09:9f:0a:84:1f:d0:96:9c:4b:64: + ea:08:09:26:1c:84:f4:06:5f:5e:b9:ba:b3:3c:6c:81:e0:93: + 46:89:07:51:95:36:77:96:76:5d:a6:68:71:bb:60:88:a7:83: + 27:7c:66:5d:64:36:cb:8e:bd:02:f7:fb:52:63:83:2f:fe:57: + 4c:d5:0c:1b:ea:ef:88:ad:8c:a9:d4:b3:2c:b8:c4:e2:90:cb: + 0f:24:0e:df:fc:2a:c6:83:08:49:45:b0:41:85:0e:b4:6f:f7: + 18:56:7b:a5:0b:f6:1b:7f:72:88:ee:c8:ef:b3:e3:3e:f0:68: + 1b:c9:55:bb:4d:21:65:6b:9e:5c:dd:60:4b:7f:f1:84:f8:67: + 51:c2:60:88:42:6e:6c:9c:14:b8:96:b0:18:10:97:2c:94:e7: + 79:14:7b:d1:a2:a4:d8:94:84:ac:a9:ca:17:95:c2:27:8b:2b: + d8:19:6a:14:4b:c3:03:a6:30:55:40:bd:ce:0c:c2:d5:af:7d: + 6d:65:89:6b:74:ed:21:12:f1:aa:c9:c9:ba:da:9a:ca:14:6c: + 39:f4:02:32 -----BEGIN CERTIFICATE----- -MIIDAzCCAeugAwIBAgIUcExr4KrMAXfyHwSM1HIDpTJfx74wDQYJKoZIhvcNAQEL -BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz -MzgyNFowETEPMA0GA1UEAwwGQ0FSb290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A -MIIBCgKCAQEA3JwBMF/FQkgQeDBdZiAOdGH2gnSfb7LtAJ5sIbaDIWtUNOip3IGD -eg6fzD3rl+7Pyg5floHc53WIkS/VZXTC2GdY2EFqX6l53Ck2Srg5INL4qFmf4775 -YYAbzmO7ElYGuXdOakBlm79b+CeI9f9A7ke8LY7DpmINGHbR9a8aayVO1FUV8OOX -G2jrdbiA6mTvfuLwXNpt1hZ7D16uckda3wuK4HTBt4INl0HXhBZRQDcVoetwDPFa -JjkRHpe5NjLOFrlCrTFbHon1PgcO1vyaRo6HiZBc8wDkm857k/6a2GXsSVzo60E9 -U7zO6G1E7HY/5psT5PjQHADmT3PhsCdvmQIDAQABo1MwUTAdBgNVHQ4EFgQUizDS -gXy+q012NxkraV7b94GVc/UwHwYDVR0jBBgwFoAUizDSgXy+q012NxkraV7b94GV -c/UwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAAkyAT6S19HC+ -gs867UD5lxciB13gm05U+EtkmfUHf4dbnGDsn2nmAJdazRRZMUW+t73EzleCGkpi -zo7IWdViQ4uUwKvCzDqgadNlFYI13oVk5nvZOiISd/dxgobXbOVp1TrypyX33PNv -y+uFSERj4m08gus6wOG9nToSEWYfBY9JZTHWzyYGRrpzx61h/BRfaNHuAl9LmLZb -DJhOYXvLNe5Eoc7hAKJW8A1yO1hm6JrcYtWVPlpIIah8+B9aE9tTMxE+5hQ5zSs/ -d1vu9wxZaS9GmjRWiQWOQJSUP5X2+vka6BqAex33DKG+4jiY/Q/naE19/q5f4zLG -XTd3eijOzA== +MIIDGjCCAgKgAwIBAgIJANUkcrXFw/TdMA0GCSqGSIb3DQEBCwUAMBExDzANBgNV +BAMMBkNBUm9vdDAeFw0yMzAyMjIwNjI2MzJaFw0zMzAyMTkwNjI2MzJaMBExDzAN +BgNVBAMMBkNBUm9vdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANCH +RQu0gxGrWrS2HBXUkmoMrDt22v+NYRu9lr3XsHAjh9QAGbLlY7eAWEqk2KimT+vI +jFQH9VZSI2T8ZlQ58TPQ5cy2QMjXmp8OxKpXsLPiQWFUyh+QOxjvYNLc7jQpMwgb +N0vEyn7LlH9QxI0WL5ADlAe/z1L/JFRWrHRs0zGMzu+zFFpbigyDLeH3TWAvoU2F +OJZ/AS+amccuPQlNXlPf/Smf/2vkwqHjZ4Xb4gJNbynU4bOiNHHgkN0/sz+GQYyX +CebD3qAO09Q+zupYcOafJKgZyt9huJzDTlPQaZZEhHYrmWUIBkLUsnanL2kS1cJl +pv8sd3MA55eld2uKnD8CAwEAAaN1MHMwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E +FgQUp1VrURB1zk5bC2T/qW0j+1eIWWkwQQYDVR0jBDowOIAUp1VrURB1zk5bC2T/ +qW0j+1eIWWmhFaQTMBExDzANBgNVBAMMBkNBUm9vdIIJANUkcrXFw/TdMA0GCSqG +SIb3DQEBCwUAA4IBAQAhsU0rFB5akV0onrrL7fGW2sP6jbV05MX7Lz45tKZZad2E +ZKjw4DnS74fMiwmfCoQf0JacS2TqCAkmHIT0Bl9eubqzPGyB4JNGiQdRlTZ3lnZd +pmhxu2CIp4MnfGZdZDbLjr0C9/tSY4Mv/ldM1Qwb6u+IrYyp1LMsuMTikMsPJA7f +/CrGgwhJRbBBhQ60b/cYVnulC/Ybf3KI7sjvs+M+8GgbyVW7TSFla55c3WBLf/GE ++GdRwmCIQm5snBS4lrAYEJcslOd5FHvRoqTYlISsqcoXlcIniyvYGWoUS8MDpjBV +QL3ODMLVr31tZYlrdO0hEvGqycm62prKFGw59AIy -----END CERTIFICATE----- diff --git a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/client-cert.pem b/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/client-cert.pem index e5d9e6e74b233..3cf236c401255 100644 --- a/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/client-cert.pem +++ b/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/client-cert.pem @@ -1,17 +1,16 @@ Certificate: Data: Version: 3 (0x2) - Serial Number: - 61:e6:1b:07:90:6a:4f:f7:cd:46:b9:59:1d:3e:1c:39:0d:f2:5e:06 - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN = CARoot + Serial Number: 15537474201172114494 (0xd7a0327703a8fc3e) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=CARoot Validity - Not Before: May 30 13:38:24 2022 GMT - Not After : May 27 13:38:24 2032 GMT - Subject: C = US, ST = CA, O = Apache, OU = Apache Pulsar, CN = superUser + Not Before: Feb 22 06:26:33 2023 GMT + Not After : Feb 19 06:26:33 2033 GMT + Subject: C=US, ST=CA, O=Apache, OU=Apache Pulsar, CN=superUser Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (2048 bit) + Public-Key: (2048 bit) Modulus: 00:cd:43:7d:98:40:f9:b0:5b:bc:ae:db:c0:0b:ad: 26:90:96:e0:62:38:ed:68:b1:70:46:3b:de:44:f9: @@ -36,37 +35,37 @@ Certificate: X509v3 Subject Alternative Name: DNS:localhost, IP Address:127.0.0.1 Signature Algorithm: sha256WithRSAEncryption - 90:62:ba:7b:6f:45:95:7a:71:2f:e7:88:0c:64:b8:6c:05:86: - 7f:47:08:ce:d6:e2:5a:32:13:0c:82:ad:a7:af:f0:a2:f7:86: - 79:87:1a:89:78:95:b1:9f:be:c5:8b:39:fd:12:94:b6:e1:69: - ff:fa:1e:c3:82:d8:6c:03:80:45:ac:1c:06:70:bb:77:c3:41: - 5f:b6:9d:fe:36:6f:ae:23:6c:bf:43:79:8e:74:85:8e:96:89: - a9:c4:6d:d9:fa:05:ba:a8:11:7c:82:45:94:3d:9f:b6:7c:2f: - 4e:6d:37:c3:fb:79:7e:0c:d2:15:fa:0e:ea:2d:c9:24:f3:34: - 13:6f:db:d7:55:e1:0c:2f:7e:fe:4c:3b:fa:7e:03:26:0f:6a: - 95:d2:22:ce:27:71:6a:97:ac:36:0a:20:ec:19:a0:78:23:0c: - 54:f3:b1:dd:33:36:7c:b7:61:23:70:8f:7f:c8:5f:e8:9e:b5: - 02:31:4d:b3:40:b0:7b:b2:ee:14:a7:69:22:8b:38:85:5d:04: - 6e:d5:44:41:31:a7:4b:71:86:fb:81:cd:3d:db:96:23:0b:bc: - e1:67:46:0e:87:86:91:4e:1a:35:37:af:a4:ac:9a:de:e3:4f: - 82:47:f1:c4:16:58:11:8f:76:d2:4d:df:a1:c6:a2:8f:33:6d: - 72:15:28:76 + b8:fc:d3:8f:8a:e0:6b:74:57:e2:a3:79:b2:18:60:0b:2c:05: + f9:e3:ae:dd:e9:ad:52:88:52:73:b4:12:b0:39:90:65:12:f5: + 95:0e:5f:4b:f2:06:4a:57:ab:e1:f9:b1:34:68:83:d7:d7:5e: + 69:0a:16:44:ea:1d:97:53:51:10:51:8b:ec:0a:b3:c8:a3:3d: + 85:4d:f4:8f:7d:b3:b5:72:e4:9e:d7:f3:01:bf:66:e1:40:92: + 54:63:16:b6:b5:66:ed:30:38:94:1d:1a:8f:28:34:27:ab:c9: + 5f:d5:16:7e:e4:f5:93:d2:19:35:44:0a:c4:2e:6a:25:38:1d: + ee:5a:c8:29:fa:96:dc:95:82:38:9e:36:3a:68:34:7b:4e:d9: + fa:0d:b2:88:a2:6c:4f:03:18:a7:e3:41:67:38:de:e5:f6:ff: + 2a:1c:f0:ec:1a:02:a7:e8:4e:3a:c3:04:72:f8:6a:4f:28:a6: + cf:0b:a2:db:33:74:d1:10:9e:ec:b4:ac:f8:b1:24:f4:ef:0e: + 05:e4:9d:1b:9a:40:f7:09:66:9c:9d:86:8b:76:96:46:e8:d1: + dc:10:c7:7d:0b:69:41:dc:a7:8e:e3:a3:36:e3:42:63:93:8c: + 91:80:0d:27:11:1c:2d:ae:fb:92:88:6c:6b:09:40:1a:30:dd: + 8f:ac:0f:62 -----BEGIN CERTIFICATE----- -MIIDFDCCAfygAwIBAgIUYeYbB5BqT/fNRrlZHT4cOQ3yXgYwDQYJKoZIhvcNAQEL -BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz -MzgyNFowVzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMQ8wDQYDVQQKEwZBcGFj -aGUxFjAUBgNVBAsTDUFwYWNoZSBQdWxzYXIxEjAQBgNVBAMTCXN1cGVyVXNlcjCC -ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAM1DfZhA+bBbvK7bwAutJpCW -4GI47WixcEY73kT5FFGGEOvKkOeI6PmRheDdtbQUuXjjhtVUbWjsFJK0+CJbBT3t -MSVlCAWEyuYMIRJYMscaYKNP0kqeKBl8RYQAjInc3orlT4iRzKTxgUVMfcL/4sGJ -xhJzleI2vduui1poapBR3iuIX6pn9KjjY9y+GYLMnX/mjfuCviIBPVYTO1sEtOjF -GOYuDfq6So3oxlqhUZpKYtev3bT84tXNrplsXGFWC9cMGndc9TpqVLWeM6ypdSia -dq/QelcAG5ETMf1CiCFHBRABL1m7xzrZ4VhMG2xxtpjv3QOCWKMy3JChtqYe4QsC -AwEAAaMeMBwwGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEB -CwUAA4IBAQCQYrp7b0WVenEv54gMZLhsBYZ/RwjO1uJaMhMMgq2nr/Ci94Z5hxqJ -eJWxn77Fizn9EpS24Wn/+h7DgthsA4BFrBwGcLt3w0Fftp3+Nm+uI2y/Q3mOdIWO -lompxG3Z+gW6qBF8gkWUPZ+2fC9ObTfD+3l+DNIV+g7qLckk8zQTb9vXVeEML37+ -TDv6fgMmD2qV0iLOJ3Fql6w2CiDsGaB4IwxU87HdMzZ8t2EjcI9/yF/onrUCMU2z -QLB7su4Up2kiiziFXQRu1URBMadLcYb7gc0925YjC7zhZ0YOh4aRTho1N6+krJre -40+CR/HEFlgRj3bSTd+hxqKPM21yFSh2 +MIIDCTCCAfGgAwIBAgIJANegMncDqPw+MA0GCSqGSIb3DQEBCwUAMBExDzANBgNV +BAMMBkNBUm9vdDAeFw0yMzAyMjIwNjI2MzNaFw0zMzAyMTkwNjI2MzNaMFcxCzAJ +BgNVBAYTAlVTMQswCQYDVQQIEwJDQTEPMA0GA1UEChMGQXBhY2hlMRYwFAYDVQQL +Ew1BcGFjaGUgUHVsc2FyMRIwEAYDVQQDEwlzdXBlclVzZXIwggEiMA0GCSqGSIb3 +DQEBAQUAA4IBDwAwggEKAoIBAQDNQ32YQPmwW7yu28ALrSaQluBiOO1osXBGO95E ++RRRhhDrypDniOj5kYXg3bW0FLl444bVVG1o7BSStPgiWwU97TElZQgFhMrmDCES +WDLHGmCjT9JKnigZfEWEAIyJ3N6K5U+Ikcyk8YFFTH3C/+LBicYSc5XiNr3brota +aGqQUd4riF+qZ/So42PcvhmCzJ1/5o37gr4iAT1WEztbBLToxRjmLg36ukqN6MZa +oVGaSmLXr920/OLVza6ZbFxhVgvXDBp3XPU6alS1njOsqXUomnav0HpXABuREzH9 +QoghRwUQAS9Zu8c62eFYTBtscbaY790DglijMtyQobamHuELAgMBAAGjHjAcMBoG +A1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEAuPzT +j4rga3RX4qN5shhgCywF+eOu3emtUohSc7QSsDmQZRL1lQ5fS/IGSler4fmxNGiD +19deaQoWROodl1NREFGL7AqzyKM9hU30j32ztXLkntfzAb9m4UCSVGMWtrVm7TA4 +lB0ajyg0J6vJX9UWfuT1k9IZNUQKxC5qJTgd7lrIKfqW3JWCOJ42Omg0e07Z+g2y +iKJsTwMYp+NBZzje5fb/Khzw7BoCp+hOOsMEcvhqTyimzwui2zN00RCe7LSs+LEk +9O8OBeSdG5pA9wlmnJ2Gi3aWRujR3BDHfQtpQdynjuOjNuNCY5OMkYANJxEcLa77 +kohsawlAGjDdj6wPYg== -----END CERTIFICATE----- diff --git a/build/regenerate_certs_for_tests.sh b/build/regenerate_certs_for_tests.sh index fb0274cc19316..fff1c057060f3 100755 --- a/build/regenerate_certs_for_tests.sh +++ b/build/regenerate_certs_for_tests.sh @@ -34,7 +34,16 @@ function reissue_certificate() { keyfile=$1 certfile=$2 openssl x509 -x509toreq -in $certfile -signkey $keyfile -out ${certfile}.csr - openssl x509 -req -CA ca-cert.pem -CAkey ca-key -in ${certfile}.csr -text -outform pem -out $certfile -days 3650 -CAcreateserial -extfile <(printf "subjectAltName = DNS:localhost, IP:127.0.0.1") + openssl x509 -req -CA ca-cert.pem -CAkey ca-key -in ${certfile}.csr -text -outform pem -days 3650 -sha256 -CAcreateserial -extfile <(printf "subjectAltName = DNS:localhost, IP:127.0.0.1") > $certfile + rm ${certfile}.csr +} + +function reissue_certificate_no_subject() { + keyfile=$1 + certfile=$2 + openssl x509 -x509toreq -in $certfile -signkey $keyfile -out ${certfile}.csr + openssl x509 -req -CA ca-cert.pem -CAkey ca-key -in ${certfile}.csr -text -outform pem -days 3650 -sha256 -CAcreateserial > $certfile + rm ${certfile}.csr } generate_ca @@ -54,6 +63,11 @@ cp ca-cert.pem $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/Prox reissue_certificate $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-key.pem \ $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cert.pem +# Use $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/cacert.pem as trusted cert +reissue_certificate_no_subject \ + $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/no-subject-alt-key.pem \ + $ROOT_DIR/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/no-subject-alt-cert.pem + generate_ca cp ca-cert.pem $ROOT_DIR/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/cacert.pem reissue_certificate $ROOT_DIR/bouncy-castle/bcfips-include-test/src/test/resources/authentication/tls/broker-key.pem \ diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationTest.java index e2362478e4782..de9bb087d3da3 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationTest.java @@ -68,18 +68,20 @@ public class ProxyWithAuthorizationTest extends ProducerConsumerBase { private final SecretKey SECRET_KEY = AuthTokenUtils.createSecretKey(SignatureAlgorithm.HS256); private final String CLIENT_TOKEN = AuthTokenUtils.createToken(SECRET_KEY, "Client", Optional.empty()); - private final String TLS_PROXY_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cacert.pem"; + // The Proxy, Client, and SuperUser Client certs are signed by this CA + private final String TLS_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem"; + + // Proxy and Broker use valid certs that have no Subject Alternative Name to test hostname verification correctly + // fails a connection to an invalid host. + private final String TLS_NO_SUBJECT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/ProxyWithAuthorizationTest/no-subject-alt-cert.pem"; + private final String TLS_NO_SUBJECT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/ProxyWithAuthorizationTest/no-subject-alt-key.pem"; private final String TLS_PROXY_CERT_FILE_PATH = "./src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cert.pem"; private final String TLS_PROXY_KEY_FILE_PATH = "./src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-key.pem"; - private final String TLS_BROKER_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-cacert.pem"; - private final String TLS_BROKER_CERT_FILE_PATH = "./src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-cert.pem"; - private final String TLS_BROKER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-key.pem"; private final String TLS_CLIENT_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-cacert.pem"; private final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-cert.pem"; private final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-key.pem"; private final String TLS_SUPERUSER_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem"; private final String TLS_SUPERUSER_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem"; - private final String TLS_SUPERUSER_CLIENT_TRUST_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem"; private ProxyService proxyService; private final ProxyConfiguration proxyConfig = new ProxyConfiguration(); @@ -166,9 +168,9 @@ protected void doInitConf() throws Exception { conf.setBrokerServicePort(Optional.empty()); conf.setWebServicePortTls(Optional.of(0)); conf.setWebServicePort(Optional.empty()); - conf.setTlsTrustCertsFilePath(TLS_PROXY_TRUST_CERT_FILE_PATH); - conf.setTlsCertificateFilePath(TLS_BROKER_CERT_FILE_PATH); - conf.setTlsKeyFilePath(TLS_BROKER_KEY_FILE_PATH); + conf.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); + conf.setTlsCertificateFilePath(TLS_NO_SUBJECT_CERT_FILE_PATH); + conf.setTlsKeyFilePath(TLS_NO_SUBJECT_KEY_FILE_PATH); conf.setTlsAllowInsecureConnection(false); Set superUserRoles = new HashSet<>(); @@ -177,8 +179,8 @@ protected void doInitConf() throws Exception { conf.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName()); conf.setBrokerClientAuthenticationParameters( - "tlsCertFile:" + TLS_BROKER_CERT_FILE_PATH + "," + "tlsKeyFile:" + TLS_BROKER_KEY_FILE_PATH); - conf.setBrokerClientTrustCertsFilePath(TLS_BROKER_TRUST_CERT_FILE_PATH); + "tlsCertFile:" + TLS_SUPERUSER_CLIENT_CERT_FILE_PATH + "," + "tlsKeyFile:" + TLS_SUPERUSER_CLIENT_KEY_FILE_PATH); + conf.setBrokerClientTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); conf.setAuthenticationProviders(Set.of(AuthenticationProviderTls.class.getName(), AuthenticationProviderToken.class.getName())); Properties properties = new Properties(); @@ -210,10 +212,10 @@ protected void setup() throws Exception { proxyConfig.setTlsEnabledWithBroker(true); // enable tls and auth&auth at proxy - proxyConfig.setTlsCertificateFilePath(TLS_PROXY_CERT_FILE_PATH); - proxyConfig.setTlsKeyFilePath(TLS_PROXY_KEY_FILE_PATH); - proxyConfig.setTlsTrustCertsFilePath(TLS_CLIENT_TRUST_CERT_FILE_PATH); - proxyConfig.setBrokerClientTrustCertsFilePath(TLS_BROKER_TRUST_CERT_FILE_PATH); + proxyConfig.setTlsCertificateFilePath(TLS_NO_SUBJECT_CERT_FILE_PATH); + proxyConfig.setTlsKeyFilePath(TLS_NO_SUBJECT_KEY_FILE_PATH); + proxyConfig.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); + proxyConfig.setBrokerClientTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); proxyConfig.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName()); proxyConfig.setBrokerClientAuthenticationParameters( "tlsCertFile:" + TLS_PROXY_CERT_FILE_PATH + "," + "tlsKeyFile:" + TLS_PROXY_KEY_FILE_PATH); @@ -446,7 +448,7 @@ public void tlsCiphersAndProtocols(Set tlsCiphers, Set tlsProtoc proxyConfig.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName()); proxyConfig.setBrokerClientAuthenticationParameters( "tlsCertFile:" + TLS_PROXY_CERT_FILE_PATH + "," + "tlsKeyFile:" + TLS_PROXY_KEY_FILE_PATH); - proxyConfig.setBrokerClientTrustCertsFilePath(TLS_BROKER_TRUST_CERT_FILE_PATH); + proxyConfig.setBrokerClientTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); Set providers = new HashSet<>(); providers.add(AuthenticationProviderTls.class.getName()); conf.setAuthenticationProviders(providers); @@ -517,7 +519,7 @@ public void testProxyTlsTransportWithAuth(Authentication auth) throws Exception .authentication(auth) .tlsKeyFilePath(TLS_CLIENT_KEY_FILE_PATH) .tlsCertificateFilePath(TLS_CLIENT_CERT_FILE_PATH) - .tlsTrustCertsFilePath(TLS_PROXY_TRUST_CERT_FILE_PATH) + .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH) .operationTimeout(1000, TimeUnit.MILLISECONDS) .build(); @@ -571,7 +573,7 @@ private void createAdminClient() throws Exception { authParams.put("tlsKeyFile", TLS_SUPERUSER_CLIENT_KEY_FILE_PATH); admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrlTls.toString()) - .tlsTrustCertsFilePath(TLS_BROKER_TRUST_CERT_FILE_PATH) + .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH) .authentication(AuthenticationTls.class.getName(), authParams).build()); } @@ -585,7 +587,7 @@ private PulsarClient createPulsarClient(String proxyServiceUrl, ClientBuilder cl authTls.configure(authParams); return clientBuilder.serviceUrl(proxyServiceUrl).statsInterval(0, TimeUnit.SECONDS) - .tlsTrustCertsFilePath(TLS_PROXY_TRUST_CERT_FILE_PATH) + .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH) .authentication(authTls).enableTls(true) .operationTimeout(1000, TimeUnit.MILLISECONDS).build(); } diff --git a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-cacert.pem b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-cacert.pem index 7d2d58d8d7a06..89de977601909 100644 --- a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-cacert.pem +++ b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-cacert.pem @@ -1,77 +1,78 @@ Certificate: Data: Version: 3 (0x2) - Serial Number: - 40:cd:a5:a5:35:76:ee:02:57:8b:30:8f:2a:12:34:03:45:c5:96:8c - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN = CARoot + Serial Number: 15670345994378439095 (0xd97840a8266469b7) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=CARoot Validity - Not Before: May 30 13:38:24 2022 GMT - Not After : May 27 13:38:24 2032 GMT - Subject: CN = CARoot + Not Before: Feb 22 06:26:33 2023 GMT + Not After : Feb 19 06:26:33 2033 GMT + Subject: CN=CARoot Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (2048 bit) + Public-Key: (2048 bit) Modulus: - 00:d8:d5:00:e0:6b:4f:4e:8a:67:08:e9:e3:3f:23: - ef:15:1d:82:10:85:f3:3b:77:9c:96:c1:aa:eb:90: - 41:0b:5b:ae:77:d9:a3:f1:cf:2a:32:40:78:33:6a: - 81:b9:c2:cd:91:36:98:df:41:84:c0:62:8a:a1:03: - 89:8d:2b:b8:91:49:a9:e8:a2:90:ad:b9:cd:23:84: - bc:60:1f:6f:b5:81:9f:9c:cf:d5:26:a8:a5:b6:4d: - 59:5f:5c:7f:da:e8:1d:3d:04:f3:b8:ef:f8:d5:73: - c6:fd:6a:b1:91:ae:16:b7:45:21:9a:1a:1a:76:74: - 01:40:ee:fc:3c:67:be:6a:7f:f4:a3:82:37:ee:43: - 41:f5:67:d5:d5:64:9c:d8:53:75:34:4d:23:80:b5: - 59:13:c2:27:47:8e:20:32:6f:f6:b3:70:bf:5e:15: - 08:7e:d1:bf:aa:4d:06:6b:0d:17:21:eb:95:47:52: - fa:d7:97:ef:1a:5d:63:26:17:36:01:20:ac:57:50: - 34:f0:57:49:38:3d:9c:68:6a:87:91:38:b6:76:9d: - bc:e9:4e:c2:58:54:8d:8a:32:05:9e:ba:cb:f0:d0: - ec:91:67:1d:77:bf:d5:02:77:d4:22:78:94:f4:9a: - 49:fa:ef:b2:9b:30:1a:8a:f0:a7:9a:2b:e5:e9:c7: - 36:c5 + 00:b2:9a:e4:e5:d4:2e:90:21:62:99:07:8a:dd:94: + 92:6a:f7:e9:b7:b5:b4:85:7e:53:04:ff:fa:72:2c: + 77:1b:23:08:c8:91:ff:28:54:67:78:12:40:fc:9e: + bd:be:56:95:8c:c0:97:9f:54:b8:03:06:f3:83:f5: + 14:af:f7:63:1f:51:b9:81:94:08:69:f8:73:ac:1a: + 9a:dc:9b:79:e4:61:36:86:54:5e:b0:4c:5d:6f:6e: + 0f:06:a3:7c:ab:10:43:01:4d:29:21:62:af:dd:b1: + f4:3f:4d:52:39:98:de:09:5b:68:fd:41:2f:00:f2: + 22:94:69:cf:e2:2a:0b:2a:67:29:31:24:f4:77:36: + b9:18:31:97:e6:2a:96:a2:eb:f2:24:c1:fd:89:1a: + f7:51:67:3e:cf:cc:6b:9b:93:3f:9a:19:9b:f2:e4: + b4:cf:b3:99:47:fb:2f:1f:50:e1:de:90:a5:e4:4c: + da:d6:7d:e6:8c:0d:77:84:6c:87:88:99:27:a4:a8: + 9a:7d:58:ac:78:32:0f:6e:8e:0d:2f:78:0d:51:20: + ae:c1:67:2c:f5:25:7a:dd:98:1c:aa:75:3a:f7:87: + 97:a4:38:b9:96:5c:91:47:30:b0:a7:fd:6e:9e:59: + e4:01:5a:e6:e6:b7:f4:01:21:20:2f:9b:54:05:2f: + 46:45 Exponent: 65537 (0x10001) X509v3 extensions: + X509v3 Basic Constraints: critical + CA:TRUE X509v3 Subject Key Identifier: - DD:AC:A0:40:6E:E9:2B:49:F2:35:DB:B4:E9:98:AD:58:7B:37:6B:55 + 03:72:4A:D9:37:06:FB:B5:C2:04:CF:0B:BF:98:07:FA:C7:6A:85:CE X509v3 Authority Key Identifier: - keyid:DD:AC:A0:40:6E:E9:2B:49:F2:35:DB:B4:E9:98:AD:58:7B:37:6B:55 + keyid:03:72:4A:D9:37:06:FB:B5:C2:04:CF:0B:BF:98:07:FA:C7:6A:85:CE + DirName:/CN=CARoot + serial:D9:78:40:A8:26:64:69:B7 - X509v3 Basic Constraints: critical - CA:TRUE Signature Algorithm: sha256WithRSAEncryption - 07:0c:90:05:fa:2c:c9:4e:05:ec:6b:7d:99:9c:52:2a:20:34: - 46:ac:8d:24:81:f9:a7:f3:1d:03:32:45:82:9a:61:af:1f:63: - 25:6b:97:ca:93:78:e5:d7:87:81:b6:29:22:d4:0d:8d:ed:0e: - bd:85:80:6c:38:e9:86:3c:bd:ee:ff:26:78:0a:f0:a7:54:0b: - af:27:9e:8b:83:b7:10:e9:44:0d:4a:7e:a8:e2:aa:1c:06:f8: - 18:f1:c4:c9:e4:bb:17:41:59:94:b4:dc:78:53:fb:1b:43:57: - 82:59:de:6c:03:52:9a:28:cb:e4:9e:ea:c5:00:93:e0:27:b4: - 4b:e6:b3:c5:88:2d:14:33:10:ff:b0:23:4e:5d:ea:17:97:7d: - f4:e2:c8:fe:c3:4a:77:83:64:ef:c9:b6:3e:77:64:32:07:91: - bd:e1:58:9a:e1:38:ab:eb:d2:e3:cb:05:7c:c7:f3:2b:47:bf: - 36:64:7e:32:5a:62:44:07:c8:8e:9d:55:1a:99:c4:14:5a:66: - ed:5f:8b:ab:dd:eb:36:28:cd:77:47:84:00:ae:a7:34:0e:0d: - 77:df:67:72:08:94:75:52:1b:4a:71:4d:31:5d:aa:1b:aa:b6: - e0:d6:86:52:7c:26:ae:1f:96:ab:06:32:cb:7a:f3:bb:76:3e: - 08:53:9f:64 + 8f:f3:3c:19:a8:82:c9:44:e0:2f:b2:dd:1c:b5:3c:9d:77:2b: + 05:fc:e3:e1:a4:95:3b:c5:7e:d9:c0:c7:51:c5:70:75:f8:e2: + 49:43:8e:78:74:dd:1d:7e:c1:9a:46:12:bd:25:24:59:e4:cd: + 54:3d:1e:b7:93:4f:dc:9b:3c:10:4b:c6:83:b6:cd:a8:36:20: + 79:7e:b7:8c:76:e0:b0:fe:6e:df:2a:8f:97:f8:36:b2:b7:1f: + 8b:7a:60:58:24:46:fe:ba:d7:f1:5b:69:14:53:09:3c:75:72: + ed:ae:10:98:a3:89:bf:0d:5d:16:2e:31:27:90:3c:61:ff:90: + de:cb:68:f9:30:c1:2f:65:a0:93:c3:e2:d0:fc:ca:f2:01:54: + 5c:f8:6e:fc:10:8b:04:c7:0e:4c:81:d7:8e:b0:16:fd:f7:5b: + 4f:fb:12:18:3b:e5:58:61:13:ce:d6:21:33:f7:43:3e:50:26: + b8:ae:37:18:1f:82:ba:76:14:ee:6b:7b:87:67:95:cc:44:55: + b2:8b:aa:af:9f:b5:78:d0:7f:de:f3:7c:91:27:88:95:b5:a6: + 10:05:40:82:57:a7:0e:f4:99:70:c2:e7:af:ea:f2:47:52:84: + 01:78:c0:56:f7:e2:bf:f9:49:b8:1c:ba:4d:e1:2d:f4:28:71: + 78:ae:ac:89 -----BEGIN CERTIFICATE----- -MIIDAzCCAeugAwIBAgIUQM2lpTV27gJXizCPKhI0A0XFlowwDQYJKoZIhvcNAQEL -BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz -MzgyNFowETEPMA0GA1UEAwwGQ0FSb290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A -MIIBCgKCAQEA2NUA4GtPTopnCOnjPyPvFR2CEIXzO3eclsGq65BBC1uud9mj8c8q -MkB4M2qBucLNkTaY30GEwGKKoQOJjSu4kUmp6KKQrbnNI4S8YB9vtYGfnM/VJqil -tk1ZX1x/2ugdPQTzuO/41XPG/Wqxka4Wt0UhmhoadnQBQO78PGe+an/0o4I37kNB -9WfV1WSc2FN1NE0jgLVZE8InR44gMm/2s3C/XhUIftG/qk0Gaw0XIeuVR1L615fv -Gl1jJhc2ASCsV1A08FdJOD2caGqHkTi2dp286U7CWFSNijIFnrrL8NDskWcdd7/V -AnfUIniU9JpJ+u+ymzAaivCnmivl6cc2xQIDAQABo1MwUTAdBgNVHQ4EFgQU3ayg -QG7pK0nyNdu06ZitWHs3a1UwHwYDVR0jBBgwFoAU3aygQG7pK0nyNdu06ZitWHs3 -a1UwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEABwyQBfosyU4F -7Gt9mZxSKiA0RqyNJIH5p/MdAzJFgpphrx9jJWuXypN45deHgbYpItQNje0OvYWA -bDjphjy97v8meArwp1QLryeei4O3EOlEDUp+qOKqHAb4GPHEyeS7F0FZlLTceFP7 -G0NXglnebANSmijL5J7qxQCT4Ce0S+azxYgtFDMQ/7AjTl3qF5d99OLI/sNKd4Nk -78m2PndkMgeRveFYmuE4q+vS48sFfMfzK0e/NmR+MlpiRAfIjp1VGpnEFFpm7V+L -q93rNijNd0eEAK6nNA4Nd99ncgiUdVIbSnFNMV2qG6q24NaGUnwmrh+WqwYyy3rz -u3Y+CFOfZA== +MIIDGjCCAgKgAwIBAgIJANl4QKgmZGm3MA0GCSqGSIb3DQEBCwUAMBExDzANBgNV +BAMMBkNBUm9vdDAeFw0yMzAyMjIwNjI2MzNaFw0zMzAyMTkwNjI2MzNaMBExDzAN +BgNVBAMMBkNBUm9vdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALKa +5OXULpAhYpkHit2Ukmr36be1tIV+UwT/+nIsdxsjCMiR/yhUZ3gSQPyevb5WlYzA +l59UuAMG84P1FK/3Yx9RuYGUCGn4c6wamtybeeRhNoZUXrBMXW9uDwajfKsQQwFN +KSFir92x9D9NUjmY3glbaP1BLwDyIpRpz+IqCypnKTEk9Hc2uRgxl+YqlqLr8iTB +/Yka91FnPs/Ma5uTP5oZm/LktM+zmUf7Lx9Q4d6QpeRM2tZ95owNd4Rsh4iZJ6So +mn1YrHgyD26ODS94DVEgrsFnLPUlet2YHKp1OveHl6Q4uZZckUcwsKf9bp5Z5AFa +5ua39AEhIC+bVAUvRkUCAwEAAaN1MHMwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E +FgQUA3JK2TcG+7XCBM8Lv5gH+sdqhc4wQQYDVR0jBDowOIAUA3JK2TcG+7XCBM8L +v5gH+sdqhc6hFaQTMBExDzANBgNVBAMMBkNBUm9vdIIJANl4QKgmZGm3MA0GCSqG +SIb3DQEBCwUAA4IBAQCP8zwZqILJROAvst0ctTyddysF/OPhpJU7xX7ZwMdRxXB1 ++OJJQ454dN0dfsGaRhK9JSRZ5M1UPR63k0/cmzwQS8aDts2oNiB5freMduCw/m7f +Ko+X+Daytx+LemBYJEb+utfxW2kUUwk8dXLtrhCYo4m/DV0WLjEnkDxh/5Dey2j5 +MMEvZaCTw+LQ/MryAVRc+G78EIsExw5MgdeOsBb991tP+xIYO+VYYRPO1iEz90M+ +UCa4rjcYH4K6dhTua3uHZ5XMRFWyi6qvn7V40H/e83yRJ4iVtaYQBUCCV6cO9Jlw +wuev6vJHUoQBeMBW9+K/+Um4HLpN4S30KHF4rqyJ -----END CERTIFICATE----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-cert.pem b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-cert.pem index 31743d0684670..8236c0a606d2a 100644 --- a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-cert.pem +++ b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/broker-cert.pem @@ -1,17 +1,16 @@ Certificate: Data: Version: 3 (0x2) - Serial Number: - 61:e6:1b:07:90:6a:4f:f7:cd:46:b9:59:1d:3e:1c:39:0d:f2:5e:07 - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN = CARoot + Serial Number: 15537474201172114495 (0xd7a0327703a8fc3f) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=CARoot Validity - Not Before: May 30 13:38:24 2022 GMT - Not After : May 27 13:38:24 2032 GMT - Subject: C = US, ST = CA, O = Apache Pulsar, OU = Broker, CN = Broker + Not Before: Feb 22 06:26:33 2023 GMT + Not After : Feb 19 06:26:33 2033 GMT + Subject: C=US, ST=CA, O=Apache Pulsar, OU=Broker, CN=Broker Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (2048 bit) + Public-Key: (2048 bit) Modulus: 00:ca:77:dc:2a:13:25:24:cb:29:62:06:12:5f:a8: 92:c9:53:d6:3f:07:ca:aa:0a:5f:72:92:cd:b7:ea: @@ -36,37 +35,37 @@ Certificate: X509v3 Subject Alternative Name: DNS:localhost, IP Address:127.0.0.1 Signature Algorithm: sha256WithRSAEncryption - 8d:1d:69:d2:44:1f:af:68:30:80:c1:91:b2:2f:9a:7e:ca:ff: - 38:46:8e:28:59:02:2d:e7:74:c4:3c:b3:ac:b3:22:53:e9:54: - 3a:e2:4d:4d:65:63:47:dd:38:86:ec:d1:7d:4f:fe:5d:c6:c8: - c8:10:b8:33:5a:4d:9e:83:e3:92:97:c5:f1:d8:e3:97:6d:01: - 50:03:de:25:d8:e4:de:62:70:b8:c4:55:5b:9f:8c:61:b8:d7: - f0:8f:6c:2d:80:cc:b8:7b:8b:b4:54:9a:d6:e1:f9:7f:52:99: - 7b:ef:23:88:61:e5:7c:85:5c:57:98:cc:a6:98:4b:71:84:5c: - ab:5e:82:48:5a:da:5f:d6:84:b5:52:43:df:3c:0f:95:06:29: - 00:94:f8:98:94:6d:1c:c8:76:21:7a:2f:61:34:ab:bd:27:59: - d1:41:99:91:69:68:f7:b6:65:21:e8:9a:b1:9b:ac:72:12:17: - 54:0b:56:08:bd:9d:6b:0e:35:4a:f8:97:b6:83:00:55:96:0c: - 66:13:06:c9:27:5f:cc:d0:81:4b:3e:6e:d2:85:cd:79:7a:8c: - a0:1e:d8:9b:e4:da:e9:ba:51:f1:29:0f:69:00:df:24:a0:55: - 5e:cd:d0:84:c9:4a:a8:b4:12:33:29:6f:8a:8c:d7:a1:b4:8b: - 4a:7d:a2:30 + 0a:35:f9:91:0b:0a:47:88:0e:86:b4:c7:b4:86:9c:b5:6a:e5: + 68:dc:38:f3:5d:f9:ae:15:1c:d9:7a:6a:09:e4:03:f4:d8:71: + 62:1d:c7:e7:ba:0e:d1:00:1a:66:8b:9d:97:6e:b3:c3:99:74: + ad:bf:a9:ab:99:a5:2d:76:d0:87:c5:f5:6e:cd:c3:ef:73:7e: + 23:13:2c:bf:b3:f4:31:93:c2:e9:25:8b:20:de:a7:9b:8a:48: + 32:5f:80:f5:e1:01:4f:14:99:f4:7e:55:62:f9:78:15:18:fa: + 76:a5:0c:88:e5:3d:8a:bf:0f:65:2a:5f:13:5b:c6:03:24:2e: + f6:be:1b:6b:53:f9:93:c2:eb:b6:ee:9d:85:a5:4a:5d:cc:79: + 43:57:9c:47:2b:fc:67:38:de:1d:d5:a3:6a:40:61:df:7e:49: + a8:e0:be:f8:62:dc:b2:86:1f:23:e9:2d:db:0d:8b:4f:e5:05: + 6d:64:6f:11:43:7d:39:e6:68:8f:ee:0a:96:e4:d1:c3:6b:c0: + 55:d7:eb:dc:1c:66:fa:28:d5:1f:92:4d:bb:1c:43:f9:b2:f8: + 4c:36:16:44:58:27:83:32:94:9f:64:d6:bd:f8:d3:fe:c9:e7: + 9d:7b:93:f4:b3:16:61:ad:ff:c3:f3:5d:d3:7b:dc:40:ea:a9: + d1:3d:a7:f5 -----BEGIN CERTIFICATE----- -MIIDETCCAfmgAwIBAgIUYeYbB5BqT/fNRrlZHT4cOQ3yXgcwDQYJKoZIhvcNAQEL -BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz -MzgyNFowVDELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMRYwFAYDVQQKEw1BcGFj -aGUgUHVsc2FyMQ8wDQYDVQQLEwZCcm9rZXIxDzANBgNVBAMTBkJyb2tlcjCCASIw -DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMp33CoTJSTLKWIGEl+okslT1j8H -yqoKX3KSzbfqRUdx8GNPWBo9+s6mc5DAqfcl8HZ17bIDF77YilbzT2pMfgNlleVF -641H6GBenjh0UFRloOzYXGVgNBuWg31x1F1/42JZZ+jw1iR9wG43A1RMPQwzOZsz -4VJExUPa6u4s8xwWLkZMfJ9dTW7+jCOe936fOcFxBlL0Jpoi1M/FJTmp0uQkxthK -SKLudiXLPPC/zRB3/4ERQyHMO8wQegeE/MwCokXekS1r0e0XGtBG9K59s4n4MXeV -5UaxqTHW2ONHALKBgduKHNnxzeNNNfY4kQ3qB/CwBk8sTHXCN/81DbFCBgsCAwEA -AaMeMBwwGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEBCwUA -A4IBAQCNHWnSRB+vaDCAwZGyL5p+yv84Ro4oWQIt53TEPLOssyJT6VQ64k1NZWNH -3TiG7NF9T/5dxsjIELgzWk2eg+OSl8Xx2OOXbQFQA94l2OTeYnC4xFVbn4xhuNfw -j2wtgMy4e4u0VJrW4fl/Upl77yOIYeV8hVxXmMymmEtxhFyrXoJIWtpf1oS1UkPf -PA+VBikAlPiYlG0cyHYhei9hNKu9J1nRQZmRaWj3tmUh6Jqxm6xyEhdUC1YIvZ1r -DjVK+Je2gwBVlgxmEwbJJ1/M0IFLPm7Shc15eoygHtib5NrpulHxKQ9pAN8koFVe -zdCEyUqotBIzKW+KjNehtItKfaIw +MIIDBjCCAe6gAwIBAgIJANegMncDqPw/MA0GCSqGSIb3DQEBCwUAMBExDzANBgNV +BAMMBkNBUm9vdDAeFw0yMzAyMjIwNjI2MzNaFw0zMzAyMTkwNjI2MzNaMFQxCzAJ +BgNVBAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEChMNQXBhY2hlIFB1bHNhcjEP +MA0GA1UECxMGQnJva2VyMQ8wDQYDVQQDEwZCcm9rZXIwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQDKd9wqEyUkyyliBhJfqJLJU9Y/B8qqCl9yks236kVH +cfBjT1gaPfrOpnOQwKn3JfB2de2yAxe+2IpW809qTH4DZZXlReuNR+hgXp44dFBU +ZaDs2FxlYDQbloN9cdRdf+NiWWfo8NYkfcBuNwNUTD0MMzmbM+FSRMVD2uruLPMc +Fi5GTHyfXU1u/owjnvd+nznBcQZS9CaaItTPxSU5qdLkJMbYSkii7nYlyzzwv80Q +d/+BEUMhzDvMEHoHhPzMAqJF3pEta9HtFxrQRvSufbOJ+DF3leVGsakx1tjjRwCy +gYHbihzZ8c3jTTX2OJEN6gfwsAZPLEx1wjf/NQ2xQgYLAgMBAAGjHjAcMBoGA1Ud +EQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEACjX5kQsK +R4gOhrTHtIactWrlaNw48135rhUc2XpqCeQD9NhxYh3H57oO0QAaZoudl26zw5l0 +rb+pq5mlLXbQh8X1bs3D73N+IxMsv7P0MZPC6SWLIN6nm4pIMl+A9eEBTxSZ9H5V +Yvl4FRj6dqUMiOU9ir8PZSpfE1vGAyQu9r4ba1P5k8Lrtu6dhaVKXcx5Q1ecRyv8 +ZzjeHdWjakBh335JqOC++GLcsoYfI+kt2w2LT+UFbWRvEUN9OeZoj+4KluTRw2vA +Vdfr3Bxm+ijVH5JNuxxD+bL4TDYWRFgngzKUn2TWvfjT/snnnXuT9LMWYa3/w/Nd +03vcQOqp0T2n9Q== -----END CERTIFICATE----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-cacert.pem b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-cacert.pem index 127f56dd777a5..2a71f0e3afc17 100644 --- a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-cacert.pem +++ b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-cacert.pem @@ -1,77 +1,78 @@ Certificate: Data: Version: 3 (0x2) - Serial Number: - 77:4f:f6:cf:99:ca:77:e8:a7:6e:1e:fd:e2:cf:ac:a9:da:68:d2:42 - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN = CARoot + Serial Number: 15559223195710621847 (0xd7ed770f69598897) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=CARoot Validity - Not Before: May 30 13:38:24 2022 GMT - Not After : May 27 13:38:24 2032 GMT - Subject: CN = CARoot + Not Before: Feb 22 06:26:32 2023 GMT + Not After : Feb 19 06:26:32 2033 GMT + Subject: CN=CARoot Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (2048 bit) + Public-Key: (2048 bit) Modulus: - 00:b8:5e:c2:60:ed:c4:ee:3c:5b:ab:fc:64:52:f3: - 30:41:fc:10:5a:ac:a6:9b:0a:93:d0:d0:c9:bf:96: - 14:a7:cf:5c:3e:23:91:7e:54:ec:fe:2d:9f:c9:34: - d1:4e:95:2f:85:9c:cc:be:90:a3:a4:cb:4d:a4:72: - d2:84:e0:c7:42:c4:bf:70:b6:fa:d2:45:8b:83:66: - 1e:a4:e9:0e:06:a3:46:ea:a7:18:cd:33:b9:f1:ff: - 76:91:72:8f:cd:f9:93:43:c3:6e:17:1f:2d:86:df: - b6:fb:2d:d6:be:2d:98:ad:de:00:c7:de:f9:68:b5: - 40:40:56:49:ae:23:e5:a1:3b:5f:15:5a:44:50:da: - fb:02:d3:42:c6:87:0d:c0:8d:3a:e6:e2:aa:73:31: - ab:79:58:51:cd:03:80:f3:12:ce:2f:35:04:8b:39: - 5f:b0:cc:b8:41:99:47:c1:17:96:8b:c2:44:84:b5: - 21:8a:15:52:fe:1a:5a:f9:88:cc:11:17:ee:48:dd: - ba:bf:ed:67:6e:27:35:42:cf:07:5e:b1:8b:81:55: - 92:01:8e:61:fd:8e:82:74:b1:70:7a:3d:52:1f:16: - 78:12:bb:b5:09:62:ce:6d:18:4a:e9:f5:27:19:bc: - 93:4e:ed:dd:53:a8:c1:bb:48:b7:18:20:7b:79:48: - 48:9d + 00:bf:3e:e6:db:fd:a2:6c:b9:45:29:e8:d3:90:4d: + 78:29:ef:fc:67:c4:e0:ae:06:fc:f8:2b:cb:3b:4b: + 89:cf:37:3c:ab:86:94:59:c3:50:54:4b:18:d5:8b: + 14:f5:cd:36:58:c6:ac:a1:67:f7:04:58:58:2f:e0: + 89:73:8b:b1:ef:97:a4:16:10:97:e6:6f:2f:18:b9: + 8c:93:7b:7c:5b:4f:8d:09:49:aa:70:59:e5:3b:fa: + c0:b9:4a:ed:14:98:0a:5f:56:b3:49:0a:4d:c0:22: + 1e:75:3d:ba:f9:19:da:68:80:18:ad:b7:8f:de:fd: + 1f:60:33:86:74:46:e0:b7:7a:84:5e:b7:af:5b:57: + 3c:93:ad:37:83:2c:1b:e0:77:a3:84:da:25:1d:16: + 77:3f:25:b1:90:49:28:a6:c8:cf:bc:e4:b9:27:85: + f9:36:4f:47:81:cd:56:26:41:23:10:8f:36:26:ce: + 78:b9:ab:45:ce:9c:eb:a3:2a:11:93:ae:b7:d4:d7: + 57:e9:97:71:5b:fa:ca:0e:71:34:43:87:bb:c0:8e: + 68:f4:4d:c8:64:45:02:5d:81:bd:3b:47:bc:ec:4e: + e4:61:e3:c4:16:f0:ef:83:fd:06:5c:05:50:d8:50: + 41:dc:d6:62:6d:b2:26:7b:d3:6b:f6:da:59:4c:a8: + db:b5 Exponent: 65537 (0x10001) X509v3 extensions: + X509v3 Basic Constraints: critical + CA:TRUE X509v3 Subject Key Identifier: - 0F:46:61:3E:6F:71:22:E6:1F:32:37:7C:B2:81:A6:CC:DB:9D:F5:7C + F9:B2:AD:C4:24:62:47:3C:1A:58:AA:66:D0:91:12:F3:20:EE:A9:6C X509v3 Authority Key Identifier: - keyid:0F:46:61:3E:6F:71:22:E6:1F:32:37:7C:B2:81:A6:CC:DB:9D:F5:7C + keyid:F9:B2:AD:C4:24:62:47:3C:1A:58:AA:66:D0:91:12:F3:20:EE:A9:6C + DirName:/CN=CARoot + serial:D7:ED:77:0F:69:59:88:97 - X509v3 Basic Constraints: critical - CA:TRUE Signature Algorithm: sha256WithRSAEncryption - 91:e8:d8:c4:32:2e:80:5c:d4:cb:24:7a:81:43:a9:c7:95:90: - 1a:2e:7a:d3:0c:5d:b6:21:05:67:4d:98:5a:0d:71:ea:80:01: - 95:42:fe:fa:f1:7c:dc:bd:76:ff:05:26:3b:f0:94:b3:09:2c: - 34:dd:43:56:46:2b:15:35:99:d9:94:54:22:cf:a6:68:b0:d1: - 79:e2:f0:9f:0b:02:7c:cf:1f:bd:d0:f6:49:c6:82:28:a5:c6: - ae:94:65:cf:fd:ad:a8:6c:c2:17:da:db:f3:be:30:1a:1b:b4: - 2c:fa:08:71:9d:64:09:45:02:92:02:ad:eb:15:47:14:43:5b: - a8:2d:1a:ec:14:93:dc:ff:bb:51:33:a3:d5:4d:e2:77:ca:e1: - a5:98:5c:7a:b6:10:19:d3:d7:f5:14:a5:d5:08:f1:97:18:3d: - 5f:a6:4e:a2:4a:0d:4b:d4:bb:56:6b:a8:44:35:62:c5:d8:c6: - 67:11:93:1c:22:64:3e:aa:15:08:dc:87:39:dd:f6:e0:a0:d5: - 00:db:27:79:3d:f4:35:7c:46:a9:fa:0c:fa:fc:74:f5:bf:f4: - fe:71:40:45:33:22:35:83:f7:1a:96:2a:fc:b2:33:e0:1a:e8: - 24:48:91:5d:90:5c:4c:93:33:4c:40:de:26:bb:24:ac:48:9b: - ae:fe:19:34 + bc:24:05:65:56:87:f9:87:f4:1b:5a:a3:a0:01:c6:58:c0:7c: + ca:c9:ee:1d:0c:d5:6b:47:28:dc:bd:2f:f1:73:03:e1:a5:08: + 05:56:11:29:d5:48:32:23:d3:7d:46:db:20:58:29:78:19:af: + 6a:a2:d1:b6:6e:30:a0:a3:b1:88:c1:b5:1f:f0:0f:63:5e:ab: + 85:5f:09:0a:3c:20:cb:61:3b:91:80:70:4e:1f:c5:37:34:0d: + 1c:9f:b5:de:93:f1:ae:94:ff:7c:76:7d:6a:a7:19:6f:22:34: + bd:66:07:28:59:f3:60:ef:39:8d:bd:9d:2b:82:08:f0:68:aa: + d6:e0:68:f1:f1:d2:c6:c6:61:88:0d:41:56:40:72:14:14:78: + f9:32:45:de:f9:4d:ef:45:32:a0:21:10:5c:76:f5:ee:fd:a9: + 37:34:04:67:94:72:14:54:5f:27:c3:d3:2d:a6:7f:94:4d:a8: + 98:c4:f9:d9:fc:cd:37:92:3c:c8:bb:2d:a1:f6:ea:14:50:08: + 46:38:9a:cd:71:1b:5b:b4:d9:18:88:77:74:dd:ae:df:b1:58: + 52:f2:8b:e7:bb:0b:0c:92:a6:41:d1:b6:17:64:08:09:7f:7d: + c6:f5:c0:11:a7:29:35:9e:23:9d:e2:08:d5:0e:cb:0c:0c:f2: + d0:7e:38:67 -----BEGIN CERTIFICATE----- -MIIDAzCCAeugAwIBAgIUd0/2z5nKd+inbh794s+sqdpo0kIwDQYJKoZIhvcNAQEL -BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz -MzgyNFowETEPMA0GA1UEAwwGQ0FSb290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A -MIIBCgKCAQEAuF7CYO3E7jxbq/xkUvMwQfwQWqymmwqT0NDJv5YUp89cPiORflTs -/i2fyTTRTpUvhZzMvpCjpMtNpHLShODHQsS/cLb60kWLg2YepOkOBqNG6qcYzTO5 -8f92kXKPzfmTQ8NuFx8tht+2+y3Wvi2Yrd4Ax975aLVAQFZJriPloTtfFVpEUNr7 -AtNCxocNwI065uKqczGreVhRzQOA8xLOLzUEizlfsMy4QZlHwReWi8JEhLUhihVS -/hpa+YjMERfuSN26v+1nbic1Qs8HXrGLgVWSAY5h/Y6CdLFwej1SHxZ4Eru1CWLO -bRhK6fUnGbyTTu3dU6jBu0i3GCB7eUhInQIDAQABo1MwUTAdBgNVHQ4EFgQUD0Zh -Pm9xIuYfMjd8soGmzNud9XwwHwYDVR0jBBgwFoAUD0ZhPm9xIuYfMjd8soGmzNud -9XwwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAkejYxDIugFzU -yyR6gUOpx5WQGi560wxdtiEFZ02YWg1x6oABlUL++vF83L12/wUmO/CUswksNN1D -VkYrFTWZ2ZRUIs+maLDReeLwnwsCfM8fvdD2ScaCKKXGrpRlz/2tqGzCF9rb874w -Ghu0LPoIcZ1kCUUCkgKt6xVHFENbqC0a7BST3P+7UTOj1U3id8rhpZhcerYQGdPX -9RSl1Qjxlxg9X6ZOokoNS9S7VmuoRDVixdjGZxGTHCJkPqoVCNyHOd324KDVANsn -eT30NXxGqfoM+vx09b/0/nFARTMiNYP3GpYq/LIz4BroJEiRXZBcTJMzTEDeJrsk -rEibrv4ZNA== +MIIDGjCCAgKgAwIBAgIJANftdw9pWYiXMA0GCSqGSIb3DQEBCwUAMBExDzANBgNV +BAMMBkNBUm9vdDAeFw0yMzAyMjIwNjI2MzJaFw0zMzAyMTkwNjI2MzJaMBExDzAN +BgNVBAMMBkNBUm9vdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL8+ +5tv9omy5RSno05BNeCnv/GfE4K4G/PgryztLic83PKuGlFnDUFRLGNWLFPXNNljG +rKFn9wRYWC/giXOLse+XpBYQl+ZvLxi5jJN7fFtPjQlJqnBZ5Tv6wLlK7RSYCl9W +s0kKTcAiHnU9uvkZ2miAGK23j979H2AzhnRG4Ld6hF63r1tXPJOtN4MsG+B3o4Ta +JR0Wdz8lsZBJKKbIz7zkuSeF+TZPR4HNViZBIxCPNibOeLmrRc6c66MqEZOut9TX +V+mXcVv6yg5xNEOHu8COaPRNyGRFAl2BvTtHvOxO5GHjxBbw74P9BlwFUNhQQdzW +Ym2yJnvTa/baWUyo27UCAwEAAaN1MHMwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E +FgQU+bKtxCRiRzwaWKpm0JES8yDuqWwwQQYDVR0jBDowOIAU+bKtxCRiRzwaWKpm +0JES8yDuqWyhFaQTMBExDzANBgNVBAMMBkNBUm9vdIIJANftdw9pWYiXMA0GCSqG +SIb3DQEBCwUAA4IBAQC8JAVlVof5h/QbWqOgAcZYwHzKye4dDNVrRyjcvS/xcwPh +pQgFVhEp1UgyI9N9RtsgWCl4Ga9qotG2bjCgo7GIwbUf8A9jXquFXwkKPCDLYTuR +gHBOH8U3NA0cn7Xek/GulP98dn1qpxlvIjS9ZgcoWfNg7zmNvZ0rggjwaKrW4Gjx +8dLGxmGIDUFWQHIUFHj5MkXe+U3vRTKgIRBcdvXu/ak3NARnlHIUVF8nw9Mtpn+U +TaiYxPnZ/M03kjzIuy2h9uoUUAhGOJrNcRtbtNkYiHd03a7fsVhS8ovnuwsMkqZB +0bYXZAgJf33G9cARpyk1niOd4gjVDssMDPLQfjhn -----END CERTIFICATE----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-cert.pem b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-cert.pem index 1a21d9d41387f..6b2387d9a07f8 100644 --- a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-cert.pem +++ b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/client-cert.pem @@ -1,17 +1,16 @@ Certificate: Data: Version: 3 (0x2) - Serial Number: - 61:e6:1b:07:90:6a:4f:f7:cd:46:b9:59:1d:3e:1c:39:0d:f2:5e:03 - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN = CARoot + Serial Number: 15537474201172114490 (0xd7a0327703a8fc3a) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=CARoot Validity - Not Before: May 30 13:38:24 2022 GMT - Not After : May 27 13:38:24 2032 GMT - Subject: C = US, ST = CA, O = Apache Pulsar, OU = Client, CN = Client + Not Before: Feb 22 06:26:32 2023 GMT + Not After : Feb 19 06:26:32 2033 GMT + Subject: C=US, ST=CA, O=Apache Pulsar, OU=Client, CN=Client Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (2048 bit) + Public-Key: (2048 bit) Modulus: 00:de:1e:10:bd:64:13:c1:6c:7a:49:86:01:3b:ab: ab:1d:ec:b2:93:41:6c:6c:21:f2:e6:15:1b:51:ce: @@ -36,37 +35,37 @@ Certificate: X509v3 Subject Alternative Name: DNS:localhost, IP Address:127.0.0.1 Signature Algorithm: sha256WithRSAEncryption - 8b:88:90:00:1a:15:fa:11:f2:f0:35:6f:0f:f2:76:74:fc:8d: - bc:03:ee:a5:c5:21:17:c9:01:6b:58:93:fa:3e:7b:e0:0d:6d: - db:1f:2a:48:fa:15:34:66:b7:cb:be:82:c6:28:91:99:42:5a: - 36:b6:0b:2f:bb:85:14:88:a9:ea:dd:0a:7a:be:c4:e7:b2:2d: - 82:a9:37:bc:d9:5c:aa:03:2e:54:68:b1:b7:e8:d6:45:a5:8f: - 48:45:2c:9c:7a:55:0a:4a:07:1b:30:8a:49:6d:f4:62:b1:9e: - 92:0e:d9:34:44:6c:6d:e7:a3:18:bb:85:58:6d:da:20:83:d5: - ca:65:63:1e:3b:e6:df:7b:97:40:4f:b1:59:63:a9:b5:80:6f: - 97:51:53:a1:d3:29:1f:1a:26:05:17:59:3e:16:4f:5f:38:36: - 76:30:c6:bf:1e:3e:ed:39:83:91:31:58:01:13:59:5c:c5:e9: - d6:61:e0:f3:5f:c7:47:8a:5f:af:23:98:89:7b:b4:e6:f6:51: - 98:a0:26:31:c8:67:91:6d:d5:68:75:3d:4d:48:44:5f:3b:9c: - df:a7:87:a0:11:02:d2:13:5f:c1:4c:3f:3e:09:59:2e:fc:cb: - c2:c5:f0:f8:91:df:c3:dd:ad:c8:fc:44:23:9b:78:0d:3b:f2: - 82:f6:02:82 + a3:0e:ff:87:38:9a:fe:1c:b7:4b:ac:b1:6c:ad:30:90:94:6e: + 75:36:f6:46:7d:9b:69:1b:0d:92:1b:fc:39:7c:7a:24:fc:4d: + 77:05:8e:70:6e:2e:db:3a:5f:5d:70:80:71:f5:00:7f:6e:12: + 7e:78:58:0b:8f:93:56:64:29:6f:bb:7d:93:a1:fa:2a:83:98: + a3:92:73:df:1d:69:7b:51:00:0f:18:68:a5:75:13:ef:3c:38: + 97:c3:31:84:d6:3c:83:50:77:c3:f6:52:69:5a:9c:35:21:a4: + c2:9f:01:14:b7:2a:a5:3d:71:b2:a6:08:73:10:8e:91:e3:3c: + 69:f9:74:ab:92:f4:16:5d:79:71:3f:3b:58:51:5b:c7:d9:a3: + 85:39:15:60:6b:f7:85:59:e4:4b:96:df:d9:f2:d4:4d:48:34: + 17:50:66:d6:e3:50:49:0d:e7:d5:d9:e0:81:4e:9b:a6:b5:6b: + 72:f3:df:b4:0b:85:5a:e6:e4:ec:28:3a:06:e0:67:e2:be:c1: + 12:7d:9d:5c:ef:3e:77:29:ee:8f:87:44:c7:79:6f:67:0b:fe: + cf:38:76:25:24:be:70:41:99:7f:47:6a:1e:ce:15:f5:bc:4a: + b9:e2:74:15:a2:05:c2:95:02:86:f2:ae:36:a6:88:bc:ad:62: + 3d:07:b8:fd -----BEGIN CERTIFICATE----- -MIIDETCCAfmgAwIBAgIUYeYbB5BqT/fNRrlZHT4cOQ3yXgMwDQYJKoZIhvcNAQEL -BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz -MzgyNFowVDELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMRYwFAYDVQQKEw1BcGFj -aGUgUHVsc2FyMQ8wDQYDVQQLEwZDbGllbnQxDzANBgNVBAMTBkNsaWVudDCCASIw -DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAN4eEL1kE8FsekmGATurqx3sspNB -bGwh8uYVG1HOrWf9GD5/emSiYl8uC1m07dkXDre8UGZBt+PEcclzcz3YbTSA8uO5 -mI8rVBSVs1Eb1pGFzbc0olC28YZuBzD6rlWgXfl8HJFQYn27FIaSCqwpPigbmcow -Y9ypXwX4OD4wEAKfzJTXR+Aa9Bxolj0SXlghQSzslq2eCFaDepJfS+a9ARZwKK+q -Jx3E/rIJv6W0R9lYS/5BgQ6iRlfBOXyN5LGnJea03fOeJMnnwIwatKvduTO/Ecu+ -uyL3/K3EQEHX7zcIGpVFH9sUXwv4SP9BJMtcjhhITF8Z6bB7ItO8QjJFmtECAwEA -AaMeMBwwGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEBCwUA -A4IBAQCLiJAAGhX6EfLwNW8P8nZ0/I28A+6lxSEXyQFrWJP6PnvgDW3bHypI+hU0 -ZrfLvoLGKJGZQlo2tgsvu4UUiKnq3Qp6vsTnsi2CqTe82VyqAy5UaLG36NZFpY9I -RSycelUKSgcbMIpJbfRisZ6SDtk0RGxt56MYu4VYbdogg9XKZWMeO+bfe5dAT7FZ -Y6m1gG+XUVOh0ykfGiYFF1k+Fk9fODZ2MMa/Hj7tOYORMVgBE1lcxenWYeDzX8dH -il+vI5iJe7Tm9lGYoCYxyGeRbdVodT1NSERfO5zfp4egEQLSE1/BTD8+CVku/MvC -xfD4kd/D3a3I/EQjm3gNO/KC9gKC +MIIDBjCCAe6gAwIBAgIJANegMncDqPw6MA0GCSqGSIb3DQEBCwUAMBExDzANBgNV +BAMMBkNBUm9vdDAeFw0yMzAyMjIwNjI2MzJaFw0zMzAyMTkwNjI2MzJaMFQxCzAJ +BgNVBAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEChMNQXBhY2hlIFB1bHNhcjEP +MA0GA1UECxMGQ2xpZW50MQ8wDQYDVQQDEwZDbGllbnQwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQDeHhC9ZBPBbHpJhgE7q6sd7LKTQWxsIfLmFRtRzq1n +/Rg+f3pkomJfLgtZtO3ZFw63vFBmQbfjxHHJc3M92G00gPLjuZiPK1QUlbNRG9aR +hc23NKJQtvGGbgcw+q5VoF35fByRUGJ9uxSGkgqsKT4oG5nKMGPcqV8F+Dg+MBAC +n8yU10fgGvQcaJY9El5YIUEs7JatnghWg3qSX0vmvQEWcCivqicdxP6yCb+ltEfZ +WEv+QYEOokZXwTl8jeSxpyXmtN3zniTJ58CMGrSr3bkzvxHLvrsi9/ytxEBB1+83 +CBqVRR/bFF8L+Ej/QSTLXI4YSExfGemweyLTvEIyRZrRAgMBAAGjHjAcMBoGA1Ud +EQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEAow7/hzia +/hy3S6yxbK0wkJRudTb2Rn2baRsNkhv8OXx6JPxNdwWOcG4u2zpfXXCAcfUAf24S +fnhYC4+TVmQpb7t9k6H6KoOYo5Jz3x1pe1EADxhopXUT7zw4l8MxhNY8g1B3w/ZS +aVqcNSGkwp8BFLcqpT1xsqYIcxCOkeM8afl0q5L0Fl15cT87WFFbx9mjhTkVYGv3 +hVnkS5bf2fLUTUg0F1Bm1uNQSQ3n1dnggU6bprVrcvPftAuFWubk7Cg6BuBn4r7B +En2dXO8+dynuj4dEx3lvZwv+zzh2JSS+cEGZf0dqHs4V9bxKueJ0FaIFwpUChvKu +NqaIvK1iPQe4/Q== -----END CERTIFICATE----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/no-subject-alt-cert.pem b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/no-subject-alt-cert.pem new file mode 100644 index 0000000000000..789a91ca712ce --- /dev/null +++ b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/no-subject-alt-cert.pem @@ -0,0 +1,67 @@ +Certificate: + Data: + Version: 1 (0x0) + Serial Number: 15537474201172114492 (0xd7a0327703a8fc3c) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=CARoot + Validity + Not Before: Feb 22 06:26:32 2023 GMT + Not After : Feb 19 06:26:32 2033 GMT + Subject: C=US, ST=CA, O=Apache Pulsar, OU=Broker, CN=Broker + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:ca:77:dc:2a:13:25:24:cb:29:62:06:12:5f:a8: + 92:c9:53:d6:3f:07:ca:aa:0a:5f:72:92:cd:b7:ea: + 45:47:71:f0:63:4f:58:1a:3d:fa:ce:a6:73:90:c0: + a9:f7:25:f0:76:75:ed:b2:03:17:be:d8:8a:56:f3: + 4f:6a:4c:7e:03:65:95:e5:45:eb:8d:47:e8:60:5e: + 9e:38:74:50:54:65:a0:ec:d8:5c:65:60:34:1b:96: + 83:7d:71:d4:5d:7f:e3:62:59:67:e8:f0:d6:24:7d: + c0:6e:37:03:54:4c:3d:0c:33:39:9b:33:e1:52:44: + c5:43:da:ea:ee:2c:f3:1c:16:2e:46:4c:7c:9f:5d: + 4d:6e:fe:8c:23:9e:f7:7e:9f:39:c1:71:06:52:f4: + 26:9a:22:d4:cf:c5:25:39:a9:d2:e4:24:c6:d8:4a: + 48:a2:ee:76:25:cb:3c:f0:bf:cd:10:77:ff:81:11: + 43:21:cc:3b:cc:10:7a:07:84:fc:cc:02:a2:45:de: + 91:2d:6b:d1:ed:17:1a:d0:46:f4:ae:7d:b3:89:f8: + 31:77:95:e5:46:b1:a9:31:d6:d8:e3:47:00:b2:81: + 81:db:8a:1c:d9:f1:cd:e3:4d:35:f6:38:91:0d:ea: + 07:f0:b0:06:4f:2c:4c:75:c2:37:ff:35:0d:b1:42: + 06:0b + Exponent: 65537 (0x10001) + Signature Algorithm: sha256WithRSAEncryption + 67:a9:c5:b1:e0:12:19:67:f7:27:db:87:90:15:29:99:fc:ea: + 62:b3:73:c3:6f:78:fe:50:17:14:8a:61:35:e3:28:ab:3e:c3: + 85:24:ff:70:81:04:0d:b7:7a:eb:e9:dc:06:97:b4:0f:2c:97: + 6d:81:f7:da:dc:f9:ff:91:94:69:5c:15:29:3a:25:87:ff:ef: + 98:6d:5a:36:19:2d:10:cf:d8:3a:d4:45:30:75:5c:52:58:ef: + e6:6c:27:a0:17:a1:a6:76:05:f4:f3:cb:89:89:61:32:c5:bf: + f0:f3:1c:85:90:78:88:c4:37:63:5c:e6:39:43:c0:b0:51:9d: + cc:51:9f:32:b3:78:47:3e:5e:da:58:12:72:df:ba:11:17:a5: + 40:b8:ef:9e:e1:40:49:38:51:7e:76:1e:6c:7f:d3:70:02:de: + af:bb:a6:e0:53:d8:1d:2e:e5:b6:98:6c:27:92:cf:86:3d:0f: + 01:13:95:5d:40:35:47:dc:1b:4c:e9:52:5e:34:98:13:35:35: + 5b:c4:df:fd:61:99:d4:7f:f4:04:fb:26:97:d7:25:8e:fc:1a: + 13:88:37:53:b2:91:3d:0f:0d:9c:31:8a:d0:76:31:dd:50:85: + 43:8a:9b:46:20:a8:3f:f7:9c:30:bb:39:cc:02:ef:7e:22:32: + 8a:df:7d:93 +-----BEGIN CERTIFICATE----- +MIIC4TCCAckCCQDXoDJ3A6j8PDANBgkqhkiG9w0BAQsFADARMQ8wDQYDVQQDDAZD +QVJvb3QwHhcNMjMwMjIyMDYyNjMyWhcNMzMwMjE5MDYyNjMyWjBUMQswCQYDVQQG +EwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAoTDUFwYWNoZSBQdWxzYXIxDzANBgNV +BAsTBkJyb2tlcjEPMA0GA1UEAxMGQnJva2VyMIIBIjANBgkqhkiG9w0BAQEFAAOC +AQ8AMIIBCgKCAQEAynfcKhMlJMspYgYSX6iSyVPWPwfKqgpfcpLNt+pFR3HwY09Y +Gj36zqZzkMCp9yXwdnXtsgMXvtiKVvNPakx+A2WV5UXrjUfoYF6eOHRQVGWg7Nhc +ZWA0G5aDfXHUXX/jYlln6PDWJH3AbjcDVEw9DDM5mzPhUkTFQ9rq7izzHBYuRkx8 +n11Nbv6MI573fp85wXEGUvQmmiLUz8UlOanS5CTG2EpIou52Jcs88L/NEHf/gRFD +Icw7zBB6B4T8zAKiRd6RLWvR7Rca0Eb0rn2zifgxd5XlRrGpMdbY40cAsoGB24oc +2fHN40019jiRDeoH8LAGTyxMdcI3/zUNsUIGCwIDAQABMA0GCSqGSIb3DQEBCwUA +A4IBAQBnqcWx4BIZZ/cn24eQFSmZ/Opis3PDb3j+UBcUimE14yirPsOFJP9wgQQN +t3rr6dwGl7QPLJdtgffa3Pn/kZRpXBUpOiWH/++YbVo2GS0Qz9g61EUwdVxSWO/m +bCegF6GmdgX088uJiWEyxb/w8xyFkHiIxDdjXOY5Q8CwUZ3MUZ8ys3hHPl7aWBJy +37oRF6VAuO+e4UBJOFF+dh5sf9NwAt6vu6bgU9gdLuW2mGwnks+GPQ8BE5VdQDVH +3BtM6VJeNJgTNTVbxN/9YZnUf/QE+yaX1yWO/BoTiDdTspE9Dw2cMYrQdjHdUIVD +iptGIKg/95wwuznMAu9+IjKK332T +-----END CERTIFICATE----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/no-subject-alt-key.pem b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/no-subject-alt-key.pem new file mode 100644 index 0000000000000..63bbb7bfea469 --- /dev/null +++ b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/no-subject-alt-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDKd9wqEyUkyyli +BhJfqJLJU9Y/B8qqCl9yks236kVHcfBjT1gaPfrOpnOQwKn3JfB2de2yAxe+2IpW +809qTH4DZZXlReuNR+hgXp44dFBUZaDs2FxlYDQbloN9cdRdf+NiWWfo8NYkfcBu +NwNUTD0MMzmbM+FSRMVD2uruLPMcFi5GTHyfXU1u/owjnvd+nznBcQZS9CaaItTP +xSU5qdLkJMbYSkii7nYlyzzwv80Qd/+BEUMhzDvMEHoHhPzMAqJF3pEta9HtFxrQ +RvSufbOJ+DF3leVGsakx1tjjRwCygYHbihzZ8c3jTTX2OJEN6gfwsAZPLEx1wjf/ +NQ2xQgYLAgMBAAECggEARpLZD2F1BQo79osfRHDCGaM7fuT8Y6ER/CHnyz/BvlGc +9UDm+N652eZzSfWeSSPUWbZpkC87y643Km/NMsRO+Ggkg7KHlMuH2G+ivxLsHT7/ +hQ81xbBu+V7Rnpxa5ex6GgIIEk5Alp+uv7w1UODyNpp0bgD7fW2zRR+93B+W7ia+ +aWLcFur1LgGUVpqmlDKZBLD+q3oJ7ddi/uam8WS41IxtUvUVW4L8Pz4sCGjVqEMC +1SbUuuNT5dWLas21c5RhLn1mfyKzLSfeL63+WLuaEobR3GpLDJeG/P6CUCJfrN+j +NtTDFq89QxGzgN6Rvy9MuHC4kHWHvgGlfZ7uZdzWgQKBgQDl3CabW+ZNPCZk3JHU +fGI0Xb3jQElooXOqZOH+FgGKnrbNb7j04Gjs1P4/XibnVsvgwCL8TbR1hgBD6/Qx +z0Sd2T0nwCmLyO9LzyOrlpcKaKF+4OYFPKiqZGV1jXhCQXH9b7IXufS8U4uXwD+Z +elw5MOD6DON7ud9V5E/J5ST58QKBgQDhfkKvtgzaLPD17Bx0M30buHzQuQHplpc4 +J0WGWUXR6rui5tCeHoASAl+UNAFReWJ7Ra+iTHMNqwolVsSQVzmX6e8342f9y0bV +3iv1ge/dA75gEqxifqSXHVm6T/j40DBIr4fwjl5L2qCB/JKCyRvoCK3pDrYZLXWP +DRWhssujuwKBgQCfQBhrWI9FgV/kT0Clo4tyVmQBtv9lAz6clgpQvDRTMsTZrgbJ +eVSYiLSheHyhmGvmCZfzj25wYed7J1Vm0P/sEJ8jFCp0k0DfF+LRtaJtbrI8sloK +1MzSSH5WpC3mUWtFOAZ+E7Kwa31yJJqrna+ZW/jypM1SYiOOYYC6Ewy8MQKBgFdq +GPQBAQ57KZZMR+OMKk3awRgxAFrLdCfioYMpjHWKJ99I10rUzBUvMlpDptcs1U6w +fxvNwzRjP/Wlo2HJTpxjpcbms2Ohr/4suKHeE1x8nQqlcopkSe4DBMvDQOND4dPr +qClLJ6cERADgJvPofpb+9lxIxbMQ+mfQTLh4lZUNAoGAPfAhkt8i6L3VkBIMaV9X +U+6q4brsT0dNLOO/lgf5FXQuCg0WIgBIb1vrGDD1i9WAUiNN8zzYK9UxqjpAtRAe +LgPYX5GHXR0ceR0MQNHdbc4RRjJbPmgey+d7pc9EUn8WWt/uXIeo01DHBPPjsgHr +k/JZjqmRla+2pklmoG2sfI0= +-----END PRIVATE KEY----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cacert.pem b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cacert.pem index 127f56dd777a5..2a71f0e3afc17 100644 --- a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cacert.pem +++ b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cacert.pem @@ -1,77 +1,78 @@ Certificate: Data: Version: 3 (0x2) - Serial Number: - 77:4f:f6:cf:99:ca:77:e8:a7:6e:1e:fd:e2:cf:ac:a9:da:68:d2:42 - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN = CARoot + Serial Number: 15559223195710621847 (0xd7ed770f69598897) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=CARoot Validity - Not Before: May 30 13:38:24 2022 GMT - Not After : May 27 13:38:24 2032 GMT - Subject: CN = CARoot + Not Before: Feb 22 06:26:32 2023 GMT + Not After : Feb 19 06:26:32 2033 GMT + Subject: CN=CARoot Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (2048 bit) + Public-Key: (2048 bit) Modulus: - 00:b8:5e:c2:60:ed:c4:ee:3c:5b:ab:fc:64:52:f3: - 30:41:fc:10:5a:ac:a6:9b:0a:93:d0:d0:c9:bf:96: - 14:a7:cf:5c:3e:23:91:7e:54:ec:fe:2d:9f:c9:34: - d1:4e:95:2f:85:9c:cc:be:90:a3:a4:cb:4d:a4:72: - d2:84:e0:c7:42:c4:bf:70:b6:fa:d2:45:8b:83:66: - 1e:a4:e9:0e:06:a3:46:ea:a7:18:cd:33:b9:f1:ff: - 76:91:72:8f:cd:f9:93:43:c3:6e:17:1f:2d:86:df: - b6:fb:2d:d6:be:2d:98:ad:de:00:c7:de:f9:68:b5: - 40:40:56:49:ae:23:e5:a1:3b:5f:15:5a:44:50:da: - fb:02:d3:42:c6:87:0d:c0:8d:3a:e6:e2:aa:73:31: - ab:79:58:51:cd:03:80:f3:12:ce:2f:35:04:8b:39: - 5f:b0:cc:b8:41:99:47:c1:17:96:8b:c2:44:84:b5: - 21:8a:15:52:fe:1a:5a:f9:88:cc:11:17:ee:48:dd: - ba:bf:ed:67:6e:27:35:42:cf:07:5e:b1:8b:81:55: - 92:01:8e:61:fd:8e:82:74:b1:70:7a:3d:52:1f:16: - 78:12:bb:b5:09:62:ce:6d:18:4a:e9:f5:27:19:bc: - 93:4e:ed:dd:53:a8:c1:bb:48:b7:18:20:7b:79:48: - 48:9d + 00:bf:3e:e6:db:fd:a2:6c:b9:45:29:e8:d3:90:4d: + 78:29:ef:fc:67:c4:e0:ae:06:fc:f8:2b:cb:3b:4b: + 89:cf:37:3c:ab:86:94:59:c3:50:54:4b:18:d5:8b: + 14:f5:cd:36:58:c6:ac:a1:67:f7:04:58:58:2f:e0: + 89:73:8b:b1:ef:97:a4:16:10:97:e6:6f:2f:18:b9: + 8c:93:7b:7c:5b:4f:8d:09:49:aa:70:59:e5:3b:fa: + c0:b9:4a:ed:14:98:0a:5f:56:b3:49:0a:4d:c0:22: + 1e:75:3d:ba:f9:19:da:68:80:18:ad:b7:8f:de:fd: + 1f:60:33:86:74:46:e0:b7:7a:84:5e:b7:af:5b:57: + 3c:93:ad:37:83:2c:1b:e0:77:a3:84:da:25:1d:16: + 77:3f:25:b1:90:49:28:a6:c8:cf:bc:e4:b9:27:85: + f9:36:4f:47:81:cd:56:26:41:23:10:8f:36:26:ce: + 78:b9:ab:45:ce:9c:eb:a3:2a:11:93:ae:b7:d4:d7: + 57:e9:97:71:5b:fa:ca:0e:71:34:43:87:bb:c0:8e: + 68:f4:4d:c8:64:45:02:5d:81:bd:3b:47:bc:ec:4e: + e4:61:e3:c4:16:f0:ef:83:fd:06:5c:05:50:d8:50: + 41:dc:d6:62:6d:b2:26:7b:d3:6b:f6:da:59:4c:a8: + db:b5 Exponent: 65537 (0x10001) X509v3 extensions: + X509v3 Basic Constraints: critical + CA:TRUE X509v3 Subject Key Identifier: - 0F:46:61:3E:6F:71:22:E6:1F:32:37:7C:B2:81:A6:CC:DB:9D:F5:7C + F9:B2:AD:C4:24:62:47:3C:1A:58:AA:66:D0:91:12:F3:20:EE:A9:6C X509v3 Authority Key Identifier: - keyid:0F:46:61:3E:6F:71:22:E6:1F:32:37:7C:B2:81:A6:CC:DB:9D:F5:7C + keyid:F9:B2:AD:C4:24:62:47:3C:1A:58:AA:66:D0:91:12:F3:20:EE:A9:6C + DirName:/CN=CARoot + serial:D7:ED:77:0F:69:59:88:97 - X509v3 Basic Constraints: critical - CA:TRUE Signature Algorithm: sha256WithRSAEncryption - 91:e8:d8:c4:32:2e:80:5c:d4:cb:24:7a:81:43:a9:c7:95:90: - 1a:2e:7a:d3:0c:5d:b6:21:05:67:4d:98:5a:0d:71:ea:80:01: - 95:42:fe:fa:f1:7c:dc:bd:76:ff:05:26:3b:f0:94:b3:09:2c: - 34:dd:43:56:46:2b:15:35:99:d9:94:54:22:cf:a6:68:b0:d1: - 79:e2:f0:9f:0b:02:7c:cf:1f:bd:d0:f6:49:c6:82:28:a5:c6: - ae:94:65:cf:fd:ad:a8:6c:c2:17:da:db:f3:be:30:1a:1b:b4: - 2c:fa:08:71:9d:64:09:45:02:92:02:ad:eb:15:47:14:43:5b: - a8:2d:1a:ec:14:93:dc:ff:bb:51:33:a3:d5:4d:e2:77:ca:e1: - a5:98:5c:7a:b6:10:19:d3:d7:f5:14:a5:d5:08:f1:97:18:3d: - 5f:a6:4e:a2:4a:0d:4b:d4:bb:56:6b:a8:44:35:62:c5:d8:c6: - 67:11:93:1c:22:64:3e:aa:15:08:dc:87:39:dd:f6:e0:a0:d5: - 00:db:27:79:3d:f4:35:7c:46:a9:fa:0c:fa:fc:74:f5:bf:f4: - fe:71:40:45:33:22:35:83:f7:1a:96:2a:fc:b2:33:e0:1a:e8: - 24:48:91:5d:90:5c:4c:93:33:4c:40:de:26:bb:24:ac:48:9b: - ae:fe:19:34 + bc:24:05:65:56:87:f9:87:f4:1b:5a:a3:a0:01:c6:58:c0:7c: + ca:c9:ee:1d:0c:d5:6b:47:28:dc:bd:2f:f1:73:03:e1:a5:08: + 05:56:11:29:d5:48:32:23:d3:7d:46:db:20:58:29:78:19:af: + 6a:a2:d1:b6:6e:30:a0:a3:b1:88:c1:b5:1f:f0:0f:63:5e:ab: + 85:5f:09:0a:3c:20:cb:61:3b:91:80:70:4e:1f:c5:37:34:0d: + 1c:9f:b5:de:93:f1:ae:94:ff:7c:76:7d:6a:a7:19:6f:22:34: + bd:66:07:28:59:f3:60:ef:39:8d:bd:9d:2b:82:08:f0:68:aa: + d6:e0:68:f1:f1:d2:c6:c6:61:88:0d:41:56:40:72:14:14:78: + f9:32:45:de:f9:4d:ef:45:32:a0:21:10:5c:76:f5:ee:fd:a9: + 37:34:04:67:94:72:14:54:5f:27:c3:d3:2d:a6:7f:94:4d:a8: + 98:c4:f9:d9:fc:cd:37:92:3c:c8:bb:2d:a1:f6:ea:14:50:08: + 46:38:9a:cd:71:1b:5b:b4:d9:18:88:77:74:dd:ae:df:b1:58: + 52:f2:8b:e7:bb:0b:0c:92:a6:41:d1:b6:17:64:08:09:7f:7d: + c6:f5:c0:11:a7:29:35:9e:23:9d:e2:08:d5:0e:cb:0c:0c:f2: + d0:7e:38:67 -----BEGIN CERTIFICATE----- -MIIDAzCCAeugAwIBAgIUd0/2z5nKd+inbh794s+sqdpo0kIwDQYJKoZIhvcNAQEL -BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz -MzgyNFowETEPMA0GA1UEAwwGQ0FSb290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A -MIIBCgKCAQEAuF7CYO3E7jxbq/xkUvMwQfwQWqymmwqT0NDJv5YUp89cPiORflTs -/i2fyTTRTpUvhZzMvpCjpMtNpHLShODHQsS/cLb60kWLg2YepOkOBqNG6qcYzTO5 -8f92kXKPzfmTQ8NuFx8tht+2+y3Wvi2Yrd4Ax975aLVAQFZJriPloTtfFVpEUNr7 -AtNCxocNwI065uKqczGreVhRzQOA8xLOLzUEizlfsMy4QZlHwReWi8JEhLUhihVS -/hpa+YjMERfuSN26v+1nbic1Qs8HXrGLgVWSAY5h/Y6CdLFwej1SHxZ4Eru1CWLO -bRhK6fUnGbyTTu3dU6jBu0i3GCB7eUhInQIDAQABo1MwUTAdBgNVHQ4EFgQUD0Zh -Pm9xIuYfMjd8soGmzNud9XwwHwYDVR0jBBgwFoAUD0ZhPm9xIuYfMjd8soGmzNud -9XwwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAkejYxDIugFzU -yyR6gUOpx5WQGi560wxdtiEFZ02YWg1x6oABlUL++vF83L12/wUmO/CUswksNN1D -VkYrFTWZ2ZRUIs+maLDReeLwnwsCfM8fvdD2ScaCKKXGrpRlz/2tqGzCF9rb874w -Ghu0LPoIcZ1kCUUCkgKt6xVHFENbqC0a7BST3P+7UTOj1U3id8rhpZhcerYQGdPX -9RSl1Qjxlxg9X6ZOokoNS9S7VmuoRDVixdjGZxGTHCJkPqoVCNyHOd324KDVANsn -eT30NXxGqfoM+vx09b/0/nFARTMiNYP3GpYq/LIz4BroJEiRXZBcTJMzTEDeJrsk -rEibrv4ZNA== +MIIDGjCCAgKgAwIBAgIJANftdw9pWYiXMA0GCSqGSIb3DQEBCwUAMBExDzANBgNV +BAMMBkNBUm9vdDAeFw0yMzAyMjIwNjI2MzJaFw0zMzAyMTkwNjI2MzJaMBExDzAN +BgNVBAMMBkNBUm9vdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL8+ +5tv9omy5RSno05BNeCnv/GfE4K4G/PgryztLic83PKuGlFnDUFRLGNWLFPXNNljG +rKFn9wRYWC/giXOLse+XpBYQl+ZvLxi5jJN7fFtPjQlJqnBZ5Tv6wLlK7RSYCl9W +s0kKTcAiHnU9uvkZ2miAGK23j979H2AzhnRG4Ld6hF63r1tXPJOtN4MsG+B3o4Ta +JR0Wdz8lsZBJKKbIz7zkuSeF+TZPR4HNViZBIxCPNibOeLmrRc6c66MqEZOut9TX +V+mXcVv6yg5xNEOHu8COaPRNyGRFAl2BvTtHvOxO5GHjxBbw74P9BlwFUNhQQdzW +Ym2yJnvTa/baWUyo27UCAwEAAaN1MHMwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E +FgQU+bKtxCRiRzwaWKpm0JES8yDuqWwwQQYDVR0jBDowOIAU+bKtxCRiRzwaWKpm +0JES8yDuqWyhFaQTMBExDzANBgNVBAMMBkNBUm9vdIIJANftdw9pWYiXMA0GCSqG +SIb3DQEBCwUAA4IBAQC8JAVlVof5h/QbWqOgAcZYwHzKye4dDNVrRyjcvS/xcwPh +pQgFVhEp1UgyI9N9RtsgWCl4Ga9qotG2bjCgo7GIwbUf8A9jXquFXwkKPCDLYTuR +gHBOH8U3NA0cn7Xek/GulP98dn1qpxlvIjS9ZgcoWfNg7zmNvZ0rggjwaKrW4Gjx +8dLGxmGIDUFWQHIUFHj5MkXe+U3vRTKgIRBcdvXu/ak3NARnlHIUVF8nw9Mtpn+U +TaiYxPnZ/M03kjzIuy2h9uoUUAhGOJrNcRtbtNkYiHd03a7fsVhS8ovnuwsMkqZB +0bYXZAgJf33G9cARpyk1niOd4gjVDssMDPLQfjhn -----END CERTIFICATE----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cert.pem b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cert.pem index e2c1e5a230c26..8b0624c0b7f3e 100644 --- a/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cert.pem +++ b/pulsar-proxy/src/test/resources/authentication/tls/ProxyWithAuthorizationTest/proxy-cert.pem @@ -1,17 +1,16 @@ Certificate: Data: Version: 3 (0x2) - Serial Number: - 61:e6:1b:07:90:6a:4f:f7:cd:46:b9:59:1d:3e:1c:39:0d:f2:5e:04 - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN = CARoot + Serial Number: 15537474201172114491 (0xd7a0327703a8fc3b) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=CARoot Validity - Not Before: May 30 13:38:24 2022 GMT - Not After : May 27 13:38:24 2032 GMT - Subject: C = US, ST = CA, O = Apache Pulsar, OU = Proxy, CN = Proxy + Not Before: Feb 22 06:26:32 2023 GMT + Not After : Feb 19 06:26:32 2033 GMT + Subject: C=US, ST=CA, O=Apache Pulsar, OU=Proxy, CN=Proxy Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (2048 bit) + Public-Key: (2048 bit) Modulus: 00:c3:5c:c5:ad:17:dc:f4:d4:c4:ea:1c:60:5a:24: 46:13:d9:cf:c0:cd:83:2e:2f:82:70:e5:e0:8d:33: @@ -36,37 +35,37 @@ Certificate: X509v3 Subject Alternative Name: DNS:localhost, IP Address:127.0.0.1 Signature Algorithm: sha256WithRSAEncryption - 8d:b6:2c:5f:87:13:06:a8:66:ce:11:2a:2c:20:1e:c7:ee:50: - 75:a7:d1:7c:ad:c6:ec:d1:18:d0:fa:aa:00:fa:08:f9:0f:cc: - df:59:9a:6b:1c:18:07:15:84:d0:9a:24:8d:dd:46:79:9c:dc: - 9e:3e:97:10:24:b2:9d:d4:f6:c5:79:58:87:7c:a6:af:cf:69: - 23:fb:43:7a:0f:4d:26:e0:e9:66:c5:ad:fa:88:e2:c5:6e:6a: - ce:70:0c:8f:73:01:d6:fd:a9:1f:31:49:41:17:45:22:cc:a6: - 71:e4:f4:0f:0f:2e:3e:49:0b:5f:04:94:36:49:fa:72:42:c9: - 25:75:84:9a:dc:16:cb:69:44:44:e5:3a:ff:26:f6:44:42:4c: - 6c:e2:56:d6:3e:bc:f2:8b:83:de:e2:91:70:65:b9:d0:dd:a3: - d1:de:53:27:77:13:2d:86:27:c3:40:2f:c1:a5:50:1c:5a:44: - 51:b4:29:11:c3:30:9d:1a:96:25:7a:d6:05:70:ad:06:0d:f2: - 9b:b1:b6:82:39:06:c7:7c:b2:49:04:19:e4:7e:87:b8:d8:42: - 1d:ab:ed:d0:b0:7f:79:6b:89:75:2f:6a:26:67:3d:33:57:5f: - 5a:49:52:98:3b:2a:e5:43:d7:f9:97:ca:75:cd:6f:e9:e4:66: - b6:d6:c2:c7 + 93:72:f7:a5:ba:a1:ba:0f:50:d4:cd:c4:63:a6:5d:1f:9e:62: + 8c:87:45:05:78:f7:27:c2:e3:1c:b3:eb:a2:00:88:f4:77:4d: + 9e:f8:cc:26:0c:fd:03:56:fc:d3:23:59:93:69:46:6f:72:94: + 4f:b9:ba:2f:d5:66:f8:ed:00:89:e9:e7:87:fa:0e:5a:9d:1b: + b9:f7:0f:dd:c6:ab:83:1d:f9:5d:1a:8a:f9:0e:34:f4:85:2c: + 69:cd:37:44:93:ab:6b:4b:14:0f:a2:72:56:0b:82:60:47:82: + 50:ba:5c:f6:2f:3d:da:f8:42:40:39:f8:4a:bf:f1:30:ec:7f: + bd:5d:ce:4a:0e:15:3b:ca:d8:12:8e:da:58:f0:d5:b2:a0:82: + d2:2b:60:21:10:57:a5:73:30:4c:67:82:32:fe:2a:d4:b1:87: + cf:33:bb:9e:c4:9c:2d:ce:99:d5:9f:9f:30:3e:5f:f4:49:40: + d0:1a:6e:90:ab:88:d6:c3:f6:36:11:52:e1:97:ea:3a:ce:ee: + 49:04:57:6d:5c:6f:a6:ca:21:21:7e:4d:0a:b6:7a:c5:14:f2: + 39:00:59:de:88:30:74:f2:46:a8:95:a4:19:a4:0f:bd:b5:b0: + f1:41:c8:8b:70:c7:67:a8:d0:b2:3b:21:4f:73:fd:16:e6:1d: + a0:71:b0:33 -----BEGIN CERTIFICATE----- -MIIDDzCCAfegAwIBAgIUYeYbB5BqT/fNRrlZHT4cOQ3yXgQwDQYJKoZIhvcNAQEL -BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz -MzgyNFowUjELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMRYwFAYDVQQKEw1BcGFj -aGUgUHVsc2FyMQ4wDAYDVQQLEwVQcm94eTEOMAwGA1UEAxMFUHJveHkwggEiMA0G -CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDDXMWtF9z01MTqHGBaJEYT2c/AzYMu -L4Jw5eCNM72Vtc/G8FTVjb2HDWJsHT9SZnT/BjMcPNXtLmPZlsbxmILHlEq8ZPKb -OlTsgZm8FIJDhwxr2gOMqgtB1/4nxPmIgTSx/yrgbdBH3cERpVSpUzLNj/Z1WI4F -5NmxrGn+tlTDrTYEonf1U7Z0g9VqAeCWtaKvUI+1152nwr34MYYJX3wKsts04YAl -F199b4vcjtX5z8/19o9q/j6WAMlWsNDjRt65popem45/6hnMolt1IjwdNkjk8hoB -lWHB8HonnYOWdMypBEIIUzSYLrfjg/nyoynhI8TtoBz2Ku3cwN+XqfONAgMBAAGj -HjAcMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOC -AQEAjbYsX4cTBqhmzhEqLCAex+5QdafRfK3G7NEY0PqqAPoI+Q/M31maaxwYBxWE -0Jokjd1GeZzcnj6XECSyndT2xXlYh3ymr89pI/tDeg9NJuDpZsWt+ojixW5qznAM -j3MB1v2pHzFJQRdFIsymceT0Dw8uPkkLXwSUNkn6ckLJJXWEmtwWy2lEROU6/yb2 -REJMbOJW1j688ouD3uKRcGW50N2j0d5TJ3cTLYYnw0AvwaVQHFpEUbQpEcMwnRqW -JXrWBXCtBg3ym7G2gjkGx3yySQQZ5H6HuNhCHavt0LB/eWuJdS9qJmc9M1dfWklS -mDsq5UPX+ZfKdc1v6eRmttbCxw== +MIIDBDCCAeygAwIBAgIJANegMncDqPw7MA0GCSqGSIb3DQEBCwUAMBExDzANBgNV +BAMMBkNBUm9vdDAeFw0yMzAyMjIwNjI2MzJaFw0zMzAyMTkwNjI2MzJaMFIxCzAJ +BgNVBAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEChMNQXBhY2hlIFB1bHNhcjEO +MAwGA1UECxMFUHJveHkxDjAMBgNVBAMTBVByb3h5MIIBIjANBgkqhkiG9w0BAQEF +AAOCAQ8AMIIBCgKCAQEAw1zFrRfc9NTE6hxgWiRGE9nPwM2DLi+CcOXgjTO9lbXP +xvBU1Y29hw1ibB0/UmZ0/wYzHDzV7S5j2ZbG8ZiCx5RKvGTymzpU7IGZvBSCQ4cM +a9oDjKoLQdf+J8T5iIE0sf8q4G3QR93BEaVUqVMyzY/2dViOBeTZsaxp/rZUw602 +BKJ39VO2dIPVagHglrWir1CPtdedp8K9+DGGCV98CrLbNOGAJRdffW+L3I7V+c/P +9faPav4+lgDJVrDQ40beuaaKXpuOf+oZzKJbdSI8HTZI5PIaAZVhwfB6J52DlnTM +qQRCCFM0mC6344P58qMp4SPE7aAc9irt3MDfl6nzjQIDAQABox4wHDAaBgNVHREE +EzARgglsb2NhbGhvc3SHBH8AAAEwDQYJKoZIhvcNAQELBQADggEBAJNy96W6oboP +UNTNxGOmXR+eYoyHRQV49yfC4xyz66IAiPR3TZ74zCYM/QNW/NMjWZNpRm9ylE+5 +ui/VZvjtAInp54f6DlqdG7n3D93Gq4Md+V0aivkONPSFLGnNN0STq2tLFA+iclYL +gmBHglC6XPYvPdr4QkA5+Eq/8TDsf71dzkoOFTvK2BKO2ljw1bKggtIrYCEQV6Vz +MExngjL+KtSxh88zu57EnC3OmdWfnzA+X/RJQNAabpCriNbD9jYRUuGX6jrO7kkE +V21cb6bKISF+TQq2esUU8jkAWd6IMHTyRqiVpBmkD721sPFByItwx2eo0LI7IU9z +/RbmHaBxsDM= -----END CERTIFICATE----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls/cacert.pem b/pulsar-proxy/src/test/resources/authentication/tls/cacert.pem index 127f56dd777a5..2a71f0e3afc17 100644 --- a/pulsar-proxy/src/test/resources/authentication/tls/cacert.pem +++ b/pulsar-proxy/src/test/resources/authentication/tls/cacert.pem @@ -1,77 +1,78 @@ Certificate: Data: Version: 3 (0x2) - Serial Number: - 77:4f:f6:cf:99:ca:77:e8:a7:6e:1e:fd:e2:cf:ac:a9:da:68:d2:42 - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN = CARoot + Serial Number: 15559223195710621847 (0xd7ed770f69598897) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=CARoot Validity - Not Before: May 30 13:38:24 2022 GMT - Not After : May 27 13:38:24 2032 GMT - Subject: CN = CARoot + Not Before: Feb 22 06:26:32 2023 GMT + Not After : Feb 19 06:26:32 2033 GMT + Subject: CN=CARoot Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (2048 bit) + Public-Key: (2048 bit) Modulus: - 00:b8:5e:c2:60:ed:c4:ee:3c:5b:ab:fc:64:52:f3: - 30:41:fc:10:5a:ac:a6:9b:0a:93:d0:d0:c9:bf:96: - 14:a7:cf:5c:3e:23:91:7e:54:ec:fe:2d:9f:c9:34: - d1:4e:95:2f:85:9c:cc:be:90:a3:a4:cb:4d:a4:72: - d2:84:e0:c7:42:c4:bf:70:b6:fa:d2:45:8b:83:66: - 1e:a4:e9:0e:06:a3:46:ea:a7:18:cd:33:b9:f1:ff: - 76:91:72:8f:cd:f9:93:43:c3:6e:17:1f:2d:86:df: - b6:fb:2d:d6:be:2d:98:ad:de:00:c7:de:f9:68:b5: - 40:40:56:49:ae:23:e5:a1:3b:5f:15:5a:44:50:da: - fb:02:d3:42:c6:87:0d:c0:8d:3a:e6:e2:aa:73:31: - ab:79:58:51:cd:03:80:f3:12:ce:2f:35:04:8b:39: - 5f:b0:cc:b8:41:99:47:c1:17:96:8b:c2:44:84:b5: - 21:8a:15:52:fe:1a:5a:f9:88:cc:11:17:ee:48:dd: - ba:bf:ed:67:6e:27:35:42:cf:07:5e:b1:8b:81:55: - 92:01:8e:61:fd:8e:82:74:b1:70:7a:3d:52:1f:16: - 78:12:bb:b5:09:62:ce:6d:18:4a:e9:f5:27:19:bc: - 93:4e:ed:dd:53:a8:c1:bb:48:b7:18:20:7b:79:48: - 48:9d + 00:bf:3e:e6:db:fd:a2:6c:b9:45:29:e8:d3:90:4d: + 78:29:ef:fc:67:c4:e0:ae:06:fc:f8:2b:cb:3b:4b: + 89:cf:37:3c:ab:86:94:59:c3:50:54:4b:18:d5:8b: + 14:f5:cd:36:58:c6:ac:a1:67:f7:04:58:58:2f:e0: + 89:73:8b:b1:ef:97:a4:16:10:97:e6:6f:2f:18:b9: + 8c:93:7b:7c:5b:4f:8d:09:49:aa:70:59:e5:3b:fa: + c0:b9:4a:ed:14:98:0a:5f:56:b3:49:0a:4d:c0:22: + 1e:75:3d:ba:f9:19:da:68:80:18:ad:b7:8f:de:fd: + 1f:60:33:86:74:46:e0:b7:7a:84:5e:b7:af:5b:57: + 3c:93:ad:37:83:2c:1b:e0:77:a3:84:da:25:1d:16: + 77:3f:25:b1:90:49:28:a6:c8:cf:bc:e4:b9:27:85: + f9:36:4f:47:81:cd:56:26:41:23:10:8f:36:26:ce: + 78:b9:ab:45:ce:9c:eb:a3:2a:11:93:ae:b7:d4:d7: + 57:e9:97:71:5b:fa:ca:0e:71:34:43:87:bb:c0:8e: + 68:f4:4d:c8:64:45:02:5d:81:bd:3b:47:bc:ec:4e: + e4:61:e3:c4:16:f0:ef:83:fd:06:5c:05:50:d8:50: + 41:dc:d6:62:6d:b2:26:7b:d3:6b:f6:da:59:4c:a8: + db:b5 Exponent: 65537 (0x10001) X509v3 extensions: + X509v3 Basic Constraints: critical + CA:TRUE X509v3 Subject Key Identifier: - 0F:46:61:3E:6F:71:22:E6:1F:32:37:7C:B2:81:A6:CC:DB:9D:F5:7C + F9:B2:AD:C4:24:62:47:3C:1A:58:AA:66:D0:91:12:F3:20:EE:A9:6C X509v3 Authority Key Identifier: - keyid:0F:46:61:3E:6F:71:22:E6:1F:32:37:7C:B2:81:A6:CC:DB:9D:F5:7C + keyid:F9:B2:AD:C4:24:62:47:3C:1A:58:AA:66:D0:91:12:F3:20:EE:A9:6C + DirName:/CN=CARoot + serial:D7:ED:77:0F:69:59:88:97 - X509v3 Basic Constraints: critical - CA:TRUE Signature Algorithm: sha256WithRSAEncryption - 91:e8:d8:c4:32:2e:80:5c:d4:cb:24:7a:81:43:a9:c7:95:90: - 1a:2e:7a:d3:0c:5d:b6:21:05:67:4d:98:5a:0d:71:ea:80:01: - 95:42:fe:fa:f1:7c:dc:bd:76:ff:05:26:3b:f0:94:b3:09:2c: - 34:dd:43:56:46:2b:15:35:99:d9:94:54:22:cf:a6:68:b0:d1: - 79:e2:f0:9f:0b:02:7c:cf:1f:bd:d0:f6:49:c6:82:28:a5:c6: - ae:94:65:cf:fd:ad:a8:6c:c2:17:da:db:f3:be:30:1a:1b:b4: - 2c:fa:08:71:9d:64:09:45:02:92:02:ad:eb:15:47:14:43:5b: - a8:2d:1a:ec:14:93:dc:ff:bb:51:33:a3:d5:4d:e2:77:ca:e1: - a5:98:5c:7a:b6:10:19:d3:d7:f5:14:a5:d5:08:f1:97:18:3d: - 5f:a6:4e:a2:4a:0d:4b:d4:bb:56:6b:a8:44:35:62:c5:d8:c6: - 67:11:93:1c:22:64:3e:aa:15:08:dc:87:39:dd:f6:e0:a0:d5: - 00:db:27:79:3d:f4:35:7c:46:a9:fa:0c:fa:fc:74:f5:bf:f4: - fe:71:40:45:33:22:35:83:f7:1a:96:2a:fc:b2:33:e0:1a:e8: - 24:48:91:5d:90:5c:4c:93:33:4c:40:de:26:bb:24:ac:48:9b: - ae:fe:19:34 + bc:24:05:65:56:87:f9:87:f4:1b:5a:a3:a0:01:c6:58:c0:7c: + ca:c9:ee:1d:0c:d5:6b:47:28:dc:bd:2f:f1:73:03:e1:a5:08: + 05:56:11:29:d5:48:32:23:d3:7d:46:db:20:58:29:78:19:af: + 6a:a2:d1:b6:6e:30:a0:a3:b1:88:c1:b5:1f:f0:0f:63:5e:ab: + 85:5f:09:0a:3c:20:cb:61:3b:91:80:70:4e:1f:c5:37:34:0d: + 1c:9f:b5:de:93:f1:ae:94:ff:7c:76:7d:6a:a7:19:6f:22:34: + bd:66:07:28:59:f3:60:ef:39:8d:bd:9d:2b:82:08:f0:68:aa: + d6:e0:68:f1:f1:d2:c6:c6:61:88:0d:41:56:40:72:14:14:78: + f9:32:45:de:f9:4d:ef:45:32:a0:21:10:5c:76:f5:ee:fd:a9: + 37:34:04:67:94:72:14:54:5f:27:c3:d3:2d:a6:7f:94:4d:a8: + 98:c4:f9:d9:fc:cd:37:92:3c:c8:bb:2d:a1:f6:ea:14:50:08: + 46:38:9a:cd:71:1b:5b:b4:d9:18:88:77:74:dd:ae:df:b1:58: + 52:f2:8b:e7:bb:0b:0c:92:a6:41:d1:b6:17:64:08:09:7f:7d: + c6:f5:c0:11:a7:29:35:9e:23:9d:e2:08:d5:0e:cb:0c:0c:f2: + d0:7e:38:67 -----BEGIN CERTIFICATE----- -MIIDAzCCAeugAwIBAgIUd0/2z5nKd+inbh794s+sqdpo0kIwDQYJKoZIhvcNAQEL -BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz -MzgyNFowETEPMA0GA1UEAwwGQ0FSb290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A -MIIBCgKCAQEAuF7CYO3E7jxbq/xkUvMwQfwQWqymmwqT0NDJv5YUp89cPiORflTs -/i2fyTTRTpUvhZzMvpCjpMtNpHLShODHQsS/cLb60kWLg2YepOkOBqNG6qcYzTO5 -8f92kXKPzfmTQ8NuFx8tht+2+y3Wvi2Yrd4Ax975aLVAQFZJriPloTtfFVpEUNr7 -AtNCxocNwI065uKqczGreVhRzQOA8xLOLzUEizlfsMy4QZlHwReWi8JEhLUhihVS -/hpa+YjMERfuSN26v+1nbic1Qs8HXrGLgVWSAY5h/Y6CdLFwej1SHxZ4Eru1CWLO -bRhK6fUnGbyTTu3dU6jBu0i3GCB7eUhInQIDAQABo1MwUTAdBgNVHQ4EFgQUD0Zh -Pm9xIuYfMjd8soGmzNud9XwwHwYDVR0jBBgwFoAUD0ZhPm9xIuYfMjd8soGmzNud -9XwwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAkejYxDIugFzU -yyR6gUOpx5WQGi560wxdtiEFZ02YWg1x6oABlUL++vF83L12/wUmO/CUswksNN1D -VkYrFTWZ2ZRUIs+maLDReeLwnwsCfM8fvdD2ScaCKKXGrpRlz/2tqGzCF9rb874w -Ghu0LPoIcZ1kCUUCkgKt6xVHFENbqC0a7BST3P+7UTOj1U3id8rhpZhcerYQGdPX -9RSl1Qjxlxg9X6ZOokoNS9S7VmuoRDVixdjGZxGTHCJkPqoVCNyHOd324KDVANsn -eT30NXxGqfoM+vx09b/0/nFARTMiNYP3GpYq/LIz4BroJEiRXZBcTJMzTEDeJrsk -rEibrv4ZNA== +MIIDGjCCAgKgAwIBAgIJANftdw9pWYiXMA0GCSqGSIb3DQEBCwUAMBExDzANBgNV +BAMMBkNBUm9vdDAeFw0yMzAyMjIwNjI2MzJaFw0zMzAyMTkwNjI2MzJaMBExDzAN +BgNVBAMMBkNBUm9vdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL8+ +5tv9omy5RSno05BNeCnv/GfE4K4G/PgryztLic83PKuGlFnDUFRLGNWLFPXNNljG +rKFn9wRYWC/giXOLse+XpBYQl+ZvLxi5jJN7fFtPjQlJqnBZ5Tv6wLlK7RSYCl9W +s0kKTcAiHnU9uvkZ2miAGK23j979H2AzhnRG4Ld6hF63r1tXPJOtN4MsG+B3o4Ta +JR0Wdz8lsZBJKKbIz7zkuSeF+TZPR4HNViZBIxCPNibOeLmrRc6c66MqEZOut9TX +V+mXcVv6yg5xNEOHu8COaPRNyGRFAl2BvTtHvOxO5GHjxBbw74P9BlwFUNhQQdzW +Ym2yJnvTa/baWUyo27UCAwEAAaN1MHMwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E +FgQU+bKtxCRiRzwaWKpm0JES8yDuqWwwQQYDVR0jBDowOIAU+bKtxCRiRzwaWKpm +0JES8yDuqWyhFaQTMBExDzANBgNVBAMMBkNBUm9vdIIJANftdw9pWYiXMA0GCSqG +SIb3DQEBCwUAA4IBAQC8JAVlVof5h/QbWqOgAcZYwHzKye4dDNVrRyjcvS/xcwPh +pQgFVhEp1UgyI9N9RtsgWCl4Ga9qotG2bjCgo7GIwbUf8A9jXquFXwkKPCDLYTuR +gHBOH8U3NA0cn7Xek/GulP98dn1qpxlvIjS9ZgcoWfNg7zmNvZ0rggjwaKrW4Gjx +8dLGxmGIDUFWQHIUFHj5MkXe+U3vRTKgIRBcdvXu/ak3NARnlHIUVF8nw9Mtpn+U +TaiYxPnZ/M03kjzIuy2h9uoUUAhGOJrNcRtbtNkYiHd03a7fsVhS8ovnuwsMkqZB +0bYXZAgJf33G9cARpyk1niOd4gjVDssMDPLQfjhn -----END CERTIFICATE----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls/client-cert.pem b/pulsar-proxy/src/test/resources/authentication/tls/client-cert.pem index 192d686246f1a..12f8d1fcea0ac 100644 --- a/pulsar-proxy/src/test/resources/authentication/tls/client-cert.pem +++ b/pulsar-proxy/src/test/resources/authentication/tls/client-cert.pem @@ -1,17 +1,16 @@ Certificate: Data: Version: 3 (0x2) - Serial Number: - 61:e6:1b:07:90:6a:4f:f7:cd:46:b9:59:1d:3e:1c:39:0d:f2:5e:01 - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN = CARoot + Serial Number: 15537474201172114488 (0xd7a0327703a8fc38) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=CARoot Validity - Not Before: May 30 13:38:24 2022 GMT - Not After : May 27 13:38:24 2032 GMT - Subject: C = US, ST = CA, O = Apache, OU = Apache Pulsar, CN = superUser + Not Before: Feb 22 06:26:32 2023 GMT + Not After : Feb 19 06:26:32 2033 GMT + Subject: C=US, ST=CA, O=Apache, OU=Apache Pulsar, CN=superUser Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (2048 bit) + Public-Key: (2048 bit) Modulus: 00:cd:43:7d:98:40:f9:b0:5b:bc:ae:db:c0:0b:ad: 26:90:96:e0:62:38:ed:68:b1:70:46:3b:de:44:f9: @@ -36,37 +35,37 @@ Certificate: X509v3 Subject Alternative Name: DNS:localhost, IP Address:127.0.0.1 Signature Algorithm: sha256WithRSAEncryption - 96:c2:23:2d:46:d0:3d:23:0e:ab:3d:b6:1e:31:96:00:eb:ae: - 17:ac:6e:c0:d4:1a:8d:0f:36:63:27:02:49:4e:24:cf:d3:80: - 88:3a:4f:d0:f1:e5:1c:df:2d:8a:ab:ae:8d:48:77:a0:d0:dc: - d5:80:1c:a1:3d:0d:49:64:bf:cb:39:84:c9:f3:5d:e0:2d:ba: - a0:f2:ac:03:85:44:a1:97:6b:0b:de:ed:a7:49:19:46:b2:18: - 49:21:62:43:52:36:6f:47:6c:21:6b:5e:41:85:28:71:6c:22: - 27:35:76:82:ed:ac:ad:d7:fa:9d:4c:7d:6f:44:7e:06:dd:8a: - 11:32:0c:d9:d0:f6:63:2a:40:ae:0d:5a:df:9e:d7:91:8a:db: - 2d:95:f3:19:f0:8f:1e:34:e3:b2:31:67:38:74:fd:3f:e6:49: - 5e:53:eb:88:ae:b1:45:71:0e:67:97:3c:99:4e:c7:ea:1e:02: - 67:b4:54:ef:4f:10:55:4a:70:c0:eb:41:e4:50:d4:48:5e:70: - c5:0f:79:f2:06:3d:35:ea:ce:5d:13:8e:14:65:fc:98:21:16: - 2d:5d:6d:f8:e0:6b:c7:c6:e4:8a:ca:c9:38:1f:93:27:86:28: - ef:96:e7:ad:6c:4a:9e:10:78:48:00:f4:4a:43:dc:87:1d:e3: - d3:39:53:68 + 3b:bd:d4:39:37:b3:a8:bb:34:9f:94:c2:a0:b6:be:89:c4:1f: + 02:0c:b4:08:11:6d:8f:ff:d0:92:2a:a0:91:d9:f9:b0:a8:22: + d1:cf:7a:f3:6b:a9:b3:ac:1c:21:47:61:09:07:5c:a1:c1:4f: + 5f:14:df:ab:9b:1d:10:bf:7f:b5:20:70:51:f9:4a:6d:ae:bb: + a4:14:86:36:b8:29:1d:28:36:9c:86:45:17:0b:b1:8b:4f:1d: + 10:f9:e1:12:1e:61:f0:88:1f:b2:2e:f8:e9:d7:2f:b7:59:98: + ec:50:96:49:11:4d:3d:30:1b:50:82:41:dd:96:11:eb:f9:4d: + 1e:af:52:9a:3c:59:65:ed:b6:db:dd:98:84:9a:f6:75:ab:a1: + ab:69:a3:6d:b4:db:f3:55:05:29:fa:91:d6:bc:60:8a:9e:8b: + 38:e2:18:18:a6:b3:9f:cc:4e:d8:26:a6:7b:29:d6:52:4d:84: + 33:6a:71:b1:35:c2:6e:cd:05:44:3b:67:bc:1a:55:86:ba:b3: + 3b:80:21:76:ed:93:ce:e3:3d:c4:28:9e:a5:4d:f4:f2:17:9a: + e8:be:e5:2d:ae:3a:49:54:0f:8d:fd:e2:65:9c:f5:ea:14:1e: + 9f:2a:fd:8d:59:7b:bf:51:72:a2:0c:85:0c:b7:e6:4f:e0:f5: + f9:06:94:4b -----BEGIN CERTIFICATE----- -MIIDFDCCAfygAwIBAgIUYeYbB5BqT/fNRrlZHT4cOQ3yXgEwDQYJKoZIhvcNAQEL -BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz -MzgyNFowVzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMQ8wDQYDVQQKEwZBcGFj -aGUxFjAUBgNVBAsTDUFwYWNoZSBQdWxzYXIxEjAQBgNVBAMTCXN1cGVyVXNlcjCC -ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAM1DfZhA+bBbvK7bwAutJpCW -4GI47WixcEY73kT5FFGGEOvKkOeI6PmRheDdtbQUuXjjhtVUbWjsFJK0+CJbBT3t -MSVlCAWEyuYMIRJYMscaYKNP0kqeKBl8RYQAjInc3orlT4iRzKTxgUVMfcL/4sGJ -xhJzleI2vduui1poapBR3iuIX6pn9KjjY9y+GYLMnX/mjfuCviIBPVYTO1sEtOjF -GOYuDfq6So3oxlqhUZpKYtev3bT84tXNrplsXGFWC9cMGndc9TpqVLWeM6ypdSia -dq/QelcAG5ETMf1CiCFHBRABL1m7xzrZ4VhMG2xxtpjv3QOCWKMy3JChtqYe4QsC -AwEAAaMeMBwwGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEB -CwUAA4IBAQCWwiMtRtA9Iw6rPbYeMZYA664XrG7A1BqNDzZjJwJJTiTP04CIOk/Q -8eUc3y2Kq66NSHeg0NzVgByhPQ1JZL/LOYTJ813gLbqg8qwDhUShl2sL3u2nSRlG -shhJIWJDUjZvR2wha15BhShxbCInNXaC7ayt1/qdTH1vRH4G3YoRMgzZ0PZjKkCu -DVrfnteRitstlfMZ8I8eNOOyMWc4dP0/5kleU+uIrrFFcQ5nlzyZTsfqHgJntFTv -TxBVSnDA60HkUNRIXnDFD3nyBj016s5dE44UZfyYIRYtXW344GvHxuSKysk4H5Mn -hijvluetbEqeEHhIAPRKQ9yHHePTOVNo +MIIDCTCCAfGgAwIBAgIJANegMncDqPw4MA0GCSqGSIb3DQEBCwUAMBExDzANBgNV +BAMMBkNBUm9vdDAeFw0yMzAyMjIwNjI2MzJaFw0zMzAyMTkwNjI2MzJaMFcxCzAJ +BgNVBAYTAlVTMQswCQYDVQQIEwJDQTEPMA0GA1UEChMGQXBhY2hlMRYwFAYDVQQL +Ew1BcGFjaGUgUHVsc2FyMRIwEAYDVQQDEwlzdXBlclVzZXIwggEiMA0GCSqGSIb3 +DQEBAQUAA4IBDwAwggEKAoIBAQDNQ32YQPmwW7yu28ALrSaQluBiOO1osXBGO95E ++RRRhhDrypDniOj5kYXg3bW0FLl444bVVG1o7BSStPgiWwU97TElZQgFhMrmDCES +WDLHGmCjT9JKnigZfEWEAIyJ3N6K5U+Ikcyk8YFFTH3C/+LBicYSc5XiNr3brota +aGqQUd4riF+qZ/So42PcvhmCzJ1/5o37gr4iAT1WEztbBLToxRjmLg36ukqN6MZa +oVGaSmLXr920/OLVza6ZbFxhVgvXDBp3XPU6alS1njOsqXUomnav0HpXABuREzH9 +QoghRwUQAS9Zu8c62eFYTBtscbaY790DglijMtyQobamHuELAgMBAAGjHjAcMBoG +A1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEAO73U +OTezqLs0n5TCoLa+icQfAgy0CBFtj//Qkiqgkdn5sKgi0c9682ups6wcIUdhCQdc +ocFPXxTfq5sdEL9/tSBwUflKba67pBSGNrgpHSg2nIZFFwuxi08dEPnhEh5h8Igf +si746dcvt1mY7FCWSRFNPTAbUIJB3ZYR6/lNHq9SmjxZZe22292YhJr2dauhq2mj +bbTb81UFKfqR1rxgip6LOOIYGKazn8xO2CameynWUk2EM2pxsTXCbs0FRDtnvBpV +hrqzO4Ahdu2TzuM9xCiepU308hea6L7lLa46SVQPjf3iZZz16hQenyr9jVl7v1Fy +ogyFDLfmT+D1+QaUSw== -----END CERTIFICATE----- diff --git a/pulsar-proxy/src/test/resources/authentication/tls/server-cert.pem b/pulsar-proxy/src/test/resources/authentication/tls/server-cert.pem index c09434c85d20a..333e1b9b80ac1 100644 --- a/pulsar-proxy/src/test/resources/authentication/tls/server-cert.pem +++ b/pulsar-proxy/src/test/resources/authentication/tls/server-cert.pem @@ -1,17 +1,16 @@ Certificate: Data: Version: 3 (0x2) - Serial Number: - 61:e6:1b:07:90:6a:4f:f7:cd:46:b9:59:1d:3e:1c:39:0d:f2:5e:02 - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN = CARoot + Serial Number: 15537474201172114489 (0xd7a0327703a8fc39) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=CARoot Validity - Not Before: May 30 13:38:24 2022 GMT - Not After : May 27 13:38:24 2032 GMT - Subject: C = US, ST = CA, O = Apache, OU = Apache Pulsar, CN = localhost + Not Before: Feb 22 06:26:32 2023 GMT + Not After : Feb 19 06:26:32 2033 GMT + Subject: C=US, ST=CA, O=Apache, OU=Apache Pulsar, CN=localhost Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (2048 bit) + Public-Key: (2048 bit) Modulus: 00:af:bf:b7:2d:98:ad:9d:f6:da:a3:13:d4:62:0f: 98:be:1c:a2:89:22:ba:6f:d5:fd:1f:67:e3:91:03: @@ -36,37 +35,37 @@ Certificate: X509v3 Subject Alternative Name: DNS:localhost, IP Address:127.0.0.1 Signature Algorithm: sha256WithRSAEncryption - 88:89:d7:52:b3:61:49:73:7d:ee:aa:6f:47:11:cd:52:f1:ef: - 9a:63:5f:43:a9:4f:66:c8:36:dd:44:24:ba:4f:c3:6c:94:90: - 85:5e:29:fb:65:cf:03:3b:37:16:5e:88:07:70:97:54:93:f0: - f3:09:d7:65:60:09:00:fd:7f:dd:6a:ab:25:3a:30:c4:89:34: - 43:82:f6:f5:f4:2d:39:3d:21:90:c4:00:27:c5:6a:23:41:20: - c6:42:35:56:91:17:fa:31:90:09:6a:4c:e4:a7:53:ae:61:b6: - d3:5b:82:71:08:d0:0b:af:34:0f:9b:bd:bc:8c:1c:31:43:43: - 97:82:9a:ac:2a:53:ca:11:ce:6f:64:ac:86:c1:f0:62:14:aa: - c3:dd:15:5b:1c:02:6f:bb:40:87:17:b7:e5:9d:93:9a:51:c9: - 1e:7a:8c:d1:22:75:44:f1:9d:90:4b:3e:1f:6c:ab:6f:e3:be: - cd:c7:15:9d:04:84:4a:1b:a7:ac:64:5d:d7:3e:23:98:b9:49: - dd:85:dd:80:4c:46:08:9b:f5:df:eb:19:c8:57:70:ac:43:f9: - d6:9c:1b:1b:2a:94:cf:c1:35:56:a2:f4:b1:00:5d:9e:1e:36: - 54:72:ab:aa:ef:49:b2:f0:dc:cf:5b:22:51:bf:e4:c9:57:dc: - d0:48:0d:f2 + 84:4c:f6:9f:40:bd:44:a2:52:f8:62:62:98:a3:8c:78:17:fb: + da:71:cb:ca:21:34:b5:98:22:88:31:12:56:7a:d3:f2:88:2c: + fe:4c:ea:b5:bc:40:f9:5b:cf:06:6d:bd:58:3f:d9:69:99:54: + e6:5d:3a:6a:4f:92:3b:02:0f:15:01:99:d4:01:86:8f:09:c3: + a8:b6:f0:c1:21:55:9b:25:c1:2a:73:ee:b5:9b:2c:97:e6:9e: + a5:f7:b6:52:4a:6a:51:13:06:1b:5a:47:13:2c:ac:26:44:05: + 44:be:47:03:33:3d:15:fc:17:91:f2:2a:44:7d:cc:b1:3c:ac: + 31:ee:48:e9:3d:1d:a6:5f:d3:60:6f:d8:e5:1c:e4:bc:0d:a4: + dc:8d:4b:4f:e2:e4:87:fe:56:00:67:86:2b:61:c1:e0:da:eb: + 57:56:d1:43:24:15:4c:8a:4a:ac:31:74:ab:46:3e:a6:6e:f6: + 3a:09:c8:bb:ae:1c:ff:17:c1:2a:33:2c:e2:0f:d4:25:71:bc: + 9b:51:28:2f:c4:bb:44:67:86:81:2d:21:f1:22:54:e8:45:09: + 39:7b:e2:19:f6:85:0d:76:c8:2a:ca:a6:e1:d2:c5:f4:49:fe: + 02:f3:8d:cc:e6:23:19:4b:b8:f4:e4:76:91:a9:6d:5a:30:0e: + 7f:00:cb:93 -----BEGIN CERTIFICATE----- -MIIDFDCCAfygAwIBAgIUYeYbB5BqT/fNRrlZHT4cOQ3yXgIwDQYJKoZIhvcNAQEL -BQAwETEPMA0GA1UEAwwGQ0FSb290MB4XDTIyMDUzMDEzMzgyNFoXDTMyMDUyNzEz -MzgyNFowVzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMQ8wDQYDVQQKEwZBcGFj -aGUxFjAUBgNVBAsTDUFwYWNoZSBQdWxzYXIxEjAQBgNVBAMTCWxvY2FsaG9zdDCC -ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK+/ty2YrZ322qMT1GIPmL4c -ookium/V/R9n45EDmICBDu3Y9nB/LDZoPVPqWDqm1YlmS70eV3ETbUsR5UCldoQk -kkBYgJbJHyzEVeujeXNwXDeaie0vumvjgnxpSgJUi4FePL9MisvqLF6D57cQCF+C -WKOJ0dqSuioo7jAoP1uuEHGWx+ESxbAarURvRDoRSpo8D40GgHs07z9s9F7FRFQe -yN3HgIWA2WjmxlMDd+H+GGEHdwVM7Vm8XUE4au9dobJgmNRIKJUCig79z3sb0hHM -EAxQc9fMOGyD3XkmqpDIm4SGvFnpYmn0mBvEgHh+oBqBndLhZt3EzPxjBKzspzUC -AwEAAaMeMBwwGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEB -CwUAA4IBAQCIiddSs2FJc33uqm9HEc1S8e+aY19DqU9myDbdRCS6T8NslJCFXin7 -Zc8DOzcWXogHcJdUk/DzCddlYAkA/X/daqslOjDEiTRDgvb19C05PSGQxAAnxWoj -QSDGQjVWkRf6MZAJakzkp1OuYbbTW4JxCNALrzQPm728jBwxQ0OXgpqsKlPKEc5v -ZKyGwfBiFKrD3RVbHAJvu0CHF7flnZOaUckeeozRInVE8Z2QSz4fbKtv477NxxWd -BIRKG6esZF3XPiOYuUndhd2ATEYIm/Xf6xnIV3CsQ/nWnBsbKpTPwTVWovSxAF2e -HjZUcquq70my8NzPWyJRv+TJV9zQSA3y +MIIDCTCCAfGgAwIBAgIJANegMncDqPw5MA0GCSqGSIb3DQEBCwUAMBExDzANBgNV +BAMMBkNBUm9vdDAeFw0yMzAyMjIwNjI2MzJaFw0zMzAyMTkwNjI2MzJaMFcxCzAJ +BgNVBAYTAlVTMQswCQYDVQQIEwJDQTEPMA0GA1UEChMGQXBhY2hlMRYwFAYDVQQL +Ew1BcGFjaGUgUHVsc2FyMRIwEAYDVQQDEwlsb2NhbGhvc3QwggEiMA0GCSqGSIb3 +DQEBAQUAA4IBDwAwggEKAoIBAQCvv7ctmK2d9tqjE9RiD5i+HKKJIrpv1f0fZ+OR +A5iAgQ7t2PZwfyw2aD1T6lg6ptWJZku9HldxE21LEeVApXaEJJJAWICWyR8sxFXr +o3lzcFw3montL7pr44J8aUoCVIuBXjy/TIrL6ixeg+e3EAhfglijidHakroqKO4w +KD9brhBxlsfhEsWwGq1Eb0Q6EUqaPA+NBoB7NO8/bPRexURUHsjdx4CFgNlo5sZT +A3fh/hhhB3cFTO1ZvF1BOGrvXaGyYJjUSCiVAooO/c97G9IRzBAMUHPXzDhsg915 +JqqQyJuEhrxZ6WJp9JgbxIB4fqAagZ3S4WbdxMz8YwSs7Kc1AgMBAAGjHjAcMBoG +A1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEAhEz2 +n0C9RKJS+GJimKOMeBf72nHLyiE0tZgiiDESVnrT8ogs/kzqtbxA+VvPBm29WD/Z +aZlU5l06ak+SOwIPFQGZ1AGGjwnDqLbwwSFVmyXBKnPutZssl+aepfe2UkpqURMG +G1pHEyysJkQFRL5HAzM9FfwXkfIqRH3MsTysMe5I6T0dpl/TYG/Y5RzkvA2k3I1L +T+Lkh/5WAGeGK2HB4NrrV1bRQyQVTIpKrDF0q0Y+pm72OgnIu64c/xfBKjMs4g/U +JXG8m1EoL8S7RGeGgS0h8SJU6EUJOXviGfaFDXbIKsqm4dLF9En+AvONzOYjGUu4 +9OR2kaltWjAOfwDLkw== -----END CERTIFICATE----- From d4be954dedcc7537b3d65b9a1d7b5662e6062fdf Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Wed, 22 Feb 2023 15:07:18 -0600 Subject: [PATCH 045/174] [fix][broker] Allow proxy to pass same role for authRole and originalRole (#19557) ### Motivation I broke the Pulsar Proxy with #19455 because that PR requires that when `X-Original-Principal` is supplied, the auth role must be a proxy role. This is not always the case for proxied admin requests. This PR seeks to fix that incorrect assumption by changing the way verification is done for the roles. Specifically, when the two roles are the same and they are not a proxy role, we will consider it a valid combination. Note that there is no inefficiency in this solution because When the `authenticatedPrincipal` is not a proxy role, that is the only role that is authenticated. Note also that we do not let the binary protocol authenticate this way, and that is consistent with the way the pulsar proxy forwards authentication data. Currently, we do the following when authentication is enabled in the proxy: 1. Authenticate the client's http request and put the resulting role in the `X-Original-Principal` header for the call to the broker. https://github.com/apache/pulsar/blob/38555851359f9cfc172650c387a58c5a03809e97/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java#L370-L373 2. Copy the `Authorization` header into the broker's http request: https://github.com/apache/pulsar/blob/38555851359f9cfc172650c387a58c5a03809e97/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java#L232-L236 3. Configure the proxy's http client to use client TLS authentication (when configured): https://github.com/apache/pulsar/blob/38555851359f9cfc172650c387a58c5a03809e97/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java#L269-L277 The problem with #19455 is that it assumes the proxy supplies its own authentication data. However, that only happens when using TLS authentication. Otherwise, the proxy forwards the client's authentication data in the `Authorization` header. As such, calls will fail because the `X-Original-Principal` header supplied without using a proxy role. ### Modifications * Consider the `authenticatedPrincipal` and the `originalPrincipal` a valid pair when they are equal and are not a `proxyRole` for http requests. ### Alternative Solutions I initially proposed that we only add the `X-Original-Principal` when we are using the proxy's authentication (see the first commit). I decided this solution is not ideal because it doesn't solve the problem, it doesn't make the brokers backwards compatible, and there isn't actually any inefficiency in passing the role as a header. ### Verifying this change When cherry-picking #19455 to branch-2.9, I discovered that `PackagesOpsWithAuthTest#testPackagesOps` was consistently failing because of the way the proxy supplies authentication data when proxying http requests. That test was removed by https://github.com/apache/pulsar/pull/12771, which explains why I didn't catch the error sooner. This PR includes a test that fails without this change. Note that the primary issue must be that we didn't have any tests doing authentication forwarding through the proxy. Now we will have both relevant tests where the proxy is and is not authenticating. ### Does this pull request potentially affect one of the following parts: This is not a breaking change. ### Documentation - [x] `doc-required` ### Matching PR in forked repository PR in forked repository: https://github.com/michaeljmarshall/pulsar/pull/31 --- .../authorization/AuthorizationService.java | 26 +++- .../pulsar/broker/service/ServerCnx.java | 2 +- .../pulsar/broker/auth/AuthorizationTest.java | 30 ++-- .../pulsar/broker/service/ServerCnxTest.java | 2 + .../server/ProxyWithAuthorizationTest.java | 143 ++++++++++-------- .../server/ProxyWithJwtAuthorizationTest.java | 14 +- 6 files changed, 129 insertions(+), 88 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java index 39f401b493f17..6fff04b33b618 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java @@ -292,23 +292,36 @@ public CompletableFuture allowSinkOpsAsync(NamespaceName namespaceName, return provider.allowSinkOpsAsync(namespaceName, role, authenticationData); } + /** + * Whether the authenticatedPrincipal and the originalPrincipal form a valid pair. This method assumes that + * authenticatedPrincipal and originalPrincipal can be equal, as long as they are not a proxy role. This use + * case is relvant for the admin server because of the way the proxy handles authentication. The binary protocol + * should not use this method. + * @return true when roles are a valid combination and false when roles are an invalid combination + */ public boolean isValidOriginalPrincipal(String authenticatedPrincipal, String originalPrincipal, AuthenticationDataSource authDataSource) { SocketAddress remoteAddress = authDataSource != null ? authDataSource.getPeerAddress() : null; - return isValidOriginalPrincipal(authenticatedPrincipal, originalPrincipal, remoteAddress); + return isValidOriginalPrincipal(authenticatedPrincipal, originalPrincipal, remoteAddress, true); } /** * Validates that the authenticatedPrincipal and the originalPrincipal are a valid combination. - * Valid combinations fulfill the following rule: the authenticatedPrincipal is in - * {@link ServiceConfiguration#getProxyRoles()}, if, and only if, the originalPrincipal is set to a role - * that is not also in {@link ServiceConfiguration#getProxyRoles()}. + * Valid combinations fulfill one of the following two rules: + *

+ * 1. The authenticatedPrincipal is in {@link ServiceConfiguration#getProxyRoles()}, if, and only if, + * the originalPrincipal is set to a role that is not also in {@link ServiceConfiguration#getProxyRoles()}. + *

+ * 2. The authenticatedPrincipal and the originalPrincipal are the same, but are not a proxyRole, when + * allowNonProxyPrincipalsToBeEqual is true. + * * @return true when roles are a valid combination and false when roles are an invalid combination */ public boolean isValidOriginalPrincipal(String authenticatedPrincipal, String originalPrincipal, - SocketAddress remoteAddress) { + SocketAddress remoteAddress, + boolean allowNonProxyPrincipalsToBeEqual) { String errorMsg = null; if (conf.getProxyRoles().contains(authenticatedPrincipal)) { if (StringUtils.isBlank(originalPrincipal)) { @@ -316,7 +329,8 @@ public boolean isValidOriginalPrincipal(String authenticatedPrincipal, } else if (conf.getProxyRoles().contains(originalPrincipal)) { errorMsg = "originalPrincipal cannot be a proxy role."; } - } else if (StringUtils.isNotBlank(originalPrincipal)) { + } else if (StringUtils.isNotBlank(originalPrincipal) + && !(allowNonProxyPrincipalsToBeEqual && originalPrincipal.equals(authenticatedPrincipal))) { errorMsg = "cannot specify originalPrincipal when connecting without valid proxy role."; } if (errorMsg != null) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index d6a6dda402eca..adee29c5a0105 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -673,7 +673,7 @@ private void completeConnect(int clientProtoVersion, String clientVersion) { if (service.isAuthenticationEnabled()) { if (service.isAuthorizationEnabled()) { if (!service.getAuthorizationService() - .isValidOriginalPrincipal(authRole, originalPrincipal, remoteAddress)) { + .isValidOriginalPrincipal(authRole, originalPrincipal, remoteAddress, false)) { state = State.Failed; service.getPulsarStats().recordConnectionCreateFail(); final ByteBuf msg = Commands.newError(-1, ServerError.AuthorizationError, "Invalid roles."); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java index c578d9ec94162..58cf4ee418ea4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java @@ -242,27 +242,31 @@ public void testOriginalRoleValidation() throws Exception { AuthorizationService auth = new AuthorizationService(conf, Mockito.mock(PulsarResources.class)); // Original principal should be supplied when authenticatedPrincipal is proxy role - assertTrue(auth.isValidOriginalPrincipal("proxy", "client", (SocketAddress) null)); + assertTrue(auth.isValidOriginalPrincipal("proxy", "client", (SocketAddress) null, false)); // Non proxy role should not supply originalPrincipal - assertTrue(auth.isValidOriginalPrincipal("client", "", (SocketAddress) null)); - assertTrue(auth.isValidOriginalPrincipal("client", null, (SocketAddress) null)); + assertTrue(auth.isValidOriginalPrincipal("client", "", (SocketAddress) null, false)); + assertTrue(auth.isValidOriginalPrincipal("client", null, (SocketAddress) null, false)); + + // Edge cases that differ because binary protocol and http protocol have different expectations + assertTrue(auth.isValidOriginalPrincipal("client", "client", (SocketAddress) null, true)); + assertFalse(auth.isValidOriginalPrincipal("client", "client", (SocketAddress) null, false)); // Only likely in cases when authentication is disabled, but we still define these to be valid. - assertTrue(auth.isValidOriginalPrincipal(null, null, (SocketAddress) null)); - assertTrue(auth.isValidOriginalPrincipal(null, "", (SocketAddress) null)); - assertTrue(auth.isValidOriginalPrincipal("", null, (SocketAddress) null)); - assertTrue(auth.isValidOriginalPrincipal("", "", (SocketAddress) null)); + assertTrue(auth.isValidOriginalPrincipal(null, null, (SocketAddress) null, false)); + assertTrue(auth.isValidOriginalPrincipal(null, "", (SocketAddress) null, false)); + assertTrue(auth.isValidOriginalPrincipal("", null, (SocketAddress) null, false)); + assertTrue(auth.isValidOriginalPrincipal("", "", (SocketAddress) null, false)); // Proxy role must supply an original principal - assertFalse(auth.isValidOriginalPrincipal("proxy", "", (SocketAddress) null)); - assertFalse(auth.isValidOriginalPrincipal("proxy", null, (SocketAddress) null)); + assertFalse(auth.isValidOriginalPrincipal("proxy", "", (SocketAddress) null, false)); + assertFalse(auth.isValidOriginalPrincipal("proxy", null, (SocketAddress) null, false)); // OriginalPrincipal cannot be proxy role - assertFalse(auth.isValidOriginalPrincipal("proxy", "proxy", (SocketAddress) null)); - assertFalse(auth.isValidOriginalPrincipal("client", "proxy", (SocketAddress) null)); - assertFalse(auth.isValidOriginalPrincipal("", "proxy", (SocketAddress) null)); - assertFalse(auth.isValidOriginalPrincipal(null, "proxy", (SocketAddress) null)); + assertFalse(auth.isValidOriginalPrincipal("proxy", "proxy", (SocketAddress) null, false)); + assertFalse(auth.isValidOriginalPrincipal("client", "proxy", (SocketAddress) null, false)); + assertFalse(auth.isValidOriginalPrincipal("", "proxy", (SocketAddress) null, false)); + assertFalse(auth.isValidOriginalPrincipal(null, "proxy", (SocketAddress) null, false)); // Must gracefully handle a missing AuthenticationDataSource assertTrue(auth.isValidOriginalPrincipal("proxy", "client", (AuthenticationDataSource) null)); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java index ab13b8aa3c7e1..a29d6dac72023 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java @@ -613,6 +613,8 @@ public void testConnectCommandWithInvalidRoleCombinations() throws Exception { verifyAuthRoleAndOriginalPrincipalBehavior(authMethodName, "pass.client", "pass.proxy"); // Invalid combinations where the original principal is set to a non-proxy role verifyAuthRoleAndOriginalPrincipalBehavior(authMethodName, "pass.client1", "pass.client"); + verifyAuthRoleAndOriginalPrincipalBehavior(authMethodName, "pass.client", "pass.client"); + verifyAuthRoleAndOriginalPrincipalBehavior(authMethodName, "pass.client", "pass.client1"); } private void verifyAuthRoleAndOriginalPrincipalBehavior(String authMethodName, String authData, diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationTest.java index de9bb087d3da3..31757cc036720 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationTest.java @@ -84,6 +84,7 @@ public class ProxyWithAuthorizationTest extends ProducerConsumerBase { private final String TLS_SUPERUSER_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem"; private ProxyService proxyService; + private WebServer webServer; private final ProxyConfiguration proxyConfig = new ProxyConfiguration(); @DataProvider(name = "hostnameVerification") @@ -175,6 +176,7 @@ protected void doInitConf() throws Exception { Set superUserRoles = new HashSet<>(); superUserRoles.add("superUser"); + superUserRoles.add("Proxy"); conf.setSuperUserRoles(superUserRoles); conf.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName()); @@ -202,12 +204,11 @@ protected void setup() throws Exception { proxyConfig.setForwardAuthorizationCredentials(true); proxyConfig.setBrokerServiceURL(pulsar.getBrokerServiceUrl()); proxyConfig.setBrokerServiceURLTLS(pulsar.getBrokerServiceUrlTls()); + proxyConfig.setBrokerWebServiceURLTLS(pulsar.getWebServiceAddressTls()); proxyConfig.setAdvertisedAddress(null); - proxyConfig.setServicePort(Optional.of(0)); proxyConfig.setBrokerProxyAllowedTargetPorts("*"); proxyConfig.setServicePortTls(Optional.of(0)); - proxyConfig.setWebServicePort(Optional.of(0)); proxyConfig.setWebServicePortTls(Optional.of(0)); proxyConfig.setTlsEnabledWithBroker(true); @@ -225,9 +226,10 @@ protected void setup() throws Exception { properties.setProperty("tokenSecretKey", AuthTokenUtils.encodeKeyBase64(SECRET_KEY)); proxyConfig.setProperties(properties); - proxyService = Mockito.spy(new ProxyService(proxyConfig, - new AuthenticationService( - PulsarConfigurationLoader.convertFrom(proxyConfig)))); + AuthenticationService authService = + new AuthenticationService(PulsarConfigurationLoader.convertFrom(proxyConfig)); + proxyService = Mockito.spy(new ProxyService(proxyConfig, authService)); + webServer = new WebServer(proxyConfig, authService); } @AfterMethod(alwaysRun = true) @@ -235,10 +237,13 @@ protected void setup() throws Exception { protected void cleanup() throws Exception { super.internalCleanup(); proxyService.close(); + webServer.stop(); } private void startProxy() throws Exception { proxyService.start(); + ProxyServiceStarter.addWebServerHandlers(webServer, proxyConfig, proxyService, null); + webServer.start(); } /** @@ -260,23 +265,15 @@ public void testProxyAuthorization() throws Exception { log.info("-- Starting {} test --", methodName); startProxy(); - createAdminClient(); + // Skip hostname verification because the certs intentionally do not have a hostname + createProxyAdminClient(false); // create a client which connects to proxy over tls and pass authData @Cleanup PulsarClient proxyClient = createPulsarClient(proxyService.getServiceUrlTls(), PulsarClient.builder()); String namespaceName = "my-tenant/my-ns"; - admin.clusters().createCluster("proxy-authorization", ClusterData.builder().serviceUrlTls(brokerUrlTls.toString()).build()); - - admin.tenants().createTenant("my-tenant", - new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("proxy-authorization"))); - admin.namespaces().createNamespace(namespaceName); - - admin.namespaces().grantPermissionOnNamespace(namespaceName, "Proxy", - Sets.newHashSet(AuthAction.consume, AuthAction.produce)); - admin.namespaces().grantPermissionOnNamespace(namespaceName, "Client", - Sets.newHashSet(AuthAction.consume, AuthAction.produce)); + initializeCluster(admin, namespaceName); Consumer consumer = proxyClient.newConsumer() .topic("persistent://my-tenant/my-ns/my-topic1") @@ -313,7 +310,8 @@ public void testTlsHostVerificationProxyToClient(boolean hostnameVerificationEna log.info("-- Starting {} test --", methodName); startProxy(); - createAdminClient(); + // Testing client to proxy hostname verification, so use the dataProvider's value here + createProxyAdminClient(hostnameVerificationEnabled); // create a client which connects to proxy over tls and pass authData @Cleanup PulsarClient proxyClient = createPulsarClient(proxyService.getServiceUrlTls(), @@ -321,17 +319,21 @@ public void testTlsHostVerificationProxyToClient(boolean hostnameVerificationEna String namespaceName = "my-tenant/my-ns"; - admin.clusters().createCluster("proxy-authorization", ClusterData.builder() - .serviceUrlTls(brokerUrlTls.toString()).build()); - - admin.tenants().createTenant("my-tenant", - new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("proxy-authorization"))); - admin.namespaces().createNamespace(namespaceName); - - admin.namespaces().grantPermissionOnNamespace(namespaceName, "Proxy", - Sets.newHashSet(AuthAction.consume, AuthAction.produce)); - admin.namespaces().grantPermissionOnNamespace(namespaceName, "Client", - Sets.newHashSet(AuthAction.consume, AuthAction.produce)); + try { + initializeCluster(admin, namespaceName); + if (hostnameVerificationEnabled) { + Assert.fail("Connection should be failed due to hostnameVerification enabled"); + } + } catch (PulsarAdminException e) { + if (!hostnameVerificationEnabled) { + Assert.fail("Cluster should initialize because hostnameverification is disabled"); + } + admin.close(); + // Need new client because the admin client to proxy is failing due to hostname verification, and we still + // want to test the binary protocol client fails to connect as well + createProxyAdminClient(false); + initializeCluster(admin, namespaceName); + } try { proxyClient.newConsumer().topic("persistent://my-tenant/my-ns/my-topic1") @@ -366,7 +368,8 @@ public void testTlsHostVerificationProxyToBroker(boolean hostnameVerificationEna proxyConfig.setTlsHostnameVerificationEnabled(hostnameVerificationEnabled); startProxy(); - createAdminClient(); + // This test skips hostname verification for client to proxy in order to test proxy to broker + createProxyAdminClient(false); // create a client which connects to proxy over tls and pass authData @Cleanup PulsarClient proxyClient = createPulsarClient(proxyService.getServiceUrlTls(), @@ -374,16 +377,22 @@ public void testTlsHostVerificationProxyToBroker(boolean hostnameVerificationEna String namespaceName = "my-tenant/my-ns"; - admin.clusters().createCluster("proxy-authorization", ClusterData.builder().serviceUrlTls(brokerUrlTls.toString()).build()); - - admin.tenants().createTenant("my-tenant", - new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("proxy-authorization"))); - admin.namespaces().createNamespace(namespaceName); - - admin.namespaces().grantPermissionOnNamespace(namespaceName, "Proxy", - Sets.newHashSet(AuthAction.consume, AuthAction.produce)); - admin.namespaces().grantPermissionOnNamespace(namespaceName, "Client", - Sets.newHashSet(AuthAction.consume, AuthAction.produce)); + try { + initializeCluster(admin, namespaceName); + if (hostnameVerificationEnabled) { + Assert.fail("Connection should be failed due to hostnameVerification enabled for proxy to broker"); + } + } catch (PulsarAdminException.ServerSideErrorException e) { + if (!hostnameVerificationEnabled) { + Assert.fail("Cluster should initialize because hostnameverification is disabled for proxy to broker"); + } + Assert.assertEquals(e.getStatusCode(), 502, "Should get bad gateway"); + admin.close(); + // Need to use broker's admin client because the proxy to broker is failing, and we still want to test + // the binary protocol client fails to connect as well + createBrokerAdminClient(); + initializeCluster(admin, namespaceName); + } try { proxyClient.newConsumer().topic("persistent://my-tenant/my-ns/my-topic1") @@ -411,19 +420,9 @@ public void tlsCiphersAndProtocols(Set tlsCiphers, Set tlsProtoc throws Exception { log.info("-- Starting {} test --", methodName); String namespaceName = "my-tenant/my-ns"; - createAdminClient(); - - admin.clusters().createCluster("proxy-authorization", ClusterData.builder() - .serviceUrlTls(brokerUrlTls.toString()).build()); + createBrokerAdminClient(); - admin.tenants().createTenant("my-tenant", - new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("proxy-authorization"))); - admin.namespaces().createNamespace(namespaceName); - - admin.namespaces().grantPermissionOnNamespace(namespaceName, "Proxy", - Sets.newHashSet(AuthAction.consume, AuthAction.produce)); - admin.namespaces().grantPermissionOnNamespace(namespaceName, "Client", - Sets.newHashSet(AuthAction.consume, AuthAction.produce)); + initializeCluster(admin, namespaceName); ProxyConfiguration proxyConfig = new ProxyConfiguration(); proxyConfig.setAuthenticationEnabled(true); @@ -510,7 +509,8 @@ public void testProxyTlsTransportWithAuth(Authentication auth) throws Exception log.info("-- Starting {} test --", methodName); startProxy(); - createAdminClient(); + // Skip hostname verification because the certs intentionally do not have a hostname + createProxyAdminClient(false); @Cleanup PulsarClient proxyClient = PulsarClient.builder() @@ -525,17 +525,7 @@ public void testProxyTlsTransportWithAuth(Authentication auth) throws Exception String namespaceName = "my-tenant/my-ns"; - admin.clusters().createCluster("proxy-authorization", - ClusterData.builder().serviceUrlTls(brokerUrlTls.toString()).build()); - - admin.tenants().createTenant("my-tenant", - new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("proxy-authorization"))); - admin.namespaces().createNamespace(namespaceName); - - admin.namespaces().grantPermissionOnNamespace(namespaceName, "Proxy", - Sets.newHashSet(AuthAction.consume, AuthAction.produce)); - admin.namespaces().grantPermissionOnNamespace(namespaceName, "Client", - Sets.newHashSet(AuthAction.consume, AuthAction.produce)); + initializeCluster(admin, namespaceName); Consumer consumer = proxyClient.newConsumer() .topic("persistent://my-tenant/my-ns/my-topic1") @@ -567,7 +557,32 @@ public void testProxyTlsTransportWithAuth(Authentication auth) throws Exception log.info("-- Exiting {} test --", methodName); } - private void createAdminClient() throws Exception { + private void initializeCluster(PulsarAdmin adminClient, String namespaceName) throws Exception { + adminClient.clusters().createCluster("proxy-authorization", ClusterData.builder() + .serviceUrlTls(brokerUrlTls.toString()).build()); + + adminClient.tenants().createTenant("my-tenant", + new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("proxy-authorization"))); + adminClient.namespaces().createNamespace(namespaceName); + + adminClient.namespaces().grantPermissionOnNamespace(namespaceName, "Proxy", + Sets.newHashSet(AuthAction.consume, AuthAction.produce)); + adminClient.namespaces().grantPermissionOnNamespace(namespaceName, "Client", + Sets.newHashSet(AuthAction.consume, AuthAction.produce)); + } + + private void createProxyAdminClient(boolean enableTlsHostnameVerification) throws Exception { + Map authParams = Maps.newHashMap(); + authParams.put("tlsCertFile", TLS_SUPERUSER_CLIENT_CERT_FILE_PATH); + authParams.put("tlsKeyFile", TLS_SUPERUSER_CLIENT_KEY_FILE_PATH); + + admin = spy(PulsarAdmin.builder().serviceHttpUrl("https://localhost:" + webServer.getListenPortHTTPS().get()) + .tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH) + .enableTlsHostnameVerification(enableTlsHostnameVerification) + .authentication(AuthenticationTls.class.getName(), authParams).build()); + } + + private void createBrokerAdminClient() throws Exception { Map authParams = Maps.newHashMap(); authParams.put("tlsCertFile", TLS_SUPERUSER_CLIENT_CERT_FILE_PATH); authParams.put("tlsKeyFile", TLS_SUPERUSER_CLIENT_KEY_FILE_PATH); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithJwtAuthorizationTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithJwtAuthorizationTest.java index f42cbe4c30e87..e912006faa022 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithJwtAuthorizationTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithJwtAuthorizationTest.java @@ -72,6 +72,7 @@ public class ProxyWithJwtAuthorizationTest extends ProducerConsumerBase { private final String CLIENT_TOKEN = Jwts.builder().setSubject(CLIENT_ROLE).signWith(SECRET_KEY).compact(); private ProxyService proxyService; + private WebServer webServer; private final ProxyConfiguration proxyConfig = new ProxyConfiguration(); @BeforeMethod @@ -105,6 +106,7 @@ protected void setup() throws Exception { proxyConfig.setAuthorizationEnabled(false); proxyConfig.getProperties().setProperty("tokenSecretKey", "data:;base64," + Base64.getEncoder().encodeToString(SECRET_KEY.getEncoded())); proxyConfig.setBrokerServiceURL(pulsar.getBrokerServiceUrl()); + proxyConfig.setBrokerWebServiceURL(pulsar.getWebServiceAddress()); proxyConfig.setServicePort(Optional.of(0)); proxyConfig.setBrokerProxyAllowedTargetPorts("*"); @@ -115,9 +117,10 @@ protected void setup() throws Exception { proxyConfig.setBrokerClientAuthenticationParameters(PROXY_TOKEN); proxyConfig.setAuthenticationProviders(providers); - proxyService = Mockito.spy(new ProxyService(proxyConfig, - new AuthenticationService( - PulsarConfigurationLoader.convertFrom(proxyConfig)))); + AuthenticationService authService = + new AuthenticationService(PulsarConfigurationLoader.convertFrom(proxyConfig)); + proxyService = Mockito.spy(new ProxyService(proxyConfig, authService)); + webServer = new WebServer(proxyConfig, authService); } @AfterMethod(alwaysRun = true) @@ -125,10 +128,13 @@ protected void setup() throws Exception { protected void cleanup() throws Exception { super.internalCleanup(); proxyService.close(); + webServer.stop(); } private void startProxy() throws Exception { proxyService.start(); + ProxyServiceStarter.addWebServerHandlers(webServer, proxyConfig, proxyService, null); + webServer.start(); } /** @@ -435,7 +441,7 @@ void testGetMetrics() throws Exception { } private void createAdminClient() throws Exception { - admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrl.toString()) + admin = spy(PulsarAdmin.builder().serviceHttpUrl(webServer.getServiceUri().toString()) .authentication(AuthenticationFactory.token(ADMIN_TOKEN)).build()); } From e6bc4999e22a763c9097494970ed6d9796278a23 Mon Sep 17 00:00:00 2001 From: AloysZhang Date: Thu, 23 Feb 2023 13:18:10 +0800 Subject: [PATCH 046/174] [fix][txn]fix receive duplicated messages due to pendingAcks in PendingAckHandle (#19581) Co-authored-by: mayozhang --- .../mledger/util/PositionAckSetUtil.java | 7 ++ .../service/AbstractBaseDispatcher.java | 13 ++++ .../client/impl/TransactionEndToEndTest.java | 78 +++++++++++++++++++ .../BitSetRecyclableRecyclableTest.java | 17 ++++ 4 files changed, 115 insertions(+) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/PositionAckSetUtil.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/PositionAckSetUtil.java index 8173b30c4fea9..1c607582076a8 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/PositionAckSetUtil.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/PositionAckSetUtil.java @@ -59,6 +59,13 @@ public static long[] andAckSet(long[] firstAckSet, long[] secondAckSet) { return ackSet; } + public static boolean isAckSetEmpty(long[] ackSet) { + BitSetRecyclable bitSet = BitSetRecyclable.create().resetWords(ackSet); + boolean isEmpty = bitSet.isEmpty(); + bitSet.recycle(); + return isEmpty; + } + //This method is compare two position which position is bigger than another one. //When the ledgerId and entryId in this position is same to another one and two position all have ack set, it will //compare the ack set next bit index is bigger than another one. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java index ef2fd80302a98..8f6caa7a20801 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.service; import static org.apache.bookkeeper.mledger.util.PositionAckSetUtil.andAckSet; +import static org.apache.bookkeeper.mledger.util.PositionAckSetUtil.isAckSetEmpty; import io.netty.buffer.ByteBuf; import io.prometheus.client.Gauge; import java.util.ArrayList; @@ -239,6 +240,18 @@ public int filterEntriesForConsumer(@Nullable MessageMetadata[] metadataArray, i // if actSet is null, use pendingAck ackSet ackSet = positionInPendingAck.getAckSet(); } + // if the result of pendingAckSet(in pendingAckHandle) AND the ackSet(in cursor) is empty + // filter this entry + if (isAckSetEmpty(ackSet)) { + entries.set(i, null); + entry.release(); + continue; + } + } else { + // filter non-batch message in pendingAck state + entries.set(i, null); + entry.release(); + continue; } } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java index 527b8532e0452..83feaa3ac1158 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java @@ -176,6 +176,84 @@ private void testIndividualAckAbortFilterAckSetInPendingAckState() throws Except assertNull(consumer.receive(2, TimeUnit.SECONDS)); } + + @Test(dataProvider="enableBatch") + private void testFilterMsgsInPendingAckStateWhenConsumerDisconnect(boolean enableBatch) throws Exception { + final String topicName = NAMESPACE1 + "/testFilterMsgsInPendingAckStateWhenConsumerDisconnect-" + enableBatch; + final int count = 10; + + @Cleanup + Producer producer = null; + if (enableBatch) { + producer = pulsarClient + .newProducer(Schema.INT32) + .topic(topicName) + .enableBatching(true) + .batchingMaxPublishDelay(1, TimeUnit.HOURS) + .batchingMaxMessages(count).create(); + } else { + producer = pulsarClient + .newProducer(Schema.INT32) + .topic(topicName) + .enableBatching(false).create(); + } + + @Cleanup + Consumer consumer = pulsarClient + .newConsumer(Schema.INT32) + .topic(topicName) + .isAckReceiptEnabled(true) + .subscriptionName("test") + .subscriptionType(SubscriptionType.Shared) + .enableBatchIndexAcknowledgment(true) + .subscribe(); + + for (int i = 0; i < count; i++) { + producer.sendAsync(i); + } + + Transaction txn1 = getTxn(); + + Transaction txn2 = getTxn(); + + + // txn1 ack half of messages and don't end the txn1 + for (int i = 0; i < count / 2; i++) { + consumer.acknowledgeAsync(consumer.receive().getMessageId(), txn1).get(); + } + + // txn2 ack the rest half of messages and commit tnx2 + for (int i = count / 2; i < count; i++) { + consumer.acknowledgeAsync(consumer.receive().getMessageId(), txn2).get(); + } + // commit txn2 + txn2.commit().get(); + + // close and re-create consumer + consumer.close(); + consumer = pulsarClient + .newConsumer(Schema.INT32) + .topic(topicName) + .isAckReceiptEnabled(true) + .subscriptionName("test") + .subscriptionType(SubscriptionType.Shared) + .enableBatchIndexAcknowledgment(true) + .subscribe(); + + Message message = consumer.receive(3, TimeUnit.SECONDS); + Assert.assertNull(message); + + // abort txn1 + txn1.abort().get(); + // after txn1 aborted, consumer will receive messages txn1 contains + int receiveCounter = 0; + while((message = consumer.receive(3, TimeUnit.SECONDS)) != null) { + Assert.assertEquals(message.getValue().intValue(), receiveCounter); + receiveCounter ++; + } + Assert.assertEquals(receiveCounter, count / 2); + } + @Test(dataProvider="enableBatch") private void produceCommitTest(boolean enableBatch) throws Exception { @Cleanup diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/BitSetRecyclableRecyclableTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/BitSetRecyclableRecyclableTest.java index 8374f2db8961a..8061f853d66c1 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/BitSetRecyclableRecyclableTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/BitSetRecyclableRecyclableTest.java @@ -45,4 +45,21 @@ public void testResetWords() { Assert.assertTrue(bitset1.get(128)); Assert.assertFalse(bitset1.get(256)); } + + @Test + public void testBitSetEmpty() { + BitSetRecyclable bitSet = BitSetRecyclable.create(); + bitSet.set(0, 5); + bitSet.clear(1); + bitSet.clear(2); + bitSet.clear(3); + long[] array = bitSet.toLongArray(); + Assert.assertFalse(bitSet.isEmpty()); + Assert.assertFalse(BitSetRecyclable.create().resetWords(array).isEmpty()); + bitSet.clear(0); + bitSet.clear(4); + Assert.assertTrue(bitSet.isEmpty()); + long[] array1 = bitSet.toLongArray(); + Assert.assertTrue(BitSetRecyclable.create().resetWords(array1).isEmpty()); + } } From e2863391e7f6f9b6c5060f0f78378493f8df37f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 23 Feb 2023 10:12:10 +0100 Subject: [PATCH 047/174] [fix][client] Broker address resolution wrong if connect through a multi-dns names proxy (#19597) --- .../client/impl/ConnectionPoolTest.java | 89 ++++++++++++++++++- .../pulsar/client/impl/ConnectionPool.java | 24 +++-- .../client/impl/PulsarChannelInitializer.java | 8 +- 3 files changed, 106 insertions(+), 15 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java index e8816894513d2..fb564bd5083c1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java @@ -20,12 +20,17 @@ import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; import io.netty.channel.EventLoopGroup; +import io.netty.resolver.AbstractAddressResolver; import io.netty.util.concurrent.DefaultThreadFactory; import java.net.InetSocketAddress; +import java.net.SocketAddress; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.function.Supplier; import java.util.stream.IntStream; +import io.netty.util.concurrent.Promise; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.common.util.netty.EventLoopUtil; @@ -66,7 +71,7 @@ public void testSingleIpAddress() throws Exception { List result = new ArrayList<>(); result.add(new InetSocketAddress("127.0.0.1", brokerPort)); Mockito.when(pool.resolveName(InetSocketAddress.createUnresolved("non-existing-dns-name", - brokerPort))) + brokerPort))) .thenReturn(CompletableFuture.completedFuture(result)); client.newProducer().topic("persistent://sample/standalone/ns/my-topic").create(); @@ -107,7 +112,7 @@ public void testNoConnectionPool() throws Exception { ConnectionPool pool = spyWithClassAndConstructorArgs(ConnectionPool.class, conf, eventLoop); InetSocketAddress brokerAddress = - InetSocketAddress.createUnresolved("127.0.0.1", brokerPort); + InetSocketAddress.createUnresolved("127.0.0.1", brokerPort); IntStream.range(1, 5).forEach(i -> { pool.getConnection(brokerAddress).thenAccept(cnx -> { Assert.assertTrue(cnx.channel().isActive()); @@ -119,6 +124,7 @@ public void testNoConnectionPool() throws Exception { pool.closeAllConnections(); pool.close(); + eventLoop.shutdownGracefully(); } @Test @@ -129,7 +135,7 @@ public void testEnableConnectionPool() throws Exception { ConnectionPool pool = spyWithClassAndConstructorArgs(ConnectionPool.class, conf, eventLoop); InetSocketAddress brokerAddress = - InetSocketAddress.createUnresolved("127.0.0.1", brokerPort); + InetSocketAddress.createUnresolved("127.0.0.1", brokerPort); IntStream.range(1, 10).forEach(i -> { pool.getConnection(brokerAddress).thenAccept(cnx -> { Assert.assertTrue(cnx.channel().isActive()); @@ -141,5 +147,82 @@ public void testEnableConnectionPool() throws Exception { pool.closeAllConnections(); pool.close(); + eventLoop.shutdownGracefully(); + } + + + @Test + public void testSetProxyToTargetBrokerAddress() throws Exception { + ClientConfigurationData conf = new ClientConfigurationData(); + conf.setConnectionsPerBroker(5); + + + EventLoopGroup eventLoop = + EventLoopUtil.newEventLoopGroup(8, false, + new DefaultThreadFactory("test")); + + final AbstractAddressResolver resolver = new AbstractAddressResolver(eventLoop.next()) { + @Override + protected boolean doIsResolved(SocketAddress socketAddress) { + return !((InetSocketAddress) socketAddress).isUnresolved(); + } + + @Override + protected void doResolve(SocketAddress socketAddress, Promise promise) throws Exception { + promise.setFailure(new IllegalStateException()); + throw new IllegalStateException(); + } + + @Override + protected void doResolveAll(SocketAddress socketAddress, Promise promise) throws Exception { + final InetSocketAddress socketAddress1 = (InetSocketAddress) socketAddress; + final boolean isProxy = socketAddress1.getHostName().equals("proxy"); + final boolean isBroker = socketAddress1.getHostName().equals("broker"); + if (!isProxy && !isBroker) { + promise.setFailure(new IllegalStateException()); + throw new IllegalStateException(); + } + List result = new ArrayList<>(); + if (isProxy) { + result.add(new InetSocketAddress("localhost", brokerPort)); + result.add(InetSocketAddress.createUnresolved("proxy", brokerPort)); + } else { + result.add(new InetSocketAddress("127.0.0.1", brokerPort)); + result.add(InetSocketAddress.createUnresolved("broker", brokerPort)); + } + promise.setSuccess(result); + } + }; + + ConnectionPool pool = spyWithClassAndConstructorArgs(ConnectionPool.class, conf, eventLoop, + (Supplier) () -> new ClientCnx(conf, eventLoop), Optional.of(resolver)); + + + ClientCnx cnx = pool.getConnection( + InetSocketAddress.createUnresolved("proxy", 9999), + InetSocketAddress.createUnresolved("proxy", 9999)).get(); + Assert.assertEquals(cnx.remoteHostName, "proxy"); + Assert.assertNull(cnx.proxyToTargetBrokerAddress); + cnx.close(); + + cnx = pool.getConnection( + InetSocketAddress.createUnresolved("broker", 9999), + InetSocketAddress.createUnresolved("proxy", 9999)).get(); + Assert.assertEquals(cnx.remoteHostName, "proxy"); + Assert.assertEquals(cnx.proxyToTargetBrokerAddress, "broker:9999"); + cnx.close(); + + + cnx = pool.getConnection( + InetSocketAddress.createUnresolved("broker", 9999), + InetSocketAddress.createUnresolved("broker", 9999)).get(); + Assert.assertEquals(cnx.remoteHostName, "broker"); + Assert.assertNull(cnx.proxyToTargetBrokerAddress); + cnx.close(); + + + pool.closeAllConnections(); + pool.close(); + eventLoop.shutdownGracefully(); } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java index 2e105b5328467..3a9a2b9b7ab94 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java @@ -305,8 +305,12 @@ private CompletableFuture createConnection(InetSocketAddress logicalAdd resolvedAddress = resolveName(unresolvedPhysicalAddress); } return resolvedAddress.thenCompose( - inetAddresses -> connectToResolvedAddresses(logicalAddress, inetAddresses.iterator(), - isSniProxy ? unresolvedPhysicalAddress : null)); + inetAddresses -> connectToResolvedAddresses( + logicalAddress, + unresolvedPhysicalAddress, + inetAddresses.iterator(), + isSniProxy ? unresolvedPhysicalAddress : null) + ); } catch (URISyntaxException e) { log.error("Invalid Proxy url {}", clientConfig.getProxyServiceUrl(), e); return FutureUtil @@ -319,17 +323,19 @@ private CompletableFuture createConnection(InetSocketAddress logicalAdd * address is working. */ private CompletableFuture connectToResolvedAddresses(InetSocketAddress logicalAddress, + InetSocketAddress unresolvedPhysicalAddress, Iterator resolvedPhysicalAddress, InetSocketAddress sniHost) { CompletableFuture future = new CompletableFuture<>(); // Successfully connected to server - connectToAddress(logicalAddress, resolvedPhysicalAddress.next(), sniHost) + connectToAddress(logicalAddress, resolvedPhysicalAddress.next(), unresolvedPhysicalAddress, sniHost) .thenAccept(future::complete) .exceptionally(exception -> { if (resolvedPhysicalAddress.hasNext()) { // Try next IP address - connectToResolvedAddresses(logicalAddress, resolvedPhysicalAddress, sniHost) + connectToResolvedAddresses(logicalAddress, unresolvedPhysicalAddress, + resolvedPhysicalAddress, sniHost) .thenAccept(future::complete) .exceptionally(ex -> { // This is already unwinding the recursive call @@ -362,20 +368,24 @@ CompletableFuture> resolveName(InetSocketAddress unresol * Attempt to establish a TCP connection to an already resolved single IP address. */ private CompletableFuture connectToAddress(InetSocketAddress logicalAddress, - InetSocketAddress physicalAddress, InetSocketAddress sniHost) { + InetSocketAddress physicalAddress, + InetSocketAddress unresolvedPhysicalAddress, + InetSocketAddress sniHost) { if (clientConfig.isUseTls()) { return toCompletableFuture(bootstrap.register()) .thenCompose(channel -> channelInitializerHandler .initTls(channel, sniHost != null ? sniHost : physicalAddress)) .thenCompose(channelInitializerHandler::initSocks5IfConfig) .thenCompose(ch -> - channelInitializerHandler.initializeClientCnx(ch, logicalAddress, physicalAddress)) + channelInitializerHandler.initializeClientCnx(ch, logicalAddress, + unresolvedPhysicalAddress)) .thenCompose(channel -> toCompletableFuture(channel.connect(physicalAddress))); } else { return toCompletableFuture(bootstrap.register()) .thenCompose(channelInitializerHandler::initSocks5IfConfig) .thenCompose(ch -> - channelInitializerHandler.initializeClientCnx(ch, logicalAddress, physicalAddress)) + channelInitializerHandler.initializeClientCnx(ch, logicalAddress, + unresolvedPhysicalAddress)) .thenCompose(channel -> toCompletableFuture(channel.connect(physicalAddress))); } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarChannelInitializer.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarChannelInitializer.java index e01b53b8ef136..ed34f7d41c130 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarChannelInitializer.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarChannelInitializer.java @@ -213,7 +213,7 @@ CompletableFuture initSocks5IfConfig(Channel ch) { CompletableFuture initializeClientCnx(Channel ch, InetSocketAddress logicalAddress, - InetSocketAddress resolvedPhysicalAddress) { + InetSocketAddress unresolvedPhysicalAddress) { return NettyFutureUtil.toCompletableFuture(ch.eventLoop().submit(() -> { final ClientCnx cnx = (ClientCnx) ch.pipeline().get("handler"); @@ -221,15 +221,13 @@ CompletableFuture initializeClientCnx(Channel ch, throw new IllegalStateException("Missing ClientCnx. This should not happen."); } - // Need to do our own equality because the physical address is resolved already - if (!(logicalAddress.getHostString().equalsIgnoreCase(resolvedPhysicalAddress.getHostString()) - && logicalAddress.getPort() == resolvedPhysicalAddress.getPort())) { + if (!logicalAddress.equals(unresolvedPhysicalAddress)) { // We are connecting through a proxy. We need to set the target broker in the ClientCnx object so that // it can be specified when sending the CommandConnect. cnx.setTargetBroker(logicalAddress); } - cnx.setRemoteHostName(resolvedPhysicalAddress.getHostString()); + cnx.setRemoteHostName(unresolvedPhysicalAddress.getHostString()); return ch; })); From 0bb0f6b786d115a7405867b701521cd4a49340c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 23 Feb 2023 13:54:27 +0100 Subject: [PATCH 048/174] [fix][broker] Copy command fields and fix potential thread-safety in ServerCnx (#19517) --- .../java/org/apache/pulsar/broker/service/ServerCnx.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index adee29c5a0105..242947f6a0fd6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -2438,10 +2438,11 @@ protected void handleAddPartitionToTxn(CommandAddPartitionToTxn command) { final TxnID txnID = new TxnID(command.getTxnidMostBits(), command.getTxnidLeastBits()); final TransactionCoordinatorID tcId = TransactionCoordinatorID.get(command.getTxnidMostBits()); final long requestId = command.getRequestId(); + final List partitionsList = command.getPartitionsList(); if (log.isDebugEnabled()) { - command.getPartitionsList().forEach(partion -> + partitionsList.forEach(partition -> log.debug("Receive add published partition to txn request {} " - + "from {} with txnId {}, topic: [{}]", requestId, remoteAddress, txnID, partion)); + + "from {} with txnId {}, topic: [{}]", requestId, remoteAddress, txnID, partition)); } if (!checkTransactionEnableAndSendError(requestId)) { @@ -2456,7 +2457,7 @@ protected void handleAddPartitionToTxn(CommandAddPartitionToTxn command) { return failedFutureTxnNotOwned(txnID); } return transactionMetadataStoreService - .addProducedPartitionToTxn(txnID, command.getPartitionsList()); + .addProducedPartitionToTxn(txnID, partitionsList); }) .whenComplete((v, ex) -> { if (ex == null) { From 389792b1fc7a56647ccfc820e83ae08dfed037df Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Thu, 23 Feb 2023 07:23:00 -0800 Subject: [PATCH 049/174] [improve][broker] PIP-192 Added Deleted and Init states in ServiceUnitState (#19546) --- .../pulsar/broker/ServiceConfiguration.java | 17 + .../ExtensibleLoadManagerWrapper.java | 4 + .../extensions/channel/ServiceUnitState.java | 67 +- .../channel/ServiceUnitStateChannelImpl.java | 513 ++++++++++----- .../ServiceUnitStateCompactionStrategy.java | 93 ++- .../channel/ServiceUnitStateData.java | 20 +- .../StrategicTwoPhaseCompactor.java | 39 +- .../ExtensibleLoadManagerImplTest.java | 187 +++--- .../channel/ServiceUnitStateChannelTest.java | 615 +++++++++++++++--- ...erviceUnitStateCompactionStrategyTest.java | 125 ++-- .../channel/ServiceUnitStateDataTest.java | 15 +- .../channel/ServiceUnitStateTest.java | 82 ++- .../ServiceUnitStateCompactionTest.java | 333 +++++++--- 13 files changed, 1508 insertions(+), 602 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 106410d855e22..4f2c8e72e131d 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -2450,6 +2450,7 @@ The delayed message index bucket time step(in seconds) in per bucket snapshot se ) private long namespaceBundleUnloadingTimeoutMs = 60000; + /**** --- Load Balancer Extension. --- ****/ @FieldContext( category = CATEGORY_LOAD_BALANCER, dynamic = true, @@ -2525,6 +2526,22 @@ The delayed message index bucket time step(in seconds) in per bucket snapshot se ) private double loadBalancerBundleLoadReportPercentage = 10; + @FieldContext( + category = CATEGORY_LOAD_BALANCER, + doc = "After this delay, the service-unit state channel tombstones any service units (e.g., bundles) " + + "in semi-terminal states. For example, after splits, parent bundles will be `deleted`, " + + "and then after this delay, the parent bundles' state will be `tombstoned` " + + "in the service-unit state channel. " + + "Pulsar does not immediately remove such semi-terminal states " + + "to avoid unnecessary system confusion, " + + "as the bundles in the `tombstoned` state might temporarily look available to reassign. " + + "Rarely, one could lower this delay in order to aggressively clean " + + "the service-unit state channel when there are a large number of bundles. " + + "minimum value = 30 secs" + + "(only used in load balancer extension logics)" + ) + private long loadBalancerServiceUnitStateCleanUpDelayTimeInSeconds = 604800; + /**** --- Replication. --- ****/ @FieldContext( category = CATEGORY_REPLICATION, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java index 48fc4bb7ff4f0..1eabbe620e213 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java @@ -142,4 +142,8 @@ public void doNamespaceBundleSplit() { throw new UnsupportedOperationException(); } + public ExtensibleLoadManagerImpl get() { + return loadManager; + } + } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitState.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitState.java index 3225c0ba7bbc7..92fef8f65992a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitState.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitState.java @@ -24,55 +24,38 @@ /** * Defines the possible states for service units. * - * The following diagram defines the valid state changes - * - * ┌───────────┐ - * ┌──────────┤ released │◄────────┐ - * │own └───────────┘ │release - * │ │ - * │ │ - * ▼ │ - * ┌────────┐ assign(transfer) ┌─────┴────┐ - * │ ├───────────────────►│ │ - * │ owned │ │ assigned │ - * │ │◄───────────────────┤ │ - * └──┬─────┤ own └──────────┘ - * │ ▲ │ ▲ - * │ │ │ │ - * │ │ └──────────────┐ │ - * │ │ │ │ - * │ │ unload │ │ assign(assignment) - * split │ │ │ │ - * │ │ │ │ - * │ │ create(child) │ │ - * │ │ │ │ - * ▼ │ │ │ - * ┌─────┴─────┐ └─────►┌───┴──────┐ - * │ │ │ │ - * │ splitting ├────────────────► │ free │ - * │ │ discard(parent)│ │ - * └───────────┘ └──────────┘ + * @see Service Unit State Channel for additional details. */ public enum ServiceUnitState { - Free, // not owned by any broker (terminal state) + Init, // initializing the state. no previous state(terminal state) + + Free, // not owned by any broker (semi-terminal state) Owned, // owned by a broker (terminal state) - Assigned, // the ownership is assigned(but the assigned broker has not been notified the ownership yet) + Assigning, // the ownership is being assigned (e.g. the new ownership is being notified to the target broker) - Released, // the source broker's ownership has been released (e.g. the topic connections are closed) + Releasing, // the source broker's ownership is being released (e.g. the topic connections are being closed) - Splitting; // the service unit(e.g. bundle) is in the process of splitting. + Splitting, // the service unit is in the process of splitting. (e.g. the metadata store is being updated) - private static Map> validTransitions = Map.of( - // (Free -> Released | Splitting) transitions are required - // when the topic is compacted in the middle of transfer or split. - Free, Set.of(Owned, Assigned, Released, Splitting), - Owned, Set.of(Assigned, Splitting, Free), - Assigned, Set.of(Owned, Released, Free), - Released, Set.of(Owned, Free), - Splitting, Set.of(Free) + Deleted; // deleted in the system (semi-terminal state) + + private static final Map> validTransitions = Map.of( + // (Init -> all states) transitions are required + // when the topic is compacted in the middle of assign, transfer or split. + Init, Set.of(Free, Owned, Assigning, Releasing, Splitting, Deleted), + Free, Set.of(Assigning, Init), + Owned, Set.of(Assigning, Splitting, Releasing), + Assigning, Set.of(Owned, Releasing), + Releasing, Set.of(Owned, Free), + Splitting, Set.of(Deleted), + Deleted, Set.of(Init) + ); + + private static final Set inFlightStates = Set.of( + Assigning, Releasing, Splitting ); public static boolean isValidTransition(ServiceUnitState from, ServiceUnitState to) { @@ -80,4 +63,8 @@ public static boolean isValidTransition(ServiceUnitState from, ServiceUnitState return transitions.contains(to); } + public static boolean isInFlightState(ServiceUnitState state) { + return inFlightStates.contains(state); + } + } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index d10138bda6805..9f205f85c5454 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -20,10 +20,13 @@ import static java.lang.String.format; import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Assigned; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Assigning; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Deleted; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Free; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Init; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Owned; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Released; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Releasing; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Splitting; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.ChannelState.Closed; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.ChannelState.Constructed; @@ -35,6 +38,7 @@ import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.MetadataState.Jittery; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.MetadataState.Stable; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.MetadataState.Unstable; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData.state; import static org.apache.pulsar.metadata.api.extended.SessionEvent.SessionLost; import static org.apache.pulsar.metadata.api.extended.SessionEvent.SessionReestablished; import com.google.common.annotations.VisibleForTesting; @@ -61,9 +65,15 @@ import org.apache.commons.lang3.mutable.MutableInt; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.LeaderElectionService; +import org.apache.pulsar.broker.loadbalance.extensions.BrokerRegistry; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerWrapper; +import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.models.Split; import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; +import org.apache.pulsar.broker.loadbalance.extensions.strategy.BrokerSelectionStrategy; +import org.apache.pulsar.broker.loadbalance.extensions.strategy.LeastResourceUsageWithWeight; import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.service.BrokerServiceException; @@ -92,37 +102,42 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel { TopicDomain.persistent.value(), NamespaceName.SYSTEM_NAMESPACE, "loadbalancer-service-unit-state").toString(); - - // TODO: define StateCompactionStrategy private static final long MAX_IN_FLIGHT_STATE_WAITING_TIME_IN_MILLIS = 30 * 1000; // 30sec + + private static final long OWNERSHIP_MONITOR_DELAY_TIME_IN_SECS = 60; public static final long MAX_CLEAN_UP_DELAY_TIME_IN_SECS = 3 * 60; // 3 mins private static final long MIN_CLEAN_UP_DELAY_TIME_IN_SECS = 0; // 0 secs to clean immediately - private static final long MAX_CHANNEL_OWNER_ELECTION_WAITING_TIME_IN_SECS = 10; private static final int MAX_OUTSTANDING_PUB_MESSAGES = 500; private final PulsarService pulsar; + private final ServiceConfiguration config; private final Schema schema; private final ConcurrentOpenHashMap> getOwnerRequests; private final String lookupServiceAddress; - // TODO: define BrokerRegistry private final ConcurrentOpenHashMap> cleanupJobs; private final LeaderElectionService leaderElectionService; + private BrokerSelectionStrategy brokerSelector; + private BrokerRegistry brokerRegistry; private TableView tableview; private Producer producer; - private ScheduledFuture cleanupTasks; + private ScheduledFuture monitorTask; private SessionEvent lastMetadataSessionEvent = SessionReestablished; private long lastMetadataSessionEventTimestamp = 0; private long inFlightStateWaitingTimeInMillis; + + private long ownershipMonitorDelayTimeInSecs; + private long semiTerminalStateWaitingTimeInMillis; private long maxCleanupDelayTimeInSecs; private long minCleanupDelayTimeInSecs; // cleanup metrics - private long totalCleanupCnt = 0; - private long totalBrokerCleanupTombstoneCnt = 0; - private long totalServiceUnitCleanupTombstoneCnt = 0; + private long totalInactiveBrokerCleanupCnt = 0; + private long totalServiceUnitTombstoneCleanupCnt = 0; + + private long totalOrphanServiceUnitCleanupCnt = 0; private AtomicLong totalCleanupErrorCnt = new AtomicLong(); - private long totalCleanupScheduledCnt = 0; - private long totalCleanupIgnoredCnt = 0; - private long totalCleanupCancelledCnt = 0; + private long totalInactiveBrokerCleanupScheduledCnt = 0; + private long totalInactiveBrokerCleanupIgnoredCnt = 0; + private long totalInactiveBrokerCleanupCancelledCnt = 0; private volatile ChannelState channelState; public enum EventType { @@ -135,30 +150,18 @@ public enum EventType { @Getter @AllArgsConstructor public static class Counters { - private AtomicLong total; - private AtomicLong failure; + private final AtomicLong total; + private final AtomicLong failure; + public Counters(){ + total = new AtomicLong(); + failure = new AtomicLong(); + } } // operation metrics - final Map ownerLookUpCounters = Map.of( - Owned, new AtomicLong(), - Assigned, new AtomicLong(), - Released, new AtomicLong(), - Splitting, new AtomicLong(), - Free, new AtomicLong() - ); - final Map eventCounters = Map.of( - Assign, new Counters(new AtomicLong(), new AtomicLong()), - Split, new Counters(new AtomicLong(), new AtomicLong()), - Unload, new Counters(new AtomicLong(), new AtomicLong()) - ); - final Map handlerCounters = Map.of( - Owned, new Counters(new AtomicLong(), new AtomicLong()), - Assigned, new Counters(new AtomicLong(), new AtomicLong()), - Released, new Counters(new AtomicLong(), new AtomicLong()), - Splitting, new Counters(new AtomicLong(), new AtomicLong()), - Free, new Counters(new AtomicLong(), new AtomicLong()) - ); + final Map ownerLookUpCounters; + final Map eventCounters; + final Map handlerCounters; enum ChannelState { Closed(0), @@ -180,25 +183,61 @@ enum MetadataState { public ServiceUnitStateChannelImpl(PulsarService pulsar) { this.pulsar = pulsar; + this.config = pulsar.getConfig(); this.lookupServiceAddress = pulsar.getLookupServiceAddress(); this.schema = Schema.JSON(ServiceUnitStateData.class); this.getOwnerRequests = ConcurrentOpenHashMap.>newBuilder().build(); this.cleanupJobs = ConcurrentOpenHashMap.>newBuilder().build(); + this.semiTerminalStateWaitingTimeInMillis = config.getLoadBalancerServiceUnitStateCleanUpDelayTimeInSeconds() + * 1000; this.inFlightStateWaitingTimeInMillis = MAX_IN_FLIGHT_STATE_WAITING_TIME_IN_MILLIS; + this.ownershipMonitorDelayTimeInSecs = OWNERSHIP_MONITOR_DELAY_TIME_IN_SECS; + if (semiTerminalStateWaitingTimeInMillis < inFlightStateWaitingTimeInMillis) { + throw new IllegalArgumentException( + "Invalid Config: loadBalancerServiceUnitStateCleanUpDelayTimeInSeconds < " + + (MAX_IN_FLIGHT_STATE_WAITING_TIME_IN_MILLIS / 1000) + " secs"); + } this.maxCleanupDelayTimeInSecs = MAX_CLEAN_UP_DELAY_TIME_IN_SECS; this.minCleanupDelayTimeInSecs = MIN_CLEAN_UP_DELAY_TIME_IN_SECS; this.leaderElectionService = new LeaderElectionService( pulsar.getCoordinationService(), pulsar.getSafeWebServiceAddress(), state -> { if (state == LeaderElectionState.Leading) { - log.debug("This broker:{} is the leader now.", lookupServiceAddress); - // TODO: schedule monitorOwnerships by brokerRegistry + log.info("This broker:{} is the leader now.", lookupServiceAddress); + this.monitorTask = this.pulsar.getLoadManagerExecutor() + .scheduleWithFixedDelay(() -> { + try { + monitorOwnerships(brokerRegistry.getAvailableBrokersAsync() + .get(inFlightStateWaitingTimeInMillis, MILLISECONDS)); + } catch (Exception e) { + log.info("Failed to monitor the ownerships. will retry..", e); + } + }, + ownershipMonitorDelayTimeInSecs, ownershipMonitorDelayTimeInSecs, SECONDS); } else { - log.debug("This broker:{} is a follower now.", lookupServiceAddress); - // TODO: cancel scheduled monitorOwnerships if any + log.info("This broker:{} is a follower now.", lookupServiceAddress); + if (monitorTask != null) { + monitorTask.cancel(false); + monitorTask = null; + log.info("This previous leader broker:{} stopped the channel clean-up monitor", + lookupServiceAddress); + } } }); + Map tmpOwnerLookUpCounters = new HashMap<>(); + Map tmpHandlerCounters = new HashMap<>(); + Map tmpEventCounters = new HashMap<>(); + for (var state : ServiceUnitState.values()) { + tmpOwnerLookUpCounters.put(state, new AtomicLong()); + tmpHandlerCounters.put(state, new Counters()); + } + for (var event : EventType.values()) { + tmpEventCounters.put(event, new Counters()); + } + ownerLookUpCounters = Map.copyOf(tmpOwnerLookUpCounters); + handlerCounters = Map.copyOf(tmpHandlerCounters); + eventCounters = Map.copyOf(tmpEventCounters); this.channelState = Constructed; } @@ -207,14 +246,22 @@ public synchronized void start() throws PulsarServerException { throw new IllegalStateException("Invalid channel state:" + channelState.name()); } + boolean debug = debug(); try { + this.brokerRegistry = getBrokerRegistry(); + this.brokerRegistry.addListener(this::handleBrokerRegistrationEvent); leaderElectionService.start(); this.channelState = LeaderElectionServiceStarted; - log.debug("Successfully started the channel leader election service."); + if (debug) { + log.info("Successfully started the channel leader election service."); + } + brokerSelector = getBrokerSelector(); if (producer != null) { producer.close(); - log.debug("Closed the channel producer."); + if (debug) { + log.info("Closed the channel producer."); + } } producer = pulsar.getClient().newProducer(schema) .enableBatching(true) @@ -223,11 +270,15 @@ public synchronized void start() throws PulsarServerException { .topic(TOPIC) .create(); - log.debug("Successfully started the channel producer."); + if (debug) { + log.info("Successfully started the channel producer."); + } if (tableview != null) { tableview.close(); - log.debug("Closed the channel tableview."); + if (debug) { + log.info("Closed the channel tableview."); + } } tableview = pulsar.getClient().newTableViewBuilder(schema) .topic(TOPIC) @@ -236,10 +287,13 @@ public synchronized void start() throws PulsarServerException { ServiceUnitStateCompactionStrategy.class.getName())) .create(); tableview.listen((key, value) -> handle(key, value)); - log.debug("Successfully started the channel tableview."); - + if (debug) { + log.info("Successfully started the channel tableview."); + } pulsar.getLocalMetadataStore().registerSessionListener(this::handleMetadataSessionEvent); - log.debug("Successfully registered the handleMetadataSessionEvent"); + if (debug) { + log.info("Successfully registered the handleMetadataSessionEvent"); + } channelState = Started; log.info("Successfully started the channel."); @@ -250,16 +304,39 @@ public synchronized void start() throws PulsarServerException { } } + @VisibleForTesting + protected BrokerRegistry getBrokerRegistry() { + return ((ExtensibleLoadManagerWrapper) pulsar.getLoadManager().get()) + .get().getBrokerRegistry(); + } + + @VisibleForTesting + protected LoadManagerContext getContext() { + return ((ExtensibleLoadManagerWrapper) pulsar.getLoadManager().get()) + .get().getContext(); + } + + @VisibleForTesting + protected BrokerSelectionStrategy getBrokerSelector() { + // TODO: make this selector configurable. + return new LeastResourceUsageWithWeight(); + } + public synchronized void close() throws PulsarServerException { channelState = Closed; + boolean debug = debug(); try { leaderElectionService.close(); - log.debug("Successfully closed the channel leader election service."); + if (debug) { + log.info("Successfully closed the channel leader election service."); + } if (tableview != null) { tableview.close(); tableview = null; - log.debug("Successfully closed the channel tableview."); + if (debug) { + log.info("Successfully closed the channel tableview."); + } } if (producer != null) { @@ -268,11 +345,13 @@ public synchronized void close() throws PulsarServerException { log.info("Successfully closed the channel producer."); } - // TODO: clean brokerRegistry + if (brokerRegistry != null) { + brokerRegistry = null; + } - if (cleanupTasks != null) { - cleanupTasks.cancel(true); - cleanupTasks = null; + if (monitorTask != null) { + monitorTask.cancel(true); + monitorTask = null; log.info("Successfully cancelled the cleanup tasks"); } @@ -294,7 +373,7 @@ private boolean validateChannelState(ChannelState targetState, boolean checkLowe } private boolean debug() { - return pulsar.getConfiguration().isLoadBalancerDebugModeEnabled() || log.isDebugEnabled(); + return config.isLoadBalancerDebugModeEnabled() || log.isDebugEnabled(); } public CompletableFuture> getChannelOwnerAsync() { @@ -348,18 +427,22 @@ public CompletableFuture> getOwnerAsync(String serviceUnit) { } ServiceUnitStateData data = tableview.get(serviceUnit); - ServiceUnitState state = data == null ? Free : data.state(); + ServiceUnitState state = state(data); ownerLookUpCounters.get(state).incrementAndGet(); switch (state) { case Owned, Splitting -> { return CompletableFuture.completedFuture(Optional.of(data.broker())); } - case Assigned, Released -> { - return deferGetOwnerRequest(serviceUnit).thenApply(Optional::of); + case Assigning, Releasing -> { + return deferGetOwnerRequest(serviceUnit).thenApply( + broker -> broker == null ? Optional.empty() : Optional.of(broker)); } - case Free -> { + case Init, Free -> { return CompletableFuture.completedFuture(Optional.empty()); } + case Deleted -> { + return CompletableFuture.failedFuture(new IllegalArgumentException(serviceUnit + " is deleted.")); + } default -> { String errorMsg = String.format("Failed to process service unit state data: %s when get owner.", data); log.error(errorMsg); @@ -372,7 +455,7 @@ public CompletableFuture publishAssignEventAsync(String serviceUnit, Str EventType eventType = Assign; eventCounters.get(eventType).getTotal().incrementAndGet(); CompletableFuture getOwnerRequest = deferGetOwnerRequest(serviceUnit); - pubAsync(serviceUnit, new ServiceUnitStateData(Assigned, broker)) + pubAsync(serviceUnit, new ServiceUnitStateData(Assigning, broker)) .whenComplete((__, ex) -> { if (ex != null) { getOwnerRequests.remove(serviceUnit, getOwnerRequest); @@ -391,11 +474,11 @@ public CompletableFuture publishUnloadEventAsync(Unload unload) { String serviceUnit = unload.serviceUnit(); CompletableFuture future; if (isTransferCommand(unload)) { - ServiceUnitStateData next = new ServiceUnitStateData(Assigned, - unload.destBroker().get(), unload.sourceBroker()); - future = pubAsync(serviceUnit, next); + future = pubAsync(serviceUnit, new ServiceUnitStateData( + Assigning, unload.destBroker().get(), unload.sourceBroker())); } else { - future = tombstoneAsync(serviceUnit); + future = pubAsync(serviceUnit, new ServiceUnitStateData( + Releasing, unload.sourceBroker())); } return future.whenComplete((__, ex) -> { @@ -424,14 +507,16 @@ private void handle(String serviceUnit, ServiceUnitStateData data) { lookupServiceAddress, serviceUnit, data, totalHandledRequests); } - ServiceUnitState state = data == null ? Free : data.state(); + ServiceUnitState state = state(data); try { switch (state) { case Owned -> handleOwnEvent(serviceUnit, data); - case Assigned -> handleAssignEvent(serviceUnit, data); - case Released -> handleReleaseEvent(serviceUnit, data); + case Assigning -> handleAssignEvent(serviceUnit, data); + case Releasing -> handleReleaseEvent(serviceUnit, data); case Splitting -> handleSplitEvent(serviceUnit, data); - case Free -> handleFreeEvent(serviceUnit); + case Deleted -> handleDeleteEvent(serviceUnit, data); + case Free -> handleFreeEvent(serviceUnit, data); + case Init -> handleInitEvent(serviceUnit); default -> throw new IllegalStateException("Failed to handle channel data:" + data); } } catch (Throwable e){ @@ -453,7 +538,7 @@ private static boolean isTransferCommand(Unload data) { } private static String getLogEventTag(ServiceUnitStateData data) { - return data == null ? "Free" : + return data == null ? Init.toString() : isTransferCommand(data) ? "Transfer:" + data.state() : data.state().toString(); } @@ -466,7 +551,7 @@ private AtomicLong getHandlerFailureCounter(ServiceUnitStateData data) { } private AtomicLong getHandlerCounter(ServiceUnitStateData data, boolean total) { - var state = data == null ? Free : data.state(); + var state = state(data); var counter = total ? handlerCounters.get(state).getTotal() : handlerCounters.get(state).getFailure(); if (counter == null) { @@ -512,22 +597,30 @@ private void handleOwnEvent(String serviceUnit, ServiceUnitStateData data) { } private void handleAssignEvent(String serviceUnit, ServiceUnitStateData data) { - deferGetOwnerRequest(serviceUnit); if (isTargetBroker(data.broker())) { ServiceUnitStateData next = new ServiceUnitStateData( - isTransferCommand(data) ? Released : Owned, data.broker(), data.sourceBroker()); + isTransferCommand(data) ? Releasing : Owned, data.broker(), data.sourceBroker()); pubAsync(serviceUnit, next) .whenComplete((__, e) -> log(e, serviceUnit, data, next)); } } private void handleReleaseEvent(String serviceUnit, ServiceUnitStateData data) { - if (isTargetBroker(data.sourceBroker())) { - ServiceUnitStateData next = new ServiceUnitStateData(Owned, data.broker(), data.sourceBroker()); - // TODO: when close, pass message to clients to connect to the new broker - closeServiceUnit(serviceUnit) - .thenCompose(__ -> pubAsync(serviceUnit, next)) - .whenComplete((__, e) -> log(e, serviceUnit, data, next)); + if (isTransferCommand(data)) { + if (isTargetBroker(data.sourceBroker())) { + ServiceUnitStateData next = new ServiceUnitStateData(Owned, data.broker(), data.sourceBroker()); + // TODO: when close, pass message to clients to connect to the new broker + closeServiceUnit(serviceUnit) + .thenCompose(__ -> pubAsync(serviceUnit, next)) + .whenComplete((__, e) -> log(e, serviceUnit, data, next)); + } + } else { + if (isTargetBroker(data.broker())) { + ServiceUnitStateData next = new ServiceUnitStateData(Free, data.broker()); + closeServiceUnit(serviceUnit) + .thenCompose(__ -> pubAsync(serviceUnit, next)) + .whenComplete((__, e) -> log(e, serviceUnit, data, next)); + } } } @@ -538,16 +631,32 @@ private void handleSplitEvent(String serviceUnit, ServiceUnitStateData data) { } } - private void handleFreeEvent(String serviceUnit) { - closeServiceUnit(serviceUnit) - .thenAccept(__ -> { - var request = getOwnerRequests.remove(serviceUnit); - if (request != null) { - request.completeExceptionally(new IllegalStateException("The ownership has been unloaded. " - + "No owner is found for serviceUnit: " + serviceUnit)); - } - }) - .whenComplete((__, e) -> log(e, serviceUnit, null, null)); + private void handleFreeEvent(String serviceUnit, ServiceUnitStateData data) { + var getOwnerRequest = getOwnerRequests.remove(serviceUnit); + if (getOwnerRequest != null) { + getOwnerRequest.complete(null); + } + if (isTargetBroker(data.broker())) { + log(null, serviceUnit, data, null); + } + } + + private void handleDeleteEvent(String serviceUnit, ServiceUnitStateData data) { + var getOwnerRequest = getOwnerRequests.remove(serviceUnit); + if (getOwnerRequest != null) { + getOwnerRequest.completeExceptionally(new IllegalStateException(serviceUnit + "has been deleted.")); + } + if (isTargetBroker(data.broker())) { + log(null, serviceUnit, data, null); + } + } + + private void handleInitEvent(String serviceUnit) { + var getOwnerRequest = getOwnerRequests.remove(serviceUnit); + if (getOwnerRequest != null) { + getOwnerRequest.complete(null); + } + log(null, serviceUnit, null, null); } private CompletableFuture pubAsync(String serviceUnit, ServiceUnitStateData data) { @@ -702,8 +811,8 @@ protected void splitServiceUnitOnceAndRetry(NamespaceService namespaceService, }); updateFuture.thenAccept(r -> { - // Free the old bundle - tombstoneAsync(serviceUnit).thenRun(() -> { + // Delete the old bundle + pubAsync(serviceUnit, new ServiceUnitStateData(Deleted, data.broker())).thenRun(() -> { // Update bundled_topic cache for load-report-generation pulsar.getBrokerService().refreshTopicToStatsMaps(bundle); // TODO: Update the load data immediately if needed. @@ -723,6 +832,8 @@ protected void splitServiceUnitOnceAndRetry(NamespaceService namespaceService, Throwable throwable = FutureUtil.unwrapCompletionException(ex); if ((throwable instanceof MetadataStoreException.BadVersionException) && (counter.incrementAndGet() < NamespaceService.BUNDLE_SPLIT_RETRY_LIMIT)) { + log.warn("Failed to update bundle range in metadata store. Retrying {} th / {} limit", + counter.get(), NamespaceService.BUNDLE_SPLIT_RETRY_LIMIT, ex); pulsar.getExecutor().schedule(() -> splitServiceUnitOnceAndRetry(namespaceService, bundleFactory, bundle, serviceUnit, data, counter, startTime, completionFuture), 100, MILLISECONDS); } else if (throwable instanceof IllegalArgumentException) { @@ -748,8 +859,10 @@ public void handleMetadataSessionEvent(SessionEvent e) { public void handleBrokerRegistrationEvent(String broker, NotificationType type) { if (type == NotificationType.Created) { + log.info("BrokerRegistry detected the broker:{} registry has been created.", broker); handleBrokerCreationEvent(broker); } else if (type == NotificationType.Deleted) { + log.info("BrokerRegistry detected the broker:{} registry has been deleted.", broker); handleBrokerDeletionEvent(broker); } } @@ -769,7 +882,7 @@ private void handleBrokerCreationEvent(String broker) { CompletableFuture future = cleanupJobs.remove(broker); if (future != null) { future.cancel(false); - totalCleanupCancelledCnt++; + totalInactiveBrokerCleanupCancelledCnt++; log.info("Successfully cancelled the ownership cleanup for broker:{}." + " Active cleanup job count:{}", broker, cleanupJobs.size()); @@ -792,7 +905,7 @@ private void handleBrokerDeletionEvent(String broker) { case Stable -> scheduleCleanup(broker, minCleanupDelayTimeInSecs); case Jittery -> scheduleCleanup(broker, maxCleanupDelayTimeInSecs); case Unstable -> { - totalCleanupIgnoredCnt++; + totalInactiveBrokerCleanupIgnoredCnt++; log.error("MetadataState state is unstable. " + "Ignoring the ownership cleanup request for the reported broker :{}", broker); } @@ -803,7 +916,7 @@ private void scheduleCleanup(String broker, long delayInSecs) { cleanupJobs.computeIfAbsent(broker, k -> { Executor delayed = CompletableFuture .delayedExecutor(delayInSecs, TimeUnit.SECONDS, pulsar.getLoadManagerExecutor()); - totalCleanupScheduledCnt++; + totalInactiveBrokerCleanupScheduledCnt++; return CompletableFuture .runAsync(() -> { try { @@ -821,27 +934,48 @@ private void scheduleCleanup(String broker, long delayInSecs) { broker, delayInSecs, cleanupJobs.size()); } + private void overrideOwnership(String serviceUnit, ServiceUnitStateData orphanData, Set availableBrokers) { + + Optional selectedBroker = brokerSelector.select(availableBrokers, null, getContext()); + if (selectedBroker.isPresent()) { + var override = new ServiceUnitStateData(Owned, selectedBroker.get(), true); + log.info("Overriding ownership serviceUnit:{} from orphanData:{} to overrideData:{}", + serviceUnit, orphanData, override); + pubAsync(serviceUnit, override).whenComplete((__, e) -> { + if (e != null) { + log.error("Failed to override serviceUnit:{} from orphanData:{} to overrideData:{}", + serviceUnit, orphanData, override, e); + } + }); + } else { + log.error("Failed to override the ownership serviceUnit:{} orphanData:{}. Empty selected broker.", + serviceUnit, orphanData); + } + } + - private void doCleanup(String broker) { + private void doCleanup(String broker) throws ExecutionException, InterruptedException, TimeoutException { long startTime = System.nanoTime(); log.info("Started ownership cleanup for the inactive broker:{}", broker); - int serviceUnitTombstoneCnt = 0; + int orphanServiceUnitCleanupCnt = 0; long totalCleanupErrorCntStart = totalCleanupErrorCnt.get(); - for (Map.Entry etr : tableview.entrySet()) { - ServiceUnitStateData stateData = etr.getValue(); - String serviceUnit = etr.getKey(); - if (StringUtils.equals(broker, stateData.broker()) - || StringUtils.equals(broker, stateData.sourceBroker())) { - log.info("Cleaning ownership serviceUnit:{}, stateData:{}.", serviceUnit, stateData); - tombstoneAsync(serviceUnit).whenComplete((__, e) -> { - if (e != null) { - log.error("Failed cleaning the ownership serviceUnit:{}, stateData:{}, " - + "cleanupErrorCnt:{}.", - serviceUnit, stateData, - totalCleanupErrorCnt.incrementAndGet() - totalCleanupErrorCntStart); - } - }); - serviceUnitTombstoneCnt++; + var availableBrokers = new HashSet<>(brokerRegistry.getAvailableBrokersAsync() + .get(inFlightStateWaitingTimeInMillis, MILLISECONDS)); + for (var etr : tableview.entrySet()) { + var stateData = etr.getValue(); + var serviceUnit = etr.getKey(); + var state = state(stateData); + if (StringUtils.equals(broker, stateData.broker())) { + if (ServiceUnitState.isInFlightState(state) || state == Owned) { + overrideOwnership(serviceUnit, stateData, availableBrokers); + orphanServiceUnitCleanupCnt++; + } + + } else if (StringUtils.equals(broker, stateData.sourceBroker())) { + if (ServiceUnitState.isInFlightState(state)) { + overrideOwnership(serviceUnit, stateData, availableBrokers); + orphanServiceUnitCleanupCnt++; + } } } @@ -851,28 +985,51 @@ private void doCleanup(String broker) { log.error("Failed to flush the in-flight messages.", e); } - if (serviceUnitTombstoneCnt > 0) { - this.totalCleanupCnt++; - this.totalServiceUnitCleanupTombstoneCnt += serviceUnitTombstoneCnt; - this.totalBrokerCleanupTombstoneCnt++; + if (orphanServiceUnitCleanupCnt > 0) { + this.totalOrphanServiceUnitCleanupCnt += orphanServiceUnitCleanupCnt; + this.totalInactiveBrokerCleanupCnt++; } double cleanupTime = TimeUnit.NANOSECONDS .toMillis((System.nanoTime() - startTime)); // TODO: clean load data stores log.info("Completed a cleanup for the inactive broker:{} in {} ms. " - + "Published tombstone for orphan service units: serviceUnitTombstoneCnt:{}, " + + "Cleaned up orphan service units: orphanServiceUnitCleanupCnt:{}, " + "approximate cleanupErrorCnt:{}, metrics:{} ", broker, cleanupTime, - serviceUnitTombstoneCnt, + orphanServiceUnitCleanupCnt, totalCleanupErrorCntStart - totalCleanupErrorCnt.get(), printCleanupMetrics()); cleanupJobs.remove(broker); } - // TODO: integrate this monitor logic when broker registry is added - private void monitorOwnerships(List brokers) { + private Optional getOverrideStateData(String serviceUnit, ServiceUnitStateData orphanData, + Set availableBrokers, + LoadManagerContext context) { + if (isTransferCommand(orphanData)) { + // rollback to the src + return Optional.of(new ServiceUnitStateData(Owned, orphanData.sourceBroker(), true)); + } else if (orphanData.state() == Assigning) { // assign + // roll-forward to another broker + Optional selectedBroker = brokerSelector.select(availableBrokers, null, context); + if (selectedBroker.isEmpty()) { + return Optional.empty(); + } + return Optional.of(new ServiceUnitStateData(Owned, selectedBroker.get(), true)); + } else if (orphanData.state() == Splitting || orphanData.state() == Releasing) { + // rollback to the target broker for split and unload + return Optional.of(new ServiceUnitStateData(Owned, orphanData.broker(), true)); + } else { + var msg = String.format("Failed to get the overrideStateData from serviceUnit=%s, orphanData=%s", + serviceUnit, orphanData); + log.error(msg); + throw new IllegalStateException(msg); + } + } + + @VisibleForTesting + protected void monitorOwnerships(List brokers) { if (!isChannelOwner()) { log.warn("This broker is not the leader now. Skipping ownership monitor"); return; @@ -886,34 +1043,69 @@ private void monitorOwnerships(List brokers) { long startTime = System.nanoTime(); Set inactiveBrokers = new HashSet<>(); Set activeBrokers = new HashSet<>(brokers); - int serviceUnitTombstoneCnt = 0; + Map orphanServiceUnits = new HashMap<>(); + int serviceUnitTombstoneCleanupCnt = 0; + int orphanServiceUnitCleanupCnt = 0; long totalCleanupErrorCntStart = totalCleanupErrorCnt.get(); long now = System.currentTimeMillis(); for (Map.Entry etr : tableview.entrySet()) { String serviceUnit = etr.getKey(); ServiceUnitStateData stateData = etr.getValue(); String broker = stateData.broker(); + var state = stateData.state(); if (!activeBrokers.contains(broker)) { inactiveBrokers.add(stateData.broker()); - } else if (stateData.state() != Owned + } else if (state != Owned && now - stateData.timestamp() > inFlightStateWaitingTimeInMillis) { - log.warn("Found long-running orphan(in-flight) serviceUnit:{}, stateData:{}", - serviceUnit, stateData); - - tombstoneAsync(serviceUnit).whenComplete((__, e) -> { - if (e != null) { - log.error("Failed cleaning the ownership serviceUnit:{}, stateData:{}, " - + "cleanupErrorCnt:{}.", - serviceUnit, stateData, - totalCleanupErrorCnt.incrementAndGet() - totalCleanupErrorCntStart); + if (state == Deleted || state == Free) { + if (now - stateData.timestamp() + > semiTerminalStateWaitingTimeInMillis) { + log.info("Found semi-terminal states to tombstone" + + " serviceUnit:{}, stateData:{}", serviceUnit, stateData); + tombstoneAsync(serviceUnit).whenComplete((__, e) -> { + if (e != null) { + log.error("Failed cleaning the ownership serviceUnit:{}, stateData:{}, " + + "cleanupErrorCnt:{}.", + serviceUnit, stateData, + totalCleanupErrorCnt.incrementAndGet() - totalCleanupErrorCntStart, e); + } + }); + serviceUnitTombstoneCleanupCnt++; } - }); - serviceUnitTombstoneCnt++; + } else { + log.warn("Found orphan serviceUnit:{}, stateData:{}", serviceUnit, stateData); + orphanServiceUnits.put(serviceUnit, stateData); + } } } - for (String inactiveBroker : inactiveBrokers) { - handleBrokerDeletionEvent(inactiveBroker); + // Skip cleaning orphan bundles if inactiveBrokers exist. This is a bigger problem. + if (!inactiveBrokers.isEmpty()) { + for (String inactiveBroker : inactiveBrokers) { + handleBrokerDeletionEvent(inactiveBroker); + } + } else if (!orphanServiceUnits.isEmpty()) { + var context = getContext(); + for (var etr : orphanServiceUnits.entrySet()) { + var orphanServiceUnit = etr.getKey(); + var orphanData = etr.getValue(); + var overrideData = getOverrideStateData( + orphanServiceUnit, orphanData, activeBrokers, context); + if (overrideData.isPresent()) { + pubAsync(orphanServiceUnit, overrideData.get()).whenComplete((__, e) -> { + if (e != null) { + log.error("Failed cleaning the ownership orphanServiceUnit:{}, orphanData:{}, " + + "cleanupErrorCnt:{}.", + orphanServiceUnit, orphanData, + totalCleanupErrorCnt.incrementAndGet() - totalCleanupErrorCntStart, e); + } + }); + orphanServiceUnitCleanupCnt++; + } else { + log.warn("Failed get the overrideStateData from orphanServiceUnit:{}, orphanData:{}. will retry..", + orphanServiceUnit, orphanData); + } + } } try { @@ -922,20 +1114,25 @@ private void monitorOwnerships(List brokers) { log.error("Failed to flush the in-flight messages.", e); } - if (serviceUnitTombstoneCnt > 0) { - this.totalServiceUnitCleanupTombstoneCnt += serviceUnitTombstoneCnt; + if (serviceUnitTombstoneCleanupCnt > 0) { + this.totalServiceUnitTombstoneCleanupCnt += serviceUnitTombstoneCleanupCnt; + } + + if (orphanServiceUnitCleanupCnt > 0) { + this.totalOrphanServiceUnitCleanupCnt += orphanServiceUnitCleanupCnt; } double monitorTime = TimeUnit.NANOSECONDS .toMillis((System.nanoTime() - startTime)); log.info("Completed the ownership monitor run in {} ms. " - + "Scheduled cleanups for inactiveBrokers:{}. inactiveBrokerCount:{}. " - + "Published tombstone for orphan service units: serviceUnitTombstoneCnt:{}, " - + "approximate cleanupErrorCnt:{}, metrics:{} ", + + "Scheduled cleanups for inactive brokers:{}. inactiveBrokerCount:{}. " + + "Published cleanups for orphan service units, orphanServiceUnitCleanupCnt:{}. " + + "Tombstoned semi-terminal state service units, serviceUnitTombstoneCleanupCnt:{}. " + + "Approximate cleanupErrorCnt:{}, metrics:{}. ", monitorTime, - inactiveBrokers, - inactiveBrokers.size(), - serviceUnitTombstoneCnt, + inactiveBrokers, inactiveBrokers.size(), + orphanServiceUnitCleanupCnt, + serviceUnitTombstoneCleanupCnt, totalCleanupErrorCntStart - totalCleanupErrorCnt.get(), printCleanupMetrics()); @@ -943,17 +1140,19 @@ private void monitorOwnerships(List brokers) { private String printCleanupMetrics() { return String.format( - "{totalCleanupCnt:%d, totalBrokerCleanupTombstoneCnt:%d, " - + "totalServiceUnitCleanupTombstoneCnt:%d, totalCleanupErrorCnt:%d, " - + "totalCleanupScheduledCnt%d, totalCleanupIgnoredCnt:%d, totalCleanupCancelledCnt:%d, " + "{totalInactiveBrokerCleanupCnt:%d, " + + "totalServiceUnitTombstoneCleanupCnt:%d, totalOrphanServiceUnitCleanupCnt:%d, " + + "totalCleanupErrorCnt:%d, " + + "totalInactiveBrokerCleanupScheduledCnt%d, totalInactiveBrokerCleanupIgnoredCnt:%d, " + + "totalInactiveBrokerCleanupCancelledCnt:%d, " + " activeCleanupJobs:%d}", - totalCleanupCnt, - totalBrokerCleanupTombstoneCnt, - totalServiceUnitCleanupTombstoneCnt, + totalInactiveBrokerCleanupCnt, + totalServiceUnitTombstoneCleanupCnt, + totalOrphanServiceUnitCleanupCnt, totalCleanupErrorCnt.get(), - totalCleanupScheduledCnt, - totalCleanupIgnoredCnt, - totalCleanupCancelledCnt, + totalInactiveBrokerCleanupScheduledCnt, + totalInactiveBrokerCleanupIgnoredCnt, + totalInactiveBrokerCleanupCancelledCnt, cleanupJobs.size() ); } @@ -1018,15 +1217,6 @@ public List getMetrics() { } } - - { - var dim = new HashMap<>(dimensions); - dim.put("result", "Total"); - var metric = Metrics.create(dim); - metric.put("brk_sunit_state_chn_cleanup_ops_total", totalCleanupCnt); - metrics.add(metric); - } - { var dim = new HashMap<>(dimensions); dim.put("result", "Failure"); @@ -1039,7 +1229,7 @@ public List getMetrics() { var dim = new HashMap<>(dimensions); dim.put("result", "Skip"); var metric = Metrics.create(dim); - metric.put("brk_sunit_state_chn_cleanup_ops_total", totalCleanupIgnoredCnt); + metric.put("brk_sunit_state_chn_inactive_broker_cleanup_ops_total", totalInactiveBrokerCleanupIgnoredCnt); metrics.add(metric); } @@ -1047,7 +1237,7 @@ public List getMetrics() { var dim = new HashMap<>(dimensions); dim.put("result", "Cancel"); var metric = Metrics.create(dim); - metric.put("brk_sunit_state_chn_cleanup_ops_total", totalCleanupCancelledCnt); + metric.put("brk_sunit_state_chn_inactive_broker_cleanup_ops_total", totalInactiveBrokerCleanupCancelledCnt); metrics.add(metric); } @@ -1055,13 +1245,14 @@ public List getMetrics() { var dim = new HashMap<>(dimensions); dim.put("result", "Schedule"); var metric = Metrics.create(dim); - metric.put("brk_sunit_state_chn_cleanup_ops_total", totalCleanupScheduledCnt); + metric.put("brk_sunit_state_chn_inactive_broker_cleanup_ops_total", totalInactiveBrokerCleanupScheduledCnt); metrics.add(metric); } var metric = Metrics.create(dimensions); - metric.put("brk_sunit_state_chn_broker_cleanup_ops_total", totalBrokerCleanupTombstoneCnt); - metric.put("brk_sunit_state_chn_su_cleanup_ops_total", totalServiceUnitCleanupTombstoneCnt); + metric.put("brk_sunit_state_chn_inactive_broker_cleanup_ops_total", totalInactiveBrokerCleanupCnt); + metric.put("brk_sunit_state_chn_orphan_su_cleanup_ops_total", totalOrphanServiceUnitCleanupCnt); + metric.put("brk_sunit_state_chn_su_tombstone_cleanup_ops_total", totalServiceUnitTombstoneCleanupCnt); metrics.add(metric); return metrics; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java index 2b21f830dda92..d2a585af9d9d5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java @@ -18,13 +18,11 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.channel; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Assigned; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Free; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Owned; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Released; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Splitting; +import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.apache.commons.lang3.StringUtils.isNotBlank; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData.state; import com.google.common.annotations.VisibleForTesting; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.common.topics.TopicCompactionStrategy; @@ -50,40 +48,69 @@ public void checkBrokers(boolean check) { @Override public boolean shouldKeepLeft(ServiceUnitStateData from, ServiceUnitStateData to) { - ServiceUnitState prevState = from == null ? Free : from.state(); - ServiceUnitState state = to == null ? Free : to.state(); + if (to == null) { + return false; + } else if (to.force()) { + return false; + } + + + ServiceUnitState prevState = state(from); + ServiceUnitState state = state(to); + if (!ServiceUnitState.isValidTransition(prevState, state)) { return true; } if (checkBrokers) { - if (prevState == Free && (state == Assigned || state == Owned)) { - // Free -> Assigned || Owned broker check - return StringUtils.isBlank(to.broker()); - } else if (prevState == Owned && state == Assigned) { - // Owned -> Assigned(transfer) broker check - return !StringUtils.equals(from.broker(), to.sourceBroker()) - || StringUtils.isBlank(to.broker()) - || StringUtils.equals(from.broker(), to.broker()); - } else if (prevState == Assigned && state == Released) { - // Assigned -> Released(transfer) broker check - return !StringUtils.equals(from.broker(), to.broker()) - || !StringUtils.equals(from.sourceBroker(), to.sourceBroker()); - } else if (prevState == Released && state == Owned) { - // Released -> Owned(transfer) broker check - return !StringUtils.equals(from.broker(), to.broker()) - || !StringUtils.equals(from.sourceBroker(), to.sourceBroker()); - } else if (prevState == Assigned && state == Owned) { - // Assigned -> Owned broker check - return !StringUtils.equals(from.broker(), to.broker()) - || !StringUtils.equals(from.sourceBroker(), to.sourceBroker()); - } else if (prevState == Owned && state == Splitting) { - // Owned -> Splitting broker check - return !StringUtils.equals(from.broker(), to.broker()); + switch (prevState) { + case Owned: + switch (state) { + case Assigning: + return invalidTransfer(from, to); + case Splitting: + case Releasing: + return isNotBlank(to.sourceBroker()) || targetNotEquals(from, to); + } + case Assigning: + switch (state) { + case Releasing: + return isBlank(to.sourceBroker()) || notEquals(from, to); + case Owned: + return isNotBlank(to.sourceBroker()) || targetNotEquals(from, to); + } + case Releasing: + switch (state) { + case Owned: + case Free: + return notEquals(from, to); + } + case Splitting: + switch (state) { + case Deleted: + return notEquals(from, to); + } + case Free: + switch (state) { + case Assigning: + return isNotBlank(to.sourceBroker()); + } } } - return false; } -} + private boolean targetNotEquals(ServiceUnitStateData from, ServiceUnitStateData to) { + return !from.broker().equals(to.broker()); + } + + private boolean notEquals(ServiceUnitStateData from, ServiceUnitStateData to) { + return !from.broker().equals(to.broker()) + || !StringUtils.equals(from.sourceBroker(), to.sourceBroker()); + } + + private boolean invalidTransfer(ServiceUnitStateData from, ServiceUnitStateData to) { + return !from.broker().equals(to.sourceBroker()) + || from.broker().equals(to.broker()); + } +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java index cba459b7875f7..6a04431de64d5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java @@ -20,24 +20,36 @@ import java.util.Objects; +import org.apache.commons.lang3.StringUtils; /** * Defines data for the service unit state changes. * This data will be broadcast in ServiceUnitStateChannel. */ -public record ServiceUnitStateData(ServiceUnitState state, String broker, String sourceBroker, long timestamp) { +public record ServiceUnitStateData( + ServiceUnitState state, String broker, String sourceBroker, boolean force, long timestamp) { public ServiceUnitStateData { Objects.requireNonNull(state); - Objects.requireNonNull(broker); + if (StringUtils.isBlank(broker)) { + throw new IllegalArgumentException("Empty broker"); + } } public ServiceUnitStateData(ServiceUnitState state, String broker, String sourceBroker) { - this(state, broker, sourceBroker, System.currentTimeMillis()); + this(state, broker, sourceBroker, false, System.currentTimeMillis()); } public ServiceUnitStateData(ServiceUnitState state, String broker) { - this(state, broker, null, System.currentTimeMillis()); + this(state, broker, null, false, System.currentTimeMillis()); + } + + public ServiceUnitStateData(ServiceUnitState state, String broker, boolean force) { + this(state, broker, null, force, System.currentTimeMillis()); + } + + public static ServiceUnitState state(ServiceUnitStateData data) { + return data == null ? ServiceUnitState.Init : data.state(); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java index 9dc4ec649b62b..37b03e275d6bf 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java @@ -38,7 +38,6 @@ import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.PulsarClient; -import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.client.impl.BatchMessageIdImpl; import org.apache.pulsar.client.impl.CompactionReaderImpl; @@ -63,6 +62,7 @@ public class StrategicTwoPhaseCompactor extends TwoPhaseCompactor { private static final Logger log = LoggerFactory.getLogger(StrategicTwoPhaseCompactor.class); private static final int MAX_OUTSTANDING = 500; + private static final int MAX_READER_RECONNECT_WAITING_TIME_IN_MILLIS = 20 * 1000; private final Duration phaseOneLoopReadTimeout; private final RawBatchMessageContainerImpl batchMessageContainer; @@ -110,7 +110,7 @@ CompletableFuture doCompaction(Reader reader, TopicCompactionStrate if (!(reader instanceof CompactionReaderImpl)) { return CompletableFuture.failedFuture( - new IllegalStateException("reader has to be DelayedAckReaderImpl")); + new IllegalStateException("reader has to be CompactionReaderImpl")); } return reader.hasMessageAvailableAsync() .thenCompose(available -> { @@ -284,9 +284,12 @@ private void phaseOneLoop(Reader reader, CompletableFuture void phaseOneLoop(Reader reader, CompletableFuture void waitForReconnection(Reader reader) { + long started = System.currentTimeMillis(); + + // initial sleep + try { + Thread.sleep(100); + } catch (InterruptedException e) { + } + while (!reader.isConnected()) { + long now = System.currentTimeMillis(); + if (now - started > MAX_READER_RECONNECT_WAITING_TIME_IN_MILLIS) { + String errorMsg = String.format( + "Reader has not been reconnected for %d secs. Stopping the compaction.", + MAX_READER_RECONNECT_WAITING_TIME_IN_MILLIS / 1000); + log.error(errorMsg); + throw new RuntimeException(errorMsg); + } + log.warn( + "Reader has not been reconnected after the cursor reset. elapsed :{} ms. Retrying " + + "soon.", now - started); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + log.warn("The thread got interrupted while waiting. continuing", e); + } + } + } + private CompletableFuture phaseTwo(PhaseOneResult phaseOneResult, Reader reader, BookKeeper bk) { log.info("Completed phase one. Result:{}. ", phaseOneResult); Map metadata = diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index 1ef4f660e4af3..001aac34a4ba2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -18,14 +18,6 @@ */ package org.apache.pulsar.broker.loadbalance.extensions; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Assigned; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Free; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Owned; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Released; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Splitting; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.EventType.Assign; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.EventType.Split; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.EventType.Unload; import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Admin; import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Bandwidth; import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.MsgRate; @@ -53,6 +45,7 @@ import static org.testng.Assert.assertTrue; import com.google.common.collect.Sets; +import java.util.LinkedHashMap; import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -73,6 +66,7 @@ import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.LeaderBroker; import org.apache.pulsar.broker.loadbalance.LeaderElectionService; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData; @@ -342,17 +336,19 @@ public void testGetMetrics() throws Exception { FieldUtils.writeDeclaredField(unloadCounter, "loadAvg", 1.5, true); FieldUtils.writeDeclaredField(unloadCounter, "loadStd", 0.3, true); FieldUtils.writeDeclaredField(unloadCounter, "breakdownCounters", Map.of( - Success, Map.of( - Overloaded, new MutableLong(1), - Underloaded, new MutableLong(2)), - Skip, Map.of( - Balanced, new MutableLong(3), - NoBundles, new MutableLong(4), - CoolDown, new MutableLong(5), - OutDatedData, new MutableLong(6), - NoLoadData, new MutableLong(7), - NoBrokers, new MutableLong(8), - Unknown, new MutableLong(9)), + Success, new LinkedHashMap<>() {{ + put(Overloaded, new MutableLong(1)); + put(Underloaded, new MutableLong(2)); + }}, + Skip, new LinkedHashMap<>() {{ + put(Balanced, new MutableLong(3)); + put(NoBundles, new MutableLong(4)); + put(CoolDown, new MutableLong(5)); + put(OutDatedData, new MutableLong(6)); + put(NoLoadData, new MutableLong(7)); + put(NoBrokers, new MutableLong(8)); + put(Unknown, new MutableLong(9)); + }}, Failure, Map.of( Unknown, new MutableLong(10)) ), true); @@ -363,19 +359,24 @@ Unknown, new MutableLong(10)) FieldUtils.readDeclaredField(primaryLoadManager, "splitMetrics", true); SplitCounter splitCounter = new SplitCounter(); FieldUtils.writeDeclaredField(splitCounter, "splitCount", 35l, true); - FieldUtils.writeDeclaredField(splitCounter, "breakdownCounters", Map.of( - SplitDecision.Label.Success, Map.of( - Topics, new MutableLong(1), - Sessions, new MutableLong(2), - MsgRate, new MutableLong(3), - Bandwidth, new MutableLong(4), - Admin, new MutableLong(5)), - SplitDecision.Label.Skip, Map.of( + FieldUtils.writeDeclaredField(splitCounter, "breakdownCounters", new LinkedHashMap<>() { + { + put(SplitDecision.Label.Success, new LinkedHashMap<>() { + { + put(Topics, new MutableLong(1)); + put(Sessions, new MutableLong(2)); + put(MsgRate, new MutableLong(3)); + put(Bandwidth, new MutableLong(4)); + put(Admin, new MutableLong(5)); + } + }); + put(SplitDecision.Label.Skip, Map.of( SplitDecision.Reason.Balanced, new MutableLong(6) - ), - SplitDecision.Label.Failure, Map.of( - SplitDecision.Reason.Unknown, new MutableLong(7)) - ), true); + )); + put(SplitDecision.Label.Failure, Map.of( + SplitDecision.Reason.Unknown, new MutableLong(7))); + } + }, true); splitMetrics.set(splitCounter.toMetrics(pulsar.getAdvertisedAddress())); } @@ -391,34 +392,39 @@ SplitDecision.Reason.Unknown, new MutableLong(7)) } { - FieldUtils.writeDeclaredField(channel1, "totalCleanupCnt", 1, true); - FieldUtils.writeDeclaredField(channel1, "totalBrokerCleanupTombstoneCnt", 2, true); - FieldUtils.writeDeclaredField(channel1, "totalServiceUnitCleanupTombstoneCnt", 3, true); - FieldUtils.writeDeclaredField(channel1, "totalCleanupErrorCnt", new AtomicLong(4), true); - FieldUtils.writeDeclaredField(channel1, "totalCleanupScheduledCnt", 5, true); - FieldUtils.writeDeclaredField(channel1, "totalCleanupIgnoredCnt", 6, true); - FieldUtils.writeDeclaredField(channel1, "totalCleanupCancelledCnt", 7, true); - FieldUtils.writeDeclaredField(channel1, "ownerLookUpCounters", Map.of( - Owned, new AtomicLong(1), - Assigned, new AtomicLong(2), - Released, new AtomicLong(3), - Splitting, new AtomicLong(4), - Free, new AtomicLong(5) - ), true); - FieldUtils.writeDeclaredField(channel1, "eventCounters", Map.of( - Assign, new ServiceUnitStateChannelImpl.Counters(new AtomicLong(1), new AtomicLong(2)), - Split, new ServiceUnitStateChannelImpl.Counters(new AtomicLong(3), new AtomicLong(4)), - Unload, new ServiceUnitStateChannelImpl.Counters(new AtomicLong(5), new AtomicLong(6)) - ), true); - - FieldUtils.writeDeclaredField(channel1, "handlerCounters", Map.of( - Owned, new ServiceUnitStateChannelImpl.Counters(new AtomicLong(1), new AtomicLong(2)), - Assigned, new ServiceUnitStateChannelImpl.Counters(new AtomicLong(3), new AtomicLong(4)), - Released, new ServiceUnitStateChannelImpl.Counters(new AtomicLong(5), new AtomicLong(6)), - Splitting, new ServiceUnitStateChannelImpl.Counters(new AtomicLong(7), new AtomicLong(8)), - Free, new ServiceUnitStateChannelImpl.Counters(new AtomicLong(9), new AtomicLong(10)) - ), true); + FieldUtils.writeDeclaredField(channel1, "totalInactiveBrokerCleanupCnt", 1, true); + FieldUtils.writeDeclaredField(channel1, "totalServiceUnitTombstoneCleanupCnt", 2, true); + FieldUtils.writeDeclaredField(channel1, "totalOrphanServiceUnitCleanupCnt", 3, true); + FieldUtils.writeDeclaredField(channel1, "totalCleanupErrorCnt", new AtomicLong(4), true); + FieldUtils.writeDeclaredField(channel1, "totalInactiveBrokerCleanupScheduledCnt", 5, true); + FieldUtils.writeDeclaredField(channel1, "totalInactiveBrokerCleanupIgnoredCnt", 6, true); + FieldUtils.writeDeclaredField(channel1, "totalInactiveBrokerCleanupCancelledCnt", 7, true); + + Map ownerLookUpCounters = new LinkedHashMap<>(); + Map handlerCounters = new LinkedHashMap<>(); + Map eventCounters = + new LinkedHashMap<>(); + int i = 1; + int j = 0; + for (var state : ServiceUnitState.values()) { + ownerLookUpCounters.put(state, new AtomicLong(i)); + handlerCounters.put(state, + new ServiceUnitStateChannelImpl.Counters( + new AtomicLong(j + 1), new AtomicLong(j + 2))); + i++; + j += 2; + } + i = 0; + for (var type : ServiceUnitStateChannelImpl.EventType.values()) { + eventCounters.put(type, + new ServiceUnitStateChannelImpl.Counters( + new AtomicLong(i + 1), new AtomicLong(i + 2))); + i += 2; + } + FieldUtils.writeDeclaredField(channel1, "ownerLookUpCounters", ownerLookUpCounters, true); + FieldUtils.writeDeclaredField(channel1, "eventCounters", eventCounters, true); + FieldUtils.writeDeclaredField(channel1, "handlerCounters", handlerCounters, true); } var expected = Set.of( @@ -428,55 +434,60 @@ Free, new AtomicLong(5) dimensions=[{broker=localhost, feature=max, metric=loadBalancing}], metrics=[{brk_lb_resource_usage=0.04}] dimensions=[{broker=localhost, metric=bundleUnloading}], metrics=[{brk_lb_unload_broker_total=2, brk_lb_unload_bundle_total=3}] dimensions=[{broker=localhost, metric=bundleUnloading, reason=Unknown, result=Failure}], metrics=[{brk_lb_unload_broker_breakdown_total=10}] + dimensions=[{broker=localhost, metric=bundleUnloading, reason=Balanced, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=3}] + dimensions=[{broker=localhost, metric=bundleUnloading, reason=NoBundles, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=4}] dimensions=[{broker=localhost, metric=bundleUnloading, reason=CoolDown, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=5}] dimensions=[{broker=localhost, metric=bundleUnloading, reason=OutDatedData, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=6}] - dimensions=[{broker=localhost, metric=bundleUnloading, reason=NoBundles, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=4}] dimensions=[{broker=localhost, metric=bundleUnloading, reason=NoLoadData, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=7}] dimensions=[{broker=localhost, metric=bundleUnloading, reason=NoBrokers, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=8}] dimensions=[{broker=localhost, metric=bundleUnloading, reason=Unknown, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=9}] - dimensions=[{broker=localhost, metric=bundleUnloading, reason=Balanced, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=3}] - dimensions=[{broker=localhost, metric=bundleUnloading, reason=Underloaded, result=Success}], metrics=[{brk_lb_unload_broker_breakdown_total=2}] dimensions=[{broker=localhost, metric=bundleUnloading, reason=Overloaded, result=Success}], metrics=[{brk_lb_unload_broker_breakdown_total=1}] + dimensions=[{broker=localhost, metric=bundleUnloading, reason=Underloaded, result=Success}], metrics=[{brk_lb_unload_broker_breakdown_total=2}] dimensions=[{broker=localhost, feature=max_ema, metric=bundleUnloading, stat=avg}], metrics=[{brk_lb_resource_usage_stats=1.5}] dimensions=[{broker=localhost, feature=max_ema, metric=bundleUnloading, stat=std}], metrics=[{brk_lb_resource_usage_stats=0.3}] dimensions=[{broker=localhost, metric=bundlesSplit}], metrics=[{brk_lb_bundles_split_total=35}] - dimensions=[{broker=localhost, metric=bundlesSplit, reason=Bandwidth, result=Success}], metrics=[{brk_lb_bundles_split_breakdown_total=4}] + dimensions=[{broker=localhost, metric=bundlesSplit, reason=Topics, result=Success}], metrics=[{brk_lb_bundles_split_breakdown_total=1}] dimensions=[{broker=localhost, metric=bundlesSplit, reason=Sessions, result=Success}], metrics=[{brk_lb_bundles_split_breakdown_total=2}] dimensions=[{broker=localhost, metric=bundlesSplit, reason=MsgRate, result=Success}], metrics=[{brk_lb_bundles_split_breakdown_total=3}] + dimensions=[{broker=localhost, metric=bundlesSplit, reason=Bandwidth, result=Success}], metrics=[{brk_lb_bundles_split_breakdown_total=4}] dimensions=[{broker=localhost, metric=bundlesSplit, reason=Admin, result=Success}], metrics=[{brk_lb_bundles_split_breakdown_total=5}] - dimensions=[{broker=localhost, metric=bundlesSplit, reason=Topics, result=Success}], metrics=[{brk_lb_bundles_split_breakdown_total=1}] dimensions=[{broker=localhost, metric=bundlesSplit, reason=Balanced, result=Skip}], metrics=[{brk_lb_bundles_split_breakdown_total=6}] dimensions=[{broker=localhost, metric=bundlesSplit, reason=Unknown, result=Failure}], metrics=[{brk_lb_bundles_split_breakdown_total=7}] dimensions=[{broker=localhost, metric=assign, result=Empty}], metrics=[{brk_lb_assign_broker_breakdown_total=2}] - dimensions=[{broker=localhost, metric=assign, result=Success}], metrics=[{brk_lb_assign_broker_breakdown_total=1}] dimensions=[{broker=localhost, metric=assign, result=Skip}], metrics=[{brk_lb_assign_broker_breakdown_total=3}] - dimensions=[{broker=localhost, metric=sunitStateChn, state=Splitting}], metrics=[{brk_sunit_state_chn_owner_lookup_total=4}] - dimensions=[{broker=localhost, metric=sunitStateChn, state=Owned}], metrics=[{brk_sunit_state_chn_owner_lookup_total=1}] - dimensions=[{broker=localhost, metric=sunitStateChn, state=Released}], metrics=[{brk_sunit_state_chn_owner_lookup_total=3}] - dimensions=[{broker=localhost, metric=sunitStateChn, state=Free}], metrics=[{brk_sunit_state_chn_owner_lookup_total=5}] - dimensions=[{broker=localhost, metric=sunitStateChn, state=Assigned}], metrics=[{brk_sunit_state_chn_owner_lookup_total=2}] - dimensions=[{broker=localhost, event=Unload, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=5}] - dimensions=[{broker=localhost, event=Unload, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=6}] - dimensions=[{broker=localhost, event=Split, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=3}] - dimensions=[{broker=localhost, event=Split, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=4}] + dimensions=[{broker=localhost, metric=assign, result=Success}], metrics=[{brk_lb_assign_broker_breakdown_total=1}] + dimensions=[{broker=localhost, metric=sunitStateChn, state=Init}], metrics=[{brk_sunit_state_chn_owner_lookup_total=1}] + dimensions=[{broker=localhost, metric=sunitStateChn, state=Free}], metrics=[{brk_sunit_state_chn_owner_lookup_total=2}] + dimensions=[{broker=localhost, metric=sunitStateChn, state=Owned}], metrics=[{brk_sunit_state_chn_owner_lookup_total=3}] + dimensions=[{broker=localhost, metric=sunitStateChn, state=Assigning}], metrics=[{brk_sunit_state_chn_owner_lookup_total=4}] + dimensions=[{broker=localhost, metric=sunitStateChn, state=Releasing}], metrics=[{brk_sunit_state_chn_owner_lookup_total=5}] + dimensions=[{broker=localhost, metric=sunitStateChn, state=Splitting}], metrics=[{brk_sunit_state_chn_owner_lookup_total=6}] + dimensions=[{broker=localhost, metric=sunitStateChn, state=Deleted}], metrics=[{brk_sunit_state_chn_owner_lookup_total=7}] dimensions=[{broker=localhost, event=Assign, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=1}] dimensions=[{broker=localhost, event=Assign, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=2}] - dimensions=[{broker=localhost, event=Splitting, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=7}] - dimensions=[{broker=localhost, event=Splitting, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=8}] - dimensions=[{broker=localhost, event=Owned, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=1}] - dimensions=[{broker=localhost, event=Owned, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=2}] - dimensions=[{broker=localhost, event=Released, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=5}] - dimensions=[{broker=localhost, event=Released, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=6}] - dimensions=[{broker=localhost, event=Free, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=9}] - dimensions=[{broker=localhost, event=Free, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=10}] - dimensions=[{broker=localhost, event=Assigned, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=3}] - dimensions=[{broker=localhost, event=Assigned, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=4}] - dimensions=[{broker=localhost, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_cleanup_ops_total=1}] + dimensions=[{broker=localhost, event=Split, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=3}] + dimensions=[{broker=localhost, event=Split, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=4}] + dimensions=[{broker=localhost, event=Unload, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=5}] + dimensions=[{broker=localhost, event=Unload, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_event_publish_ops_total=6}] + dimensions=[{broker=localhost, event=Init, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=1}] + dimensions=[{broker=localhost, event=Init, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=2}] + dimensions=[{broker=localhost, event=Free, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=3}] + dimensions=[{broker=localhost, event=Free, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=4}] + dimensions=[{broker=localhost, event=Owned, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=5}] + dimensions=[{broker=localhost, event=Owned, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=6}] + dimensions=[{broker=localhost, event=Assigning, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=7}] + dimensions=[{broker=localhost, event=Assigning, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=8}] + dimensions=[{broker=localhost, event=Releasing, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=9}] + dimensions=[{broker=localhost, event=Releasing, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=10}] + dimensions=[{broker=localhost, event=Splitting, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=11}] + dimensions=[{broker=localhost, event=Splitting, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=12}] + dimensions=[{broker=localhost, event=Deleted, metric=sunitStateChn, result=Total}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=13}] + dimensions=[{broker=localhost, event=Deleted, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_subscribe_ops_total=14}] dimensions=[{broker=localhost, metric=sunitStateChn, result=Failure}], metrics=[{brk_sunit_state_chn_cleanup_ops_total=4}] - dimensions=[{broker=localhost, metric=sunitStateChn, result=Skip}], metrics=[{brk_sunit_state_chn_cleanup_ops_total=6}] - dimensions=[{broker=localhost, metric=sunitStateChn, result=Cancel}], metrics=[{brk_sunit_state_chn_cleanup_ops_total=7}] - dimensions=[{broker=localhost, metric=sunitStateChn, result=Schedule}], metrics=[{brk_sunit_state_chn_cleanup_ops_total=5}] - dimensions=[{broker=localhost, metric=sunitStateChn}], metrics=[{brk_sunit_state_chn_broker_cleanup_ops_total=2, brk_sunit_state_chn_su_cleanup_ops_total=3}] + dimensions=[{broker=localhost, metric=sunitStateChn, result=Skip}], metrics=[{brk_sunit_state_chn_inactive_broker_cleanup_ops_total=6}] + dimensions=[{broker=localhost, metric=sunitStateChn, result=Cancel}], metrics=[{brk_sunit_state_chn_inactive_broker_cleanup_ops_total=7}] + dimensions=[{broker=localhost, metric=sunitStateChn, result=Schedule}], metrics=[{brk_sunit_state_chn_inactive_broker_cleanup_ops_total=5}] + dimensions=[{broker=localhost, metric=sunitStateChn}], metrics=[{brk_sunit_state_chn_inactive_broker_cleanup_ops_total=1, brk_sunit_state_chn_orphan_su_cleanup_ops_total=3, brk_sunit_state_chn_su_tombstone_cleanup_ops_total=2}] """.split("\n")); var actual = primaryLoadManager.getMetrics().stream().map(m -> m.toString()).collect(Collectors.toSet()); assertEquals(actual, expected); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java index 327afa3cb8891..49eee6ecb7aef 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java @@ -18,15 +18,18 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.channel; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Assigned; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Assigning; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Deleted; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Free; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Init; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Owned; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Released; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Releasing; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Splitting; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.EventType.Assign; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.EventType.Split; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.EventType.Unload; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.MAX_CLEAN_UP_DELAY_TIME_IN_SECS; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData.state; import static org.apache.pulsar.metadata.api.extended.SessionEvent.ConnectionLost; import static org.apache.pulsar.metadata.api.extended.SessionEvent.Reconnected; import static org.apache.pulsar.metadata.api.extended.SessionEvent.SessionLost; @@ -52,6 +55,7 @@ import static org.testng.AssertJUnit.assertNotNull; import java.lang.reflect.Field; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -71,8 +75,11 @@ import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.loadbalance.LeaderElectionService; +import org.apache.pulsar.broker.loadbalance.extensions.BrokerRegistryImpl; +import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.models.Split; import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; +import org.apache.pulsar.broker.loadbalance.extensions.strategy.BrokerSelectionStrategy; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.api.Producer; @@ -104,6 +111,12 @@ public class ServiceUnitStateChannelTest extends MockedPulsarServiceBaseTest { private String bundle2; private PulsarTestContext additionalPulsarTestContext; + private LoadManagerContext loadManagerContext; + + private BrokerRegistryImpl registry; + + private BrokerSelectionStrategy brokerSelector; + @BeforeClass @Override protected void setup() throws Exception { @@ -117,11 +130,16 @@ protected void setup() throws Exception { admin.namespaces().createNamespace("public/default"); pulsar1 = pulsar; + registry = new BrokerRegistryImpl(pulsar); + loadManagerContext = mock(LoadManagerContext.class); + brokerSelector = mock(BrokerSelectionStrategy.class); additionalPulsarTestContext = createAdditionalPulsarTestContext(getDefaultConf()); pulsar2 = additionalPulsarTestContext.getPulsarService(); - channel1 = spy(new ServiceUnitStateChannelImpl(pulsar1)); + + channel1 = createChannel(pulsar1); channel1.start(); - channel2 = spy(new ServiceUnitStateChannelImpl(pulsar2)); + + channel2 = createChannel(pulsar2); channel2.start(); lookupServiceAddress1 = (String) FieldUtils.readDeclaredField(channel1, "lookupServiceAddress", true); @@ -137,6 +155,8 @@ protected void setup() throws Exception { protected void initTableViews() throws Exception { cleanTableView(channel1, bundle); cleanTableView(channel2, bundle); + cleanOwnershipMonitorCounters(channel1); + cleanOwnershipMonitorCounters(channel2); cleanOpsCounters(channel1); cleanOpsCounters(channel2); } @@ -187,7 +207,7 @@ public void channelOwnerTest() throws Exception { public void channelValidationTest() throws ExecutionException, InterruptedException, IllegalAccessException, PulsarServerException, TimeoutException { - var channel = new ServiceUnitStateChannelImpl(pulsar); + var channel = createChannel(pulsar); int errorCnt = validateChannelStart(channel); assertEquals(6, errorCnt); ExecutorService executor = Executors.newSingleThreadExecutor(); @@ -333,8 +353,8 @@ public void assignmentTest() assertEquals(getOwnerRequests1.size(), 0); assertEquals(getOwnerRequests2.size(), 0); - validateHandlerCounters(channel1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0); - validateHandlerCounters(channel2, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0); + validateHandlerCounters(channel1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateHandlerCounters(channel2, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); validateEventCounters(channel1, 1, 0, 0, 0, 0, 0); validateEventCounters(channel2, 1, 0, 0, 0, 0, 0); } @@ -379,7 +399,7 @@ public void assignmentTestWhenOneAssignmentFails() } @Test(priority = 4) - public void unloadTest() + public void transferTest() throws ExecutionException, InterruptedException, TimeoutException, IllegalAccessException { var owner1 = channel1.getOwnerAsync(bundle); @@ -409,14 +429,14 @@ public void unloadTest() assertEquals(ownerAddr1, ownerAddr2); assertEquals(ownerAddr1, Optional.of(lookupServiceAddress2)); - validateHandlerCounters(channel1, 2, 0, 2, 0, 1, 0, 0, 0, 0, 0); - validateHandlerCounters(channel2, 2, 0, 2, 0, 1, 0, 0, 0, 0, 0); + validateHandlerCounters(channel1, 2, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0); + validateHandlerCounters(channel2, 2, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0); validateEventCounters(channel1, 1, 0, 0, 0, 1, 0); validateEventCounters(channel2, 0, 0, 0, 0, 0, 0); } @Test(priority = 5) - public void unloadTestWhenDestBrokerFails() + public void transferTestWhenDestBrokerFails() throws ExecutionException, InterruptedException, IllegalAccessException { var getOwnerRequests1 = getOwnerRequests(channel1); @@ -450,8 +470,8 @@ public void unloadTestWhenDestBrokerFails() Unload unload = new Unload(lookupServiceAddress1, bundle, Optional.of(lookupServiceAddress2)); channel1.publishUnloadEventAsync(unload); // channel1 is broken. the ownership transfer won't be complete. - waitUntilNewState(channel1, bundle); - waitUntilNewState(channel2, bundle); + waitUntilState(channel1, bundle); + waitUntilState(channel2, bundle); var owner1 = channel1.getOwnerAsync(bundle); var owner2 = channel2.getOwnerAsync(bundle); @@ -461,7 +481,7 @@ public void unloadTestWhenDestBrokerFails() assertEquals(1, getOwnerRequests1.size()); assertEquals(1, getOwnerRequests2.size()); - // In 10 secs, the getOwnerAsync requests(lookup requests) should time out. + // In 5 secs, the getOwnerAsync requests(lookup requests) should time out. Awaitility.await().atMost(5, TimeUnit.SECONDS) .untilAsserted(() -> assertTrue(owner1.isCompletedExceptionally())); Awaitility.await().atMost(5, TimeUnit.SECONDS) @@ -470,19 +490,42 @@ public void unloadTestWhenDestBrokerFails() assertEquals(0, getOwnerRequests1.size()); assertEquals(0, getOwnerRequests2.size()); + // recovered, check the monitor update state : Assigned -> Owned + FieldUtils.writeDeclaredField(channel2, "producer", producer, true); + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 1 , true); + FieldUtils.writeDeclaredField(channel2, + "inFlightStateWaitingTimeInMillis", 1 , true); + + ((ServiceUnitStateChannelImpl) channel1).monitorOwnerships( + List.of(lookupServiceAddress1, lookupServiceAddress2)); + ((ServiceUnitStateChannelImpl) channel2).monitorOwnerships( + List.of(lookupServiceAddress1, lookupServiceAddress2)); + + + waitUntilNewOwner(channel1, bundle, lookupServiceAddress1); + waitUntilNewOwner(channel2, bundle, lookupServiceAddress1); + ownerAddr1 = channel1.getOwnerAsync(bundle).get(); + ownerAddr2 = channel2.getOwnerAsync(bundle).get(); + + assertEquals(ownerAddr1, ownerAddr2); + assertEquals(ownerAddr1, Optional.of(lookupServiceAddress1)); + + var leader = channel1.isChannelOwnerAsync().get() ? channel1 : channel2; + validateMonitorCounters(leader, + 0, + 0, + 1, + 0, + 0, + 0, + 0); - // TODO: retry lookups and assert that the monitor cleans up the stuck assignments - /* - owner1 = channel1.getOwnerAsync(bundle); - owner2 = channel2.getOwnerAsync(bundle); - assertFalse(channel1.getOwnerAsync(bundle).isDone()); - assertFalse(channel1.getOwnerAsync(bundle).isDone()); - */ FieldUtils.writeDeclaredField(channel1, "inFlightStateWaitingTimeInMillis", 30 * 1000, true); FieldUtils.writeDeclaredField(channel2, "inFlightStateWaitingTimeInMillis", 30 * 1000, true); - FieldUtils.writeDeclaredField(channel2, "producer", producer, true); + } @Test(priority = 6) @@ -514,11 +557,11 @@ public void splitAndRetryTest() throws Exception { Split split = new Split(bundle, ownerAddr1.get(), new HashMap<>()); channel1.publishSplitEventAsync(split); - waitUntilNewOwner(channel1, bundle, null); - waitUntilNewOwner(channel2, bundle, null); + waitUntilState(channel1, bundle, Deleted); + waitUntilState(channel2, bundle, Deleted); - validateHandlerCounters(channel1, 1, 0, 9, 0, 0, 0, 1, 0, 7, 0); - validateHandlerCounters(channel2, 1, 0, 9, 0, 0, 0, 1, 0, 7, 0); + validateHandlerCounters(channel1, 1, 0, 9, 0, 0, 0, 1, 0, 0, 0, 6, 0, 1, 0); + validateHandlerCounters(channel2, 1, 0, 9, 0, 0, 0, 1, 0, 0, 0, 6, 0, 1, 0); validateEventCounters(channel1, 1, 0, 1, 0, 0, 0); validateEventCounters(channel2, 0, 0, 0, 0, 0, 0); // Verify the retry count @@ -538,10 +581,49 @@ public void splitAndRetryTest() throws Exception { assertEquals(Optional.of(lookupServiceAddress1), channel2.getOwnerAsync(childBundle1).get()); assertEquals(Optional.of(lookupServiceAddress1), channel2.getOwnerAsync(childBundle2).get()); + + // try the monitor and check the monitor moves `Deleted` -> `Init` + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 1 , true); + FieldUtils.writeDeclaredField(channel1, + "semiTerminalStateWaitingTimeInMillis", 1, true); + + FieldUtils.writeDeclaredField(channel2, + "inFlightStateWaitingTimeInMillis", 1 , true); + FieldUtils.writeDeclaredField(channel2, + "semiTerminalStateWaitingTimeInMillis", 1, true); + + ((ServiceUnitStateChannelImpl) channel1).monitorOwnerships( + List.of(lookupServiceAddress1, lookupServiceAddress2)); + ((ServiceUnitStateChannelImpl) channel2).monitorOwnerships( + List.of(lookupServiceAddress1, lookupServiceAddress2)); + waitUntilState(channel1, bundle, Init); + waitUntilState(channel2, bundle, Init); + + var leader = channel1.isChannelOwnerAsync().get() ? channel1 : channel2; + validateMonitorCounters(leader, + 0, + 1, + 0, + 0, + 0, + 0, + 0); + cleanTableView(channel1, childBundle1); cleanTableView(channel2, childBundle1); cleanTableView(channel1, childBundle2); cleanTableView(channel2, childBundle2); + + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 30 * 1000, true); + FieldUtils.writeDeclaredField(channel1, + "semiTerminalStateWaitingTimeInMillis", 300 * 1000, true); + + FieldUtils.writeDeclaredField(channel2, + "inFlightStateWaitingTimeInMillis", 30 * 1000, true); + FieldUtils.writeDeclaredField(channel2, + "semiTerminalStateWaitingTimeInMillis", 300 * 1000, true); } @Test(priority = 7) @@ -622,6 +704,7 @@ public void handleBrokerDeletionEventTest() var owner1 = channel1.getOwnerAsync(bundle1); var owner2 = channel2.getOwnerAsync(bundle2); + doReturn(Optional.of(lookupServiceAddress2)).when(brokerSelector).select(any(), any(), any()); assertTrue(owner1.get().isEmpty()); assertTrue(owner2.get().isEmpty()); @@ -634,6 +717,10 @@ public void handleBrokerDeletionEventTest() waitUntilNewOwner(channel1, bundle2, broker); waitUntilNewOwner(channel2, bundle2, broker); + channel1.publishUnloadEventAsync(new Unload(broker, bundle1, Optional.of(lookupServiceAddress2))); + waitUntilNewOwner(channel1, bundle1, lookupServiceAddress2); + waitUntilNewOwner(channel2, bundle1, lookupServiceAddress2); + // test stable metadata state leaderChannel.handleMetadataSessionEvent(SessionReestablished); followerChannel.handleMetadataSessionEvent(SessionReestablished); @@ -644,26 +731,30 @@ public void handleBrokerDeletionEventTest() leaderChannel.handleBrokerRegistrationEvent(broker, NotificationType.Deleted); followerChannel.handleBrokerRegistrationEvent(broker, NotificationType.Deleted); - waitUntilNewOwner(channel1, bundle1, null); - waitUntilNewOwner(channel2, bundle1, null); - waitUntilNewOwner(channel1, bundle2, null); - waitUntilNewOwner(channel2, bundle2, null); + waitUntilNewOwner(channel1, bundle1, lookupServiceAddress2); + waitUntilNewOwner(channel2, bundle1, lookupServiceAddress2); + waitUntilNewOwner(channel1, bundle2, lookupServiceAddress2); + waitUntilNewOwner(channel2, bundle2, lookupServiceAddress2); verify(leaderCleanupJobs, times(1)).computeIfAbsent(eq(broker), any()); verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any()); + + assertEquals(0, leaderCleanupJobs.size()); assertEquals(0, followerCleanupJobs.size()); - assertEquals(1, getCleanupMetric(leaderChannel, "totalCleanupCnt")); - assertEquals(1, getCleanupMetric(leaderChannel, "totalBrokerCleanupTombstoneCnt")); - assertEquals(2, getCleanupMetric(leaderChannel, "totalServiceUnitCleanupTombstoneCnt")); - assertEquals(0, getCleanupMetric(leaderChannel, "totalCleanupErrorCnt")); - assertEquals(1, getCleanupMetric(leaderChannel, "totalCleanupScheduledCnt")); - assertEquals(0, getCleanupMetric(leaderChannel, "totalCleanupIgnoredCnt")); - assertEquals(0, getCleanupMetric(leaderChannel, "totalCleanupCancelledCnt")); + validateMonitorCounters(leaderChannel, + 1, + 0, + 1, + 0, + 1, + 0, + 0); + // test jittery metadata state - channel1.publishAssignEventAsync(bundle1, broker); - channel2.publishAssignEventAsync(bundle2, broker); + channel1.publishUnloadEventAsync(new Unload(lookupServiceAddress2, bundle1, Optional.of(broker))); + channel1.publishUnloadEventAsync(new Unload(lookupServiceAddress2, bundle2, Optional.of(broker))); waitUntilNewOwner(channel1, bundle1, broker); waitUntilNewOwner(channel2, bundle1, broker); waitUntilNewOwner(channel1, bundle2, broker); @@ -678,13 +769,14 @@ public void handleBrokerDeletionEventTest() verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any()); assertEquals(1, leaderCleanupJobs.size()); assertEquals(0, followerCleanupJobs.size()); - assertEquals(1, getCleanupMetric(leaderChannel, "totalCleanupCnt")); - assertEquals(1, getCleanupMetric(leaderChannel, "totalBrokerCleanupTombstoneCnt")); - assertEquals(2, getCleanupMetric(leaderChannel, "totalServiceUnitCleanupTombstoneCnt")); - assertEquals(0, getCleanupMetric(leaderChannel, "totalCleanupErrorCnt")); - assertEquals(2, getCleanupMetric(leaderChannel, "totalCleanupScheduledCnt")); - assertEquals(0, getCleanupMetric(leaderChannel, "totalCleanupIgnoredCnt")); - assertEquals(0, getCleanupMetric(leaderChannel, "totalCleanupCancelledCnt")); + validateMonitorCounters(leaderChannel, + 1, + 0, + 1, + 0, + 2, + 0, + 0); // broker is back online leaderChannel.handleBrokerRegistrationEvent(broker, NotificationType.Created); @@ -694,13 +786,14 @@ public void handleBrokerDeletionEventTest() verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any()); assertEquals(0, leaderCleanupJobs.size()); assertEquals(0, followerCleanupJobs.size()); - assertEquals(1, getCleanupMetric(leaderChannel, "totalCleanupCnt")); - assertEquals(1, getCleanupMetric(leaderChannel, "totalBrokerCleanupTombstoneCnt")); - assertEquals(2, getCleanupMetric(leaderChannel, "totalServiceUnitCleanupTombstoneCnt")); - assertEquals(0, getCleanupMetric(leaderChannel, "totalCleanupErrorCnt")); - assertEquals(2, getCleanupMetric(leaderChannel, "totalCleanupScheduledCnt")); - assertEquals(0, getCleanupMetric(leaderChannel, "totalCleanupIgnoredCnt")); - assertEquals(1, getCleanupMetric(leaderChannel, "totalCleanupCancelledCnt")); + validateMonitorCounters(leaderChannel, + 1, + 0, + 1, + 0, + 2, + 0, + 1); // broker is offline again @@ -712,35 +805,37 @@ public void handleBrokerDeletionEventTest() verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any()); assertEquals(1, leaderCleanupJobs.size()); assertEquals(0, followerCleanupJobs.size()); - assertEquals(1, getCleanupMetric(leaderChannel, "totalCleanupCnt")); - assertEquals(1, getCleanupMetric(leaderChannel, "totalBrokerCleanupTombstoneCnt")); - assertEquals(2, getCleanupMetric(leaderChannel, "totalServiceUnitCleanupTombstoneCnt")); - assertEquals(0, getCleanupMetric(leaderChannel, "totalCleanupErrorCnt")); - assertEquals(3, getCleanupMetric(leaderChannel, "totalCleanupScheduledCnt")); - assertEquals(0, getCleanupMetric(leaderChannel, "totalCleanupIgnoredCnt")); - assertEquals(1, getCleanupMetric(leaderChannel, "totalCleanupCancelledCnt")); + validateMonitorCounters(leaderChannel, + 1, + 0, + 1, + 0, + 3, + 0, + 1); // finally cleanup - waitUntilNewOwner(channel1, bundle1, null); - waitUntilNewOwner(channel2, bundle1, null); - waitUntilNewOwner(channel1, bundle2, null); - waitUntilNewOwner(channel2, bundle2, null); + waitUntilNewOwner(channel1, bundle1, lookupServiceAddress2); + waitUntilNewOwner(channel2, bundle1, lookupServiceAddress2); + waitUntilNewOwner(channel1, bundle2, lookupServiceAddress2); + waitUntilNewOwner(channel2, bundle2, lookupServiceAddress2); verify(leaderCleanupJobs, times(3)).computeIfAbsent(eq(broker), any()); verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any()); assertEquals(0, leaderCleanupJobs.size()); assertEquals(0, followerCleanupJobs.size()); - assertEquals(2, getCleanupMetric(leaderChannel, "totalCleanupCnt")); - assertEquals(2, getCleanupMetric(leaderChannel, "totalBrokerCleanupTombstoneCnt")); - assertEquals(4, getCleanupMetric(leaderChannel, "totalServiceUnitCleanupTombstoneCnt")); - assertEquals(0, getCleanupMetric(leaderChannel, "totalCleanupErrorCnt")); - assertEquals(3, getCleanupMetric(leaderChannel, "totalCleanupScheduledCnt")); - assertEquals(0, getCleanupMetric(leaderChannel, "totalCleanupIgnoredCnt")); - assertEquals(1, getCleanupMetric(leaderChannel, "totalCleanupCancelledCnt")); + validateMonitorCounters(leaderChannel, + 2, + 0, + 3, + 0, + 3, + 0, + 1); // test unstable state - channel1.publishAssignEventAsync(bundle1, broker); - channel2.publishAssignEventAsync(bundle2, broker); + channel1.publishUnloadEventAsync(new Unload(lookupServiceAddress2, bundle1, Optional.of(broker))); + channel1.publishUnloadEventAsync(new Unload(lookupServiceAddress2, bundle2, Optional.of(broker))); waitUntilNewOwner(channel1, bundle1, broker); waitUntilNewOwner(channel2, bundle1, broker); waitUntilNewOwner(channel1, bundle2, broker); @@ -755,13 +850,14 @@ public void handleBrokerDeletionEventTest() verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any()); assertEquals(0, leaderCleanupJobs.size()); assertEquals(0, followerCleanupJobs.size()); - assertEquals(2, getCleanupMetric(leaderChannel, "totalCleanupCnt")); - assertEquals(2, getCleanupMetric(leaderChannel, "totalBrokerCleanupTombstoneCnt")); - assertEquals(4, getCleanupMetric(leaderChannel, "totalServiceUnitCleanupTombstoneCnt")); - assertEquals(0, getCleanupMetric(leaderChannel, "totalCleanupErrorCnt")); - assertEquals(3, getCleanupMetric(leaderChannel, "totalCleanupScheduledCnt")); - assertEquals(1, getCleanupMetric(leaderChannel, "totalCleanupIgnoredCnt")); - assertEquals(1, getCleanupMetric(leaderChannel, "totalCleanupCancelledCnt")); + validateMonitorCounters(leaderChannel, + 2, + 0, + 3, + 0, + 3, + 1, + 1); // clean-up FieldUtils.writeDeclaredField(leaderChannel, "maxCleanupDelayTimeInSecs", 3 * 60, true); @@ -774,9 +870,7 @@ public void handleBrokerDeletionEventTest() @Test(priority = 10) public void conflictAndCompactionTest() throws ExecutionException, InterruptedException, TimeoutException, IllegalAccessException, PulsarClientException, PulsarServerException { - - var producer = (Producer) FieldUtils.readDeclaredField(channel1, "producer", true); - producer.newMessage().key(bundle).send(); + String bundle = String.format("%s/%s", "public/default", "0x0000000a_0xffffffff"); var owner1 = channel1.getOwnerAsync(bundle); var owner2 = channel2.getOwnerAsync(bundle); assertTrue(owner1.get().isEmpty()); @@ -815,7 +909,7 @@ public void conflictAndCompactionTest() throws ExecutionException, InterruptedEx .untilAsserted(() -> verify(compactor, times(1)) .compact(eq(ServiceUnitStateChannelImpl.TOPIC), any())); - var channel3 = new ServiceUnitStateChannelImpl(pulsar1); + var channel3 = createChannel(pulsar); channel3.start(); Awaitility.await() .pollInterval(200, TimeUnit.MILLISECONDS) @@ -830,10 +924,7 @@ public void conflictAndCompactionTest() throws ExecutionException, InterruptedEx @Test(priority = 11) public void ownerLookupCountTests() throws IllegalAccessException { - overrideTableView(channel1, bundle, null); - channel1.getOwnerAsync(bundle); - - overrideTableView(channel1, bundle, new ServiceUnitStateData(Assigned, "b1")); + overrideTableView(channel1, bundle, new ServiceUnitStateData(Assigning, "b1")); channel1.getOwnerAsync(bundle); channel1.getOwnerAsync(bundle); @@ -842,19 +933,273 @@ public void ownerLookupCountTests() throws IllegalAccessException { channel1.getOwnerAsync(bundle); channel1.getOwnerAsync(bundle); - overrideTableView(channel1, bundle, new ServiceUnitStateData(Released, "b1")); + overrideTableView(channel1, bundle, new ServiceUnitStateData(Releasing, "b1")); channel1.getOwnerAsync(bundle); channel1.getOwnerAsync(bundle); overrideTableView(channel1, bundle, new ServiceUnitStateData(Splitting, "b1")); channel1.getOwnerAsync(bundle); - validateOwnerLookUpCounters(channel1, 2, 3, 2, 1, 1); + overrideTableView(channel1, bundle, new ServiceUnitStateData(Free, "b1")); + channel1.getOwnerAsync(bundle); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Deleted, "b1")); + channel1.getOwnerAsync(bundle); + channel1.getOwnerAsync(bundle); + + overrideTableView(channel1, bundle, null); + channel1.getOwnerAsync(bundle); + channel1.getOwnerAsync(bundle); + channel1.getOwnerAsync(bundle); + + validateOwnerLookUpCounters(channel1, 2, 3, 2, 1, 1, 2, 3); + + } + + @Test(priority = 12) + public void unloadTest() + throws ExecutionException, InterruptedException, IllegalAccessException { + + channel1.publishAssignEventAsync(bundle, lookupServiceAddress1); + + waitUntilNewOwner(channel1, bundle, lookupServiceAddress1); + waitUntilNewOwner(channel2, bundle, lookupServiceAddress1); + var ownerAddr1 = channel1.getOwnerAsync(bundle).get(); + var ownerAddr2 = channel2.getOwnerAsync(bundle).get(); + + assertEquals(ownerAddr1, ownerAddr2); + assertEquals(ownerAddr1, Optional.of(lookupServiceAddress1)); + Unload unload = new Unload(lookupServiceAddress1, bundle, Optional.empty()); + + channel1.publishUnloadEventAsync(unload); + + waitUntilState(channel1, bundle, Free); + waitUntilState(channel2, bundle, Free); + var owner1 = channel1.getOwnerAsync(bundle); + var owner2 = channel2.getOwnerAsync(bundle); + + assertEquals(Optional.empty(), owner1.get()); + assertEquals(Optional.empty(), owner2.get()); + + channel2.publishAssignEventAsync(bundle, lookupServiceAddress2); + + waitUntilNewOwner(channel1, bundle, lookupServiceAddress2); + waitUntilNewOwner(channel2, bundle, lookupServiceAddress2); + + ownerAddr1 = channel1.getOwnerAsync(bundle).get(); + ownerAddr2 = channel2.getOwnerAsync(bundle).get(); + + assertEquals(ownerAddr1, ownerAddr2); + assertEquals(ownerAddr1, Optional.of(lookupServiceAddress2)); + Unload unload2 = new Unload(lookupServiceAddress2, bundle, Optional.empty()); + + channel2.publishUnloadEventAsync(unload2); + + waitUntilState(channel1, bundle, Free); + waitUntilState(channel2, bundle, Free); + + // test monitor if Free -> Init + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 1 , true); + FieldUtils.writeDeclaredField(channel1, + "semiTerminalStateWaitingTimeInMillis", 1, true); + + FieldUtils.writeDeclaredField(channel2, + "inFlightStateWaitingTimeInMillis", 1 , true); + FieldUtils.writeDeclaredField(channel2, + "semiTerminalStateWaitingTimeInMillis", 1, true); + + ((ServiceUnitStateChannelImpl) channel1).monitorOwnerships( + List.of(lookupServiceAddress1, lookupServiceAddress2)); + ((ServiceUnitStateChannelImpl) channel2).monitorOwnerships( + List.of(lookupServiceAddress1, lookupServiceAddress2)); + waitUntilState(channel1, bundle, Init); + waitUntilState(channel2, bundle, Init); + + var leader = channel1.isChannelOwnerAsync().get() ? channel1 : channel2; + validateMonitorCounters(leader, + 0, + 1, + 0, + 0, + 0, + 0, + 0); + + + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 30 * 1000, true); + FieldUtils.writeDeclaredField(channel1, + "semiTerminalStateWaitingTimeInMillis", 30 * 1000, true); + + FieldUtils.writeDeclaredField(channel2, + "inFlightStateWaitingTimeInMillis", 300 * 1000, true); + FieldUtils.writeDeclaredField(channel2, + "semiTerminalStateWaitingTimeInMillis", 300 * 1000, true); + } + + @Test(priority = 13) + public void assignTestWhenDestBrokerFails() + throws ExecutionException, InterruptedException, IllegalAccessException { + + Unload unload = new Unload(lookupServiceAddress1, bundle, Optional.empty()); + + channel1.publishUnloadEventAsync(unload); + + waitUntilState(channel1, bundle, Free); + waitUntilState(channel2, bundle, Free); + + assertEquals(Optional.empty(), channel1.getOwnerAsync(bundle).get()); + assertEquals(Optional.empty(), channel2.getOwnerAsync(bundle).get()); + + var producer = (Producer) FieldUtils.readDeclaredField(channel1, + "producer", true); + var spyProducer = spy(producer); + var msg = mock(TypedMessageBuilder.class); + var future = CompletableFuture.failedFuture(new RuntimeException()); + doReturn(msg).when(spyProducer).newMessage(); + doReturn(msg).when(msg).key(any()); + doReturn(msg).when(msg).value(any()); + doReturn(future).when(msg).sendAsync(); + FieldUtils.writeDeclaredField(channel2, "producer", spyProducer, true); + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 3 * 1000, true); + FieldUtils.writeDeclaredField(channel2, + "inFlightStateWaitingTimeInMillis", 3 * 1000, true); + doReturn(Optional.of(lookupServiceAddress2)).when(brokerSelector).select(any(), any(), any()); + channel1.publishAssignEventAsync(bundle, lookupServiceAddress2); + // channel1 is broken. the assign won't be complete. + waitUntilState(channel1, bundle); + waitUntilState(channel2, bundle); + var owner1 = channel1.getOwnerAsync(bundle); + var owner2 = channel2.getOwnerAsync(bundle); + + assertFalse(owner1.isDone()); + assertFalse(owner2.isDone()); + + // In 5 secs, the getOwnerAsync requests(lookup requests) should time out. + Awaitility.await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> assertTrue(owner1.isCompletedExceptionally())); + Awaitility.await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> assertTrue(owner2.isCompletedExceptionally())); + + // recovered, check the monitor update state : Assigned -> Owned + FieldUtils.writeDeclaredField(channel2, "producer", producer, true); + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 1 , true); + FieldUtils.writeDeclaredField(channel2, + "inFlightStateWaitingTimeInMillis", 1 , true); + + ((ServiceUnitStateChannelImpl) channel1).monitorOwnerships( + List.of(lookupServiceAddress1, lookupServiceAddress2)); + ((ServiceUnitStateChannelImpl) channel2).monitorOwnerships( + List.of(lookupServiceAddress1, lookupServiceAddress2)); + + + waitUntilNewOwner(channel1, bundle, lookupServiceAddress2); + waitUntilNewOwner(channel2, bundle, lookupServiceAddress2); + var ownerAddr1 = channel1.getOwnerAsync(bundle).get(); + var ownerAddr2 = channel2.getOwnerAsync(bundle).get(); + + assertEquals(ownerAddr1, ownerAddr2); + assertEquals(ownerAddr1, Optional.of(lookupServiceAddress2)); + + var leader = channel1.isChannelOwnerAsync().get() ? channel1 : channel2; + validateMonitorCounters(leader, + 0, + 0, + 1, + 0, + 0, + 0, + 0); + + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 30 * 1000, true); + FieldUtils.writeDeclaredField(channel2, + "inFlightStateWaitingTimeInMillis", 30 * 1000, true); } + @Test(priority = 14) + public void splitTestWhenDestBrokerFails() + throws ExecutionException, InterruptedException, IllegalAccessException { + + + Unload unload = new Unload(lookupServiceAddress1, bundle, Optional.empty()); + + channel1.publishUnloadEventAsync(unload); + + waitUntilState(channel1, bundle, Free); + waitUntilState(channel2, bundle, Free); + + channel1.publishAssignEventAsync(bundle, lookupServiceAddress1); + + waitUntilState(channel1, bundle, Owned); + waitUntilState(channel2, bundle, Owned); + + assertEquals(lookupServiceAddress1, channel1.getOwnerAsync(bundle).get().get()); + assertEquals(lookupServiceAddress1, channel2.getOwnerAsync(bundle).get().get()); + + var producer = (Producer) FieldUtils.readDeclaredField(channel1, + "producer", true); + var spyProducer = spy(producer); + var msg = mock(TypedMessageBuilder.class); + var future = CompletableFuture.failedFuture(new RuntimeException()); + doReturn(msg).when(spyProducer).newMessage(); + doReturn(msg).when(msg).key(any()); + doReturn(msg).when(msg).value(any()); + doReturn(future).when(msg).sendAsync(); + FieldUtils.writeDeclaredField(channel1, "producer", spyProducer, true); + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 3 * 1000, true); + FieldUtils.writeDeclaredField(channel2, + "inFlightStateWaitingTimeInMillis", 3 * 1000, true); + channel2.publishSplitEventAsync(new Split(bundle, lookupServiceAddress1, null)); + // channel1 is broken. the split won't be complete. + waitUntilState(channel1, bundle); + waitUntilState(channel2, bundle); + var owner1 = channel1.getOwnerAsync(bundle); + var owner2 = channel2.getOwnerAsync(bundle); + + + // recovered, check the monitor update state : Splitting -> Owned + FieldUtils.writeDeclaredField(channel1, "producer", producer, true); + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 1 , true); + FieldUtils.writeDeclaredField(channel2, + "inFlightStateWaitingTimeInMillis", 1 , true); + + ((ServiceUnitStateChannelImpl) channel1).monitorOwnerships( + List.of(lookupServiceAddress1, lookupServiceAddress2)); + ((ServiceUnitStateChannelImpl) channel2).monitorOwnerships( + List.of(lookupServiceAddress1, lookupServiceAddress2)); + + + waitUntilNewOwner(channel1, bundle, lookupServiceAddress1); + waitUntilNewOwner(channel2, bundle, lookupServiceAddress1); + var ownerAddr1 = channel1.getOwnerAsync(bundle).get(); + var ownerAddr2 = channel2.getOwnerAsync(bundle).get(); + + assertEquals(ownerAddr1, ownerAddr2); + assertEquals(ownerAddr1, Optional.of(lookupServiceAddress1)); + + var leader = channel1.isChannelOwnerAsync().get() ? channel1 : channel2; + validateMonitorCounters(leader, + 0, + 0, + 1, + 0, + 0, + 0, + 0); + + FieldUtils.writeDeclaredField(channel1, + "inFlightStateWaitingTimeInMillis", 30 * 1000, true); + FieldUtils.writeDeclaredField(channel2, + "inFlightStateWaitingTimeInMillis", 30 * 1000, true); - // TODO: add the channel recovery test when broker registry is added. + } private static ConcurrentOpenHashMap>> getOwnerRequests( ServiceUnitStateChannel channel) throws IllegalAccessException { @@ -926,7 +1271,7 @@ private static void waitUntilNewOwner(ServiceUnitStateChannel channel, String se }); } - private static void waitUntilNewState(ServiceUnitStateChannel channel, String key) + private static void waitUntilState(ServiceUnitStateChannel channel, String key) throws IllegalAccessException { TableViewImpl tv = (TableViewImpl) FieldUtils.readField(channel, "tableview", true); @@ -943,6 +1288,20 @@ private static void waitUntilNewState(ServiceUnitStateChannel channel, String ke }); } + private static void waitUntilState(ServiceUnitStateChannel channel, String key, ServiceUnitState expected) + throws IllegalAccessException { + TableViewImpl tv = (TableViewImpl) + FieldUtils.readField(channel, "tableview", true); + Awaitility.await() + .pollInterval(200, TimeUnit.MILLISECONDS) + .atMost(10, TimeUnit.SECONDS) + .until(() -> { // wait until true + ServiceUnitStateData data = tv.get(key); + ServiceUnitState actual = state(data); + return actual == expected; + }); + } + private static void cleanTableView(ServiceUnitStateChannel channel, String serviceUnit) throws IllegalAccessException { var tv = (TableViewImpl) @@ -994,6 +1353,16 @@ private static void cleanOpsCounters(ServiceUnitStateChannel channel) } } + private void cleanOwnershipMonitorCounters(ServiceUnitStateChannel channel) throws IllegalAccessException { + FieldUtils.writeDeclaredField(channel, "totalInactiveBrokerCleanupCnt", 0, true); + FieldUtils.writeDeclaredField(channel, "totalServiceUnitTombstoneCleanupCnt", 0, true); + FieldUtils.writeDeclaredField(channel, "totalOrphanServiceUnitCleanupCnt", 0, true); + FieldUtils.writeDeclaredField(channel, "totalCleanupErrorCnt", new AtomicLong(0), true); + FieldUtils.writeDeclaredField(channel, "totalInactiveBrokerCleanupScheduledCnt", 0, true); + FieldUtils.writeDeclaredField(channel, "totalInactiveBrokerCleanupIgnoredCnt", 0, true); + FieldUtils.writeDeclaredField(channel, "totalInactiveBrokerCleanupCancelledCnt", 0, true); + } + private static long getCleanupMetric(ServiceUnitStateChannel channel, String metric) throws IllegalAccessException { Object var = FieldUtils.readDeclaredField(channel, metric, true); @@ -1009,7 +1378,9 @@ private static void validateHandlerCounters(ServiceUnitStateChannel channel, long ownedT, long ownedF, long releasedT, long releasedF, long splittingT, long splittingF, - long freeT, long freeF) + long freeT, long freeF, + long initT, long initF, + long deletedT, long deletedF) throws IllegalAccessException { var handlerCounters = (Map) @@ -1019,16 +1390,20 @@ private static void validateHandlerCounters(ServiceUnitStateChannel channel, .pollInterval(200, TimeUnit.MILLISECONDS) .atMost(10, TimeUnit.SECONDS) .untilAsserted(() -> { // wait until true - assertEquals(assignedT, handlerCounters.get(Assigned).getTotal().get()); - assertEquals(assignedF, handlerCounters.get(Assigned).getFailure().get()); + assertEquals(assignedT, handlerCounters.get(Assigning).getTotal().get()); + assertEquals(assignedF, handlerCounters.get(Assigning).getFailure().get()); assertEquals(ownedT, handlerCounters.get(Owned).getTotal().get()); assertEquals(ownedF, handlerCounters.get(Owned).getFailure().get()); - assertEquals(releasedT, handlerCounters.get(Released).getTotal().get()); - assertEquals(releasedF, handlerCounters.get(Released).getFailure().get()); + assertEquals(releasedT, handlerCounters.get(Releasing).getTotal().get()); + assertEquals(releasedF, handlerCounters.get(Releasing).getFailure().get()); assertEquals(splittingT, handlerCounters.get(Splitting).getTotal().get()); assertEquals(splittingF, handlerCounters.get(Splitting).getFailure().get()); assertEquals(freeT, handlerCounters.get(Free).getTotal().get()); assertEquals(freeF, handlerCounters.get(Free).getFailure().get()); + assertEquals(initT, handlerCounters.get(Init).getTotal().get()); + assertEquals(initF, handlerCounters.get(Init).getFailure().get()); + assertEquals(deletedT, handlerCounters.get(Deleted).getTotal().get()); + assertEquals(deletedF, handlerCounters.get(Deleted).getFailure().get()); }); } @@ -1059,7 +1434,10 @@ private static void validateOwnerLookUpCounters(ServiceUnitStateChannel channel, long owned, long released, long splitting, - long free) + long free, + long deleted, + long init + ) throws IllegalAccessException { var ownerLookUpCounters = (Map) @@ -1069,11 +1447,48 @@ private static void validateOwnerLookUpCounters(ServiceUnitStateChannel channel, .pollInterval(200, TimeUnit.MILLISECONDS) .atMost(10, TimeUnit.SECONDS) .untilAsserted(() -> { // wait until true - assertEquals(assigned, ownerLookUpCounters.get(Assigned).get()); + assertEquals(assigned, ownerLookUpCounters.get(Assigning).get()); assertEquals(owned, ownerLookUpCounters.get(Owned).get()); - assertEquals(released, ownerLookUpCounters.get(Released).get()); + assertEquals(released, ownerLookUpCounters.get(Releasing).get()); assertEquals(splitting, ownerLookUpCounters.get(Splitting).get()); assertEquals(free, ownerLookUpCounters.get(Free).get()); + assertEquals(deleted, ownerLookUpCounters.get(Deleted).get()); + assertEquals(init, ownerLookUpCounters.get(Init).get()); }); } + + private static void validateMonitorCounters(ServiceUnitStateChannel channel, + long totalInactiveBrokerCleanupCnt, + long totalServiceUnitTombstoneCleanupCnt, + long totalOrphanServiceUnitCleanupCnt, + long totalCleanupErrorCnt, + long totalInactiveBrokerCleanupScheduledCnt, + long totalInactiveBrokerCleanupIgnoredCnt, + long totalInactiveBrokerCleanupCancelledCnt) + throws IllegalAccessException { + assertEquals(totalInactiveBrokerCleanupCnt, getCleanupMetric(channel, "totalInactiveBrokerCleanupCnt")); + assertEquals(totalServiceUnitTombstoneCleanupCnt, + getCleanupMetric(channel, "totalServiceUnitTombstoneCleanupCnt")); + assertEquals(totalOrphanServiceUnitCleanupCnt, getCleanupMetric(channel, "totalOrphanServiceUnitCleanupCnt")); + assertEquals(totalCleanupErrorCnt, getCleanupMetric(channel, "totalCleanupErrorCnt")); + assertEquals(totalInactiveBrokerCleanupScheduledCnt, + getCleanupMetric(channel, "totalInactiveBrokerCleanupScheduledCnt")); + assertEquals(totalInactiveBrokerCleanupIgnoredCnt, + getCleanupMetric(channel, "totalInactiveBrokerCleanupIgnoredCnt")); + assertEquals(totalInactiveBrokerCleanupCancelledCnt, + getCleanupMetric(channel, "totalInactiveBrokerCleanupCancelledCnt")); + } + + ServiceUnitStateChannelImpl createChannel(PulsarService pulsar) + throws IllegalAccessException { + var tmpChannel = new ServiceUnitStateChannelImpl(pulsar); + FieldUtils.writeDeclaredField(tmpChannel, "ownershipMonitorDelayTimeInSecs", 5, true); + var channel = spy(tmpChannel); + + doReturn(loadManagerContext).when(channel).getContext(); + doReturn(registry).when(channel).getBrokerRegistry(); + doReturn(brokerSelector).when(channel).getBrokerSelector(); + + return channel; + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java index 49b55f7660a81..1a4aba15f9e6f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java @@ -18,10 +18,12 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.channel; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Assigned; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Assigning; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Deleted; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Free; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Init; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Owned; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Released; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Releasing; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Splitting; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.testng.Assert.assertTrue; @@ -36,7 +38,7 @@ ServiceUnitStateData data(ServiceUnitState state) { } ServiceUnitStateData data(ServiceUnitState state, String dst) { - return new ServiceUnitStateData(state, dst, "broker"); + return new ServiceUnitStateData(state, dst, null); } ServiceUnitStateData data(ServiceUnitState state, String src, String dst) { return new ServiceUnitStateData(state, dst, src); @@ -45,46 +47,95 @@ ServiceUnitStateData data(ServiceUnitState state, String src, String dst) { @Test public void test() throws InterruptedException { String dst = "dst"; - assertTrue(strategy.shouldKeepLeft(data(Free), data(Free))); - assertFalse(strategy.shouldKeepLeft(data(Free), data(Assigned))); - assertTrue(strategy.shouldKeepLeft(data(Free), data(Assigned, ""))); - assertFalse(strategy.shouldKeepLeft(data(Free), data(Owned))); - assertTrue(strategy.shouldKeepLeft(data(Free), data(Owned, ""))); - assertFalse(strategy.shouldKeepLeft(data(Free), data(Released))); - assertFalse(strategy.shouldKeepLeft(data(Free), data(Splitting))); + String src = "src"; + + assertFalse(strategy.shouldKeepLeft( + new ServiceUnitStateData(Init, dst), + new ServiceUnitStateData(Init, dst, true))); + + assertFalse(strategy.shouldKeepLeft( + data(Owned), null)); + + assertTrue(strategy.shouldKeepLeft(data(Init), data(Init))); + assertFalse(strategy.shouldKeepLeft(data(Init), data(Free))); + assertFalse(strategy.shouldKeepLeft(data(Init), data(Assigning))); + assertFalse(strategy.shouldKeepLeft(data(Init), data(Owned))); + assertFalse(strategy.shouldKeepLeft(data(Init), data(Releasing))); + assertFalse(strategy.shouldKeepLeft(data(Init), data(Splitting))); + assertFalse(strategy.shouldKeepLeft(data(Init), data(Deleted))); - assertFalse(strategy.shouldKeepLeft(data(Assigned), data(Free))); - assertTrue(strategy.shouldKeepLeft(data(Assigned), data(Assigned))); - assertTrue(strategy.shouldKeepLeft(data(Assigned, "dst2"), data(Owned, dst))); - assertTrue(strategy.shouldKeepLeft(data(Assigned, "src1", dst), data(Owned, "src2", dst))); - assertFalse(strategy.shouldKeepLeft(data(Assigned), data(Owned))); - assertTrue(strategy.shouldKeepLeft(data(Assigned, "dst2"), data(Released, dst))); - assertTrue(strategy.shouldKeepLeft(data(Assigned, "src1", dst), data(Released, "src2", dst))); - assertFalse(strategy.shouldKeepLeft(data(Assigned, dst), data(Released, dst))); - assertTrue(strategy.shouldKeepLeft(data(Assigned), data(Splitting))); + assertTrue(strategy.shouldKeepLeft(data(Assigning), data(Init))); + assertTrue(strategy.shouldKeepLeft(data(Assigning), data(Free))); + assertTrue(strategy.shouldKeepLeft(data(Assigning), data(Assigning))); + assertTrue(strategy.shouldKeepLeft(data(Assigning, "dst1"), data(Owned, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Assigning, dst), data(Owned, src, dst))); + assertFalse(strategy.shouldKeepLeft(data(Assigning, dst), data(Owned, dst))); + assertTrue(strategy.shouldKeepLeft(data(Assigning, src, dst), data(Releasing, dst))); + assertTrue(strategy.shouldKeepLeft(data(Assigning, src, "dst1"), data(Releasing, src, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Assigning, "src1", dst), data(Releasing, "src2", dst))); + assertFalse(strategy.shouldKeepLeft(data(Assigning, src, dst), data(Releasing, src, dst))); + assertTrue(strategy.shouldKeepLeft(data(Assigning), data(Splitting, dst))); + assertTrue(strategy.shouldKeepLeft(data(Assigning), data(Deleted, dst))); - assertFalse(strategy.shouldKeepLeft(data(Owned), data(Free))); - assertTrue(strategy.shouldKeepLeft(data(Owned), data(Assigned))); - assertTrue(strategy.shouldKeepLeft(data(Owned), data(Assigned, ""))); - assertTrue(strategy.shouldKeepLeft(data(Owned), data(Assigned, "src", dst))); - assertFalse(strategy.shouldKeepLeft(data(Owned), data(Assigned, dst))); + assertTrue(strategy.shouldKeepLeft(data(Owned), data(Init))); + assertTrue(strategy.shouldKeepLeft(data(Owned), data(Free))); + assertTrue(strategy.shouldKeepLeft(data(Owned, src, "dst1"), data(Assigning, src, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Owned, src, dst), data(Assigning, dst))); + assertTrue(strategy.shouldKeepLeft(data(Owned, src, dst), data(Assigning, src, dst))); + assertTrue(strategy.shouldKeepLeft(data(Owned, src, dst), data(Assigning, dst, dst))); + assertFalse(strategy.shouldKeepLeft(data(Owned, src, dst), data(Assigning, dst, "dst1"))); assertTrue(strategy.shouldKeepLeft(data(Owned), data(Owned))); - assertTrue(strategy.shouldKeepLeft(data(Owned), data(Released))); - assertTrue(strategy.shouldKeepLeft(data(Owned,"dst2"), data(Splitting, dst))); - assertFalse(strategy.shouldKeepLeft(data(Owned), data(Splitting))); + assertTrue(strategy.shouldKeepLeft(data(Owned), data(Releasing, dst))); + assertTrue(strategy.shouldKeepLeft(data(Owned, src, "dst1"), data(Releasing, src, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Owned, "dst1"), data(Releasing, "dst2"))); + assertFalse(strategy.shouldKeepLeft(data(Owned, dst), data(Releasing, dst))); + assertFalse(strategy.shouldKeepLeft(data(Owned, src, dst), data(Releasing, dst))); + assertTrue(strategy.shouldKeepLeft(data(Owned, src, "dst1"), data(Splitting, src, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Owned, "dst1"), data(Splitting, "dst2"))); + assertFalse(strategy.shouldKeepLeft(data(Owned, dst), data(Splitting, dst))); + assertFalse(strategy.shouldKeepLeft(data(Owned, src, dst), data(Splitting, dst))); + assertTrue(strategy.shouldKeepLeft(data(Owned), data(Deleted, dst))); - assertFalse(strategy.shouldKeepLeft(data(Released), data(Free))); - assertTrue(strategy.shouldKeepLeft(data(Released), data(Assigned))); - assertTrue(strategy.shouldKeepLeft(data(Released, "dst2"), data(Owned, dst))); - assertTrue(strategy.shouldKeepLeft(data(Released, "src1", dst), data(Owned, "src2", dst))); - assertFalse(strategy.shouldKeepLeft(data(Released), data(Owned))); - assertTrue(strategy.shouldKeepLeft(data(Released), data(Released))); - assertTrue(strategy.shouldKeepLeft(data(Released), data(Splitting))); + assertTrue(strategy.shouldKeepLeft(data(Releasing), data(Init))); + assertFalse(strategy.shouldKeepLeft(data(Releasing), data(Free))); + assertTrue(strategy.shouldKeepLeft(data(Releasing, "dst1"), data(Free, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Releasing, "src1", dst), data(Free, "src2", dst))); + assertTrue(strategy.shouldKeepLeft(data(Releasing), data(Assigning))); + assertTrue(strategy.shouldKeepLeft(data(Releasing, "dst1"), data(Owned, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Releasing, src, "dst1"), data(Owned, src, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Releasing, "src1", dst), data(Owned, "src2", dst))); + assertFalse(strategy.shouldKeepLeft(data(Releasing, src, dst), data(Owned, src, dst))); + assertTrue(strategy.shouldKeepLeft(data(Releasing), data(Releasing))); + assertTrue(strategy.shouldKeepLeft(data(Releasing), data(Splitting))); + assertTrue(strategy.shouldKeepLeft(data(Releasing), data(Deleted, dst))); - assertFalse(strategy.shouldKeepLeft(data(Splitting), data(Free))); - assertTrue(strategy.shouldKeepLeft(data(Splitting), data(Assigned))); + assertTrue(strategy.shouldKeepLeft(data(Splitting), data(Init))); + assertTrue(strategy.shouldKeepLeft(data(Splitting), data(Free))); + assertTrue(strategy.shouldKeepLeft(data(Splitting), data(Assigning))); assertTrue(strategy.shouldKeepLeft(data(Splitting), data(Owned))); - assertTrue(strategy.shouldKeepLeft(data(Splitting), data(Released))); + assertTrue(strategy.shouldKeepLeft(data(Splitting), data(Releasing))); assertTrue(strategy.shouldKeepLeft(data(Splitting), data(Splitting))); + assertTrue(strategy.shouldKeepLeft(data(Splitting, src, "dst1"), data(Deleted, src, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Splitting, "dst1"), data(Deleted, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Splitting, "src1", dst), data(Deleted, "src2", dst))); + assertFalse(strategy.shouldKeepLeft(data(Splitting, dst), data(Deleted, dst))); + assertFalse(strategy.shouldKeepLeft(data(Splitting, src, dst), data(Deleted, src, dst))); + + assertFalse(strategy.shouldKeepLeft(data(Deleted), data(Init))); + assertTrue(strategy.shouldKeepLeft(data(Deleted), data(Free))); + assertTrue(strategy.shouldKeepLeft(data(Deleted), data(Assigning))); + assertTrue(strategy.shouldKeepLeft(data(Deleted), data(Owned))); + assertTrue(strategy.shouldKeepLeft(data(Deleted), data(Releasing))); + assertTrue(strategy.shouldKeepLeft(data(Deleted), data(Splitting))); + assertTrue(strategy.shouldKeepLeft(data(Deleted), data(Deleted))); + + assertFalse(strategy.shouldKeepLeft(data(Free), data(Init))); + assertTrue(strategy.shouldKeepLeft(data(Free), data(Free))); + assertFalse(strategy.shouldKeepLeft(data(Free), data(Assigning))); + assertTrue(strategy.shouldKeepLeft(data(Free), data(Assigning, src, dst))); + assertTrue(strategy.shouldKeepLeft(data(Free), data(Owned))); + assertTrue(strategy.shouldKeepLeft(data(Free), data(Releasing))); + assertTrue(strategy.shouldKeepLeft(data(Free), data(Splitting))); + assertTrue(strategy.shouldKeepLeft(data(Free), data(Deleted))); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataTest.java index 7b9afee9ce2d1..9617c8a8c2bd0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataTest.java @@ -18,7 +18,7 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.channel; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Assigned; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Assigning; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Owned; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; @@ -41,8 +41,8 @@ public void testConstructors() throws InterruptedException { Thread.sleep(10); - ServiceUnitStateData data2 = new ServiceUnitStateData(Assigned, "A", "B"); - assertEquals(data2.state(), Assigned); + ServiceUnitStateData data2 = new ServiceUnitStateData(Assigning, "A", "B"); + assertEquals(data2.state(), Assigning); assertEquals(data2.broker(), "A"); assertEquals(data2.sourceBroker(), "B"); assertThat(data2.timestamp()).isGreaterThan(data1.timestamp()); @@ -53,15 +53,20 @@ public void testNullState() { new ServiceUnitStateData(null, "A"); } - @Test(expectedExceptions = NullPointerException.class) + @Test(expectedExceptions = IllegalArgumentException.class) public void testNullBroker() { new ServiceUnitStateData(Owned, null); } + @Test(expectedExceptions = IllegalArgumentException.class) + public void testEmptyBroker() { + new ServiceUnitStateData(Owned, ""); + } + @Test public void jsonWriteAndReadTest() throws JsonProcessingException { ObjectMapper mapper = ObjectMapperFactory.create(); - final ServiceUnitStateData src = new ServiceUnitStateData(Assigned, "A", "B"); + final ServiceUnitStateData src = new ServiceUnitStateData(Assigning, "A", "B"); String json = mapper.writeValueAsString(src); ServiceUnitStateData dst = mapper.readValue(json, ServiceUnitStateData.class); assertEquals(dst, src); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTest.java index 69e6a2d204c0e..f5f1fe7bc575f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTest.java @@ -18,10 +18,12 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.channel; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Assigned; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Assigning; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Deleted; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Free; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Init; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Owned; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Released; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Releasing; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Splitting; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; @@ -29,39 +31,75 @@ @Test(groups = "broker") public class ServiceUnitStateTest { + @Test + public void testInFlights() { + assertFalse(ServiceUnitState.isInFlightState(Init)); + assertFalse(ServiceUnitState.isInFlightState(Free)); + assertFalse(ServiceUnitState.isInFlightState(Owned)); + assertTrue(ServiceUnitState.isInFlightState(Assigning)); + assertTrue(ServiceUnitState.isInFlightState(Releasing)); + assertTrue(ServiceUnitState.isInFlightState(Splitting)); + assertFalse(ServiceUnitState.isInFlightState(Deleted)); + } @Test public void testTransitions() { + assertFalse(ServiceUnitState.isValidTransition(Init, Init)); + assertTrue(ServiceUnitState.isValidTransition(Init, Free)); + assertTrue(ServiceUnitState.isValidTransition(Init, Owned)); + assertTrue(ServiceUnitState.isValidTransition(Init, Assigning)); + assertTrue(ServiceUnitState.isValidTransition(Init, Releasing)); + assertTrue(ServiceUnitState.isValidTransition(Init, Splitting)); + assertTrue(ServiceUnitState.isValidTransition(Init, Deleted)); + + assertTrue(ServiceUnitState.isValidTransition(Free, Init)); assertFalse(ServiceUnitState.isValidTransition(Free, Free)); - assertTrue(ServiceUnitState.isValidTransition(Free, Assigned)); - assertTrue(ServiceUnitState.isValidTransition(Free, Owned)); - assertTrue(ServiceUnitState.isValidTransition(Free, Released)); - assertTrue(ServiceUnitState.isValidTransition(Free, Splitting)); + assertFalse(ServiceUnitState.isValidTransition(Free, Owned)); + assertTrue(ServiceUnitState.isValidTransition(Free, Assigning)); + assertFalse(ServiceUnitState.isValidTransition(Free, Releasing)); + assertFalse(ServiceUnitState.isValidTransition(Free, Splitting)); + assertFalse(ServiceUnitState.isValidTransition(Free, Deleted)); - assertTrue(ServiceUnitState.isValidTransition(Assigned, Free)); - assertFalse(ServiceUnitState.isValidTransition(Assigned, Assigned)); - assertTrue(ServiceUnitState.isValidTransition(Assigned, Owned)); - assertTrue(ServiceUnitState.isValidTransition(Assigned, Released)); - assertFalse(ServiceUnitState.isValidTransition(Assigned, Splitting)); + assertFalse(ServiceUnitState.isValidTransition(Assigning, Init)); + assertFalse(ServiceUnitState.isValidTransition(Assigning, Free)); + assertFalse(ServiceUnitState.isValidTransition(Assigning, Assigning)); + assertTrue(ServiceUnitState.isValidTransition(Assigning, Owned)); + assertTrue(ServiceUnitState.isValidTransition(Assigning, Releasing)); + assertFalse(ServiceUnitState.isValidTransition(Assigning, Splitting)); + assertFalse(ServiceUnitState.isValidTransition(Assigning, Deleted)); - assertTrue(ServiceUnitState.isValidTransition(Owned, Free)); - assertTrue(ServiceUnitState.isValidTransition(Owned, Assigned)); + assertFalse(ServiceUnitState.isValidTransition(Owned, Init)); + assertFalse(ServiceUnitState.isValidTransition(Owned, Free)); + assertTrue(ServiceUnitState.isValidTransition(Owned, Assigning)); assertFalse(ServiceUnitState.isValidTransition(Owned, Owned)); - assertFalse(ServiceUnitState.isValidTransition(Owned, Released)); + assertTrue(ServiceUnitState.isValidTransition(Owned, Releasing)); assertTrue(ServiceUnitState.isValidTransition(Owned, Splitting)); + assertFalse(ServiceUnitState.isValidTransition(Owned, Deleted)); - assertTrue(ServiceUnitState.isValidTransition(Released, Free)); - assertFalse(ServiceUnitState.isValidTransition(Released, Assigned)); - assertTrue(ServiceUnitState.isValidTransition(Released, Owned)); - assertFalse(ServiceUnitState.isValidTransition(Released, Released)); - assertFalse(ServiceUnitState.isValidTransition(Released, Splitting)); + assertFalse(ServiceUnitState.isValidTransition(Releasing, Init)); + assertTrue(ServiceUnitState.isValidTransition(Releasing, Free)); + assertFalse(ServiceUnitState.isValidTransition(Releasing, Assigning)); + assertTrue(ServiceUnitState.isValidTransition(Releasing, Owned)); + assertFalse(ServiceUnitState.isValidTransition(Releasing, Releasing)); + assertFalse(ServiceUnitState.isValidTransition(Releasing, Splitting)); + assertFalse(ServiceUnitState.isValidTransition(Releasing, Deleted)); - assertTrue(ServiceUnitState.isValidTransition(Splitting, Free)); - assertFalse(ServiceUnitState.isValidTransition(Splitting, Assigned)); + assertFalse(ServiceUnitState.isValidTransition(Splitting, Init)); + assertFalse(ServiceUnitState.isValidTransition(Splitting, Free)); + assertFalse(ServiceUnitState.isValidTransition(Splitting, Assigning)); assertFalse(ServiceUnitState.isValidTransition(Splitting, Owned)); - assertFalse(ServiceUnitState.isValidTransition(Splitting, Released)); + assertFalse(ServiceUnitState.isValidTransition(Splitting, Releasing)); assertFalse(ServiceUnitState.isValidTransition(Splitting, Splitting)); + assertTrue(ServiceUnitState.isValidTransition(Splitting, Deleted)); + + assertTrue(ServiceUnitState.isValidTransition(Deleted, Init)); + assertFalse(ServiceUnitState.isValidTransition(Deleted, Free)); + assertFalse(ServiceUnitState.isValidTransition(Deleted, Assigning)); + assertFalse(ServiceUnitState.isValidTransition(Deleted, Owned)); + assertFalse(ServiceUnitState.isValidTransition(Deleted, Releasing)); + assertFalse(ServiceUnitState.isValidTransition(Deleted, Splitting)); + assertFalse(ServiceUnitState.isValidTransition(Deleted, Deleted)); } } \ No newline at end of file diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java index 41eaa640d28db..4c1d4f7d2a89d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java @@ -18,11 +18,14 @@ */ package org.apache.pulsar.compaction; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Free; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Deleted; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Init; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Owned; -import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Assigned; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Assigning; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Releasing; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Splitting; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.isValidTransition; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData.state; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; @@ -42,7 +45,9 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import org.apache.bookkeeper.client.BookKeeper; import org.apache.commons.lang.reflect.FieldUtils; @@ -81,55 +86,36 @@ public class ServiceUnitStateCompactionTest extends MockedPulsarServiceBaseTest private Schema schema; private ServiceUnitStateCompactionStrategy strategy; - private ServiceUnitState testState0 = Free; - private ServiceUnitState testState1 = Free; - private ServiceUnitState testState2 = Free; - private ServiceUnitState testState3 = Free; - private ServiceUnitState testState4 = Free; + private ServiceUnitState testState = Init; private static Random RANDOM = new Random(); private ServiceUnitStateData testValue(ServiceUnitState state, String broker) { - if (state == Free) { + if (state == Init) { return null; } return new ServiceUnitStateData(state, broker); } - private ServiceUnitStateData testValue0(String broker) { - ServiceUnitState to = nextValidState(testState0); - testState0 = to; + private ServiceUnitStateData testValue(String broker) { + ServiceUnitState to = nextValidStateNonSplit(testState); + testState = to; return testValue(to, broker); } - private ServiceUnitStateData testValue1(String broker) { - ServiceUnitState to = nextValidState(testState1); - testState1 = to; - return testValue(to, broker); - } - - private ServiceUnitStateData testValue2(String broker) { - ServiceUnitState to = nextValidState(testState2); - testState2 = to; - return testValue(to, broker); - } - - private ServiceUnitStateData testValue3(String broker) { - ServiceUnitState to = nextValidState(testState3); - testState3 = to; - return testValue(to, broker); - } - - private ServiceUnitStateData testValue4(String broker) { - ServiceUnitState to = nextValidState(testState4); - testState4 = to; - return testValue(to, broker); + private ServiceUnitState nextValidState(ServiceUnitState from) { + List candidates = Arrays.stream(ServiceUnitState.values()) + .filter(to -> isValidTransition(from, to)) + .collect(Collectors.toList()); + var state= candidates.get(RANDOM.nextInt(candidates.size())); + return state; } - private ServiceUnitState nextValidState(ServiceUnitState from) { + private ServiceUnitState nextValidStateNonSplit(ServiceUnitState from) { List candidates = Arrays.stream(ServiceUnitState.values()) - .filter(to -> to != Free && to != Splitting && isValidTransition(from, to)) + .filter(to -> to != Init && to != Splitting && to != Deleted + && isValidTransition(from, to)) .collect(Collectors.toList()); var state= candidates.get(RANDOM.nextInt(candidates.size())); return state; @@ -140,23 +126,11 @@ private ServiceUnitState nextInvalidState(ServiceUnitState from) { .filter(to -> !isValidTransition(from, to)) .collect(Collectors.toList()); if (candidates.size() == 0) { - return null; + return Init; } return candidates.get(RANDOM.nextInt(candidates.size())); } - private List nextStatesToNull(ServiceUnitState from) { - if (from == null) { - return List.of(); - } - return switch (from) { - case Assigned -> List.of(Owned); - case Owned -> List.of(); - case Splitting -> List.of(); - default -> List.of(); - }; - } - @BeforeMethod @Override public void setup() throws Exception { @@ -174,6 +148,7 @@ public void setup() throws Exception { strategy = new ServiceUnitStateCompactionStrategy(); strategy.checkBrokers(false); + testState = Init; } @@ -222,10 +197,21 @@ TestData generateTestData() throws PulsarAdminException, PulsarClientException { int keyIndex = r.nextInt(maxKeys); String key = "key" + keyIndex; ServiceUnitStateData prev = expected.get(key); - ServiceUnitState prevState = prev == null ? Free : prev.state(); - ServiceUnitState state = r.nextBoolean() ? nextInvalidState(prevState) : + ServiceUnitState prevState = state(prev); + boolean invalid = r.nextBoolean(); + ServiceUnitState state = invalid ? nextInvalidState(prevState) : nextValidState(prevState); - ServiceUnitStateData value = new ServiceUnitStateData(state, key + ":" + j); + ServiceUnitStateData value; + if (invalid) { + value = new ServiceUnitStateData(state, key + ":" + j, false); + } else { + if (state == Init) { + value = new ServiceUnitStateData(state, key + ":" + j, true); + } else { + value = new ServiceUnitStateData(state, key + ":" + j, false); + } + } + producer.newMessage().key(key).value(value).send(); if (!strategy.shouldKeepLeft(prev, value)) { expected.put(key, value); @@ -387,24 +373,26 @@ public void testReadCompactedBeforeCompaction() throws Exception { .create(); pulsarClient.newConsumer(schema).topic(topic).subscriptionName("sub1").readCompacted(true).subscribe().close(); - - producer.newMessage().key("key0").value(testValue0( "content0")).send(); - producer.newMessage().key("key0").value(testValue0("content1")).send(); - producer.newMessage().key("key0").value(testValue0( "content2")).send(); + String key = "key0"; + var testValues = Arrays.asList( + testValue("content0"), testValue("content1"), testValue("content2")); + for (var val : testValues) { + producer.newMessage().key(key).value(val).send(); + } try (Consumer consumer = pulsarClient.newConsumer(schema).topic(topic).subscriptionName("sub1") .readCompacted(true).subscribe()) { Message m = consumer.receive(); - Assert.assertEquals(m.getKey(), "key0"); - Assert.assertEquals(m.getValue().broker(), "content0"); + Assert.assertEquals(m.getKey(), key); + Assert.assertEquals(m.getValue(), testValues.get(0)); m = consumer.receive(); - Assert.assertEquals(m.getKey(), "key0"); - Assert.assertEquals(m.getValue().broker(), "content1"); + Assert.assertEquals(m.getKey(), key); + Assert.assertEquals(m.getValue(), testValues.get(1)); m = consumer.receive(); - Assert.assertEquals(m.getKey(), "key0"); - Assert.assertEquals(m.getValue().broker(), "content2"); + Assert.assertEquals(m.getKey(), key); + Assert.assertEquals(m.getValue(), testValues.get(2)); } StrategicTwoPhaseCompactor compactor @@ -414,8 +402,8 @@ public void testReadCompactedBeforeCompaction() throws Exception { try (Consumer consumer = pulsarClient.newConsumer(schema).topic(topic).subscriptionName("sub1") .readCompacted(true).subscribe()) { Message m = consumer.receive(); - Assert.assertEquals(m.getKey(), "key0"); - Assert.assertEquals(m.getValue().broker(), "content2"); + Assert.assertEquals(m.getKey(), key); + Assert.assertEquals(m.getValue(), testValues.get(2)); } } @@ -430,30 +418,37 @@ public void testReadEntriesAfterCompaction() throws Exception { pulsarClient.newConsumer(schema).topic(topic).subscriptionName("sub1").readCompacted(true).subscribe().close(); - producer.newMessage().key("key0").value(testValue0( "content0")).send(); - producer.newMessage().key("key0").value(testValue0("content1")).send(); - producer.newMessage().key("key0").value(testValue0( "content2")).send(); + String key = "key0"; + var testValues = Arrays.asList( + testValue( "content0"), + testValue("content1"), + testValue( "content2"), + testValue("content3")); + producer.newMessage().key(key).value(testValues.get(0)).send(); + producer.newMessage().key(key).value(testValues.get(1)).send(); + producer.newMessage().key(key).value(testValues.get(2)).send(); StrategicTwoPhaseCompactor compactor = new StrategicTwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); compactor.compact(topic, strategy).get(); - producer.newMessage().key("key0").value(testValue0("content3")).send(); + producer.newMessage().key(key).value(testValues.get(3)).send(); try (Consumer consumer = pulsarClient.newConsumer(schema).topic(topic).subscriptionName("sub1") .readCompacted(true).subscribe()) { Message m = consumer.receive(); - Assert.assertEquals(m.getKey(), "key0"); - Assert.assertEquals(m.getValue().broker(), "content2"); + Assert.assertEquals(m.getKey(), key); + Assert.assertEquals(m.getValue(), testValues.get(2)); m = consumer.receive(); - Assert.assertEquals(m.getKey(), "key0"); - Assert.assertEquals(m.getValue().broker(), "content3"); + Assert.assertEquals(m.getKey(), key); + Assert.assertEquals(m.getValue(), testValues.get(3)); } } @Test public void testSeekEarliestAfterCompaction() throws Exception { + String topic = "persistent://my-property/use/my-ns/my-topic1"; Producer producer = pulsarClient.newProducer(schema) @@ -461,9 +456,14 @@ public void testSeekEarliestAfterCompaction() throws Exception { .enableBatching(true) .create(); - producer.newMessage().key("key0").value(testValue0( "content0")).send(); - producer.newMessage().key("key0").value(testValue0("content1")).send(); - producer.newMessage().key("key0").value(testValue0( "content2")).send(); + String key = "key0"; + var testValues = Arrays.asList( + testValue("content0"), + testValue("content1"), + testValue("content2")); + for (var val : testValues) { + producer.newMessage().key(key).value(val).send(); + } StrategicTwoPhaseCompactor compactor = new StrategicTwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); @@ -473,8 +473,8 @@ public void testSeekEarliestAfterCompaction() throws Exception { .readCompacted(true).subscribe()) { consumer.seek(MessageId.earliest); Message m = consumer.receive(); - Assert.assertEquals(m.getKey(), "key0"); - Assert.assertEquals(m.getValue().broker(), "content2"); + Assert.assertEquals(m.getKey(), key); + Assert.assertEquals(m.getValue(), testValues.get(2)); } try (Consumer consumer = pulsarClient.newConsumer(schema).topic(topic).subscriptionName("sub1") @@ -482,34 +482,153 @@ public void testSeekEarliestAfterCompaction() throws Exception { consumer.seek(MessageId.earliest); Message m = consumer.receive(); - Assert.assertEquals(m.getKey(), "key0"); - Assert.assertEquals(m.getValue().broker(), "content0"); + Assert.assertEquals(m.getKey(), key); + Assert.assertEquals(m.getValue(), testValues.get(0)); m = consumer.receive(); - Assert.assertEquals(m.getKey(), "key0"); - Assert.assertEquals(m.getValue().broker(), "content1"); + Assert.assertEquals(m.getKey(), key); + Assert.assertEquals(m.getValue(), testValues.get(1)); m = consumer.receive(); - Assert.assertEquals(m.getKey(), "key0"); - Assert.assertEquals(m.getValue().broker(), "content2"); + Assert.assertEquals(m.getKey(), key); + Assert.assertEquals(m.getValue(), testValues.get(2)); } } @Test - public void testBrokerRestartAfterCompaction() throws Exception { + public void testSlowTableviewAfterCompaction() throws Exception { String topic = "persistent://my-property/use/my-ns/my-topic1"; + String strategyClassName = "topicCompactionStrategyClassName"; + strategy.checkBrokers(true); + + pulsarClient.newConsumer(schema) + .topic(topic) + .subscriptionName("sub1") + .readCompacted(true) + .subscribe().close(); + + var fastTV = pulsar.getClient().newTableViewBuilder(schema) + .topic(topic) + .subscriptionName("fastTV") + .loadConf(Map.of( + strategyClassName, + ServiceUnitStateCompactionStrategy.class.getName())) + .create(); + + var defaultConf = getDefaultConf(); + var additionalPulsarTestContext = createAdditionalPulsarTestContext(defaultConf); + var pulsar2 = additionalPulsarTestContext.getPulsarService(); + + var slowTV = pulsar2.getClient().newTableViewBuilder(schema) + .topic(topic) + .subscriptionName("slowTV") + .loadConf(Map.of( + strategyClassName, + ServiceUnitStateCompactionStrategy.class.getName())) + .create(); + + var semaphore = new Semaphore(0); + AtomicBoolean handledReleased = new AtomicBoolean(false); + + slowTV.listen((k, v) -> { + if (v.state() == Assigning) { + try { + // Stuck at handling Assigned + handledReleased.set(false); + semaphore.acquire(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } else if (v.state() == Releasing) { + handledReleased.set(true); + } + }); + + // Configure retention to ensue data is retained for reader + admin.namespaces().setRetention("my-property/use/my-ns", + new RetentionPolicies(-1, -1)); Producer producer = pulsarClient.newProducer(schema) .topic(topic) .enableBatching(true) + .messageRoutingMode(MessageRoutingMode.SinglePartition) .create(); - pulsarClient.newConsumer(schema).topic(topic).subscriptionName("sub1").readCompacted(true).subscribe().close(); + StrategicTwoPhaseCompactor compactor + = new StrategicTwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); + + String bundle = "bundle1"; + String src = "broker0"; + String dst = "broker1"; + producer.newMessage().key(bundle).value(new ServiceUnitStateData(Owned, src)).send(); + for (int i = 0; i < 3; i++) { + var assignedStateData = new ServiceUnitStateData(Assigning, dst, src); + producer.newMessage().key(bundle).value(assignedStateData).send(); + producer.newMessage().key(bundle).value(assignedStateData).send(); + var releasedStateData = new ServiceUnitStateData(Releasing, dst, src); + producer.newMessage().key(bundle).value(releasedStateData).send(); + producer.newMessage().key(bundle).value(releasedStateData).send(); + var ownedStateData = new ServiceUnitStateData(Owned, dst, src); + producer.newMessage().key(bundle).value(ownedStateData).send(); + producer.newMessage().key(bundle).value(ownedStateData).send(); + compactor.compact(topic, strategy).get(); + + Awaitility.await() + .pollInterval(200, TimeUnit.MILLISECONDS) + .atMost(10, TimeUnit.SECONDS) + .untilAsserted(() -> assertEquals(fastTV.get(bundle), ownedStateData)); + + Awaitility.await() + .pollInterval(200, TimeUnit.MILLISECONDS) + .atMost(10, TimeUnit.SECONDS) + .untilAsserted(() -> assertEquals(slowTV.get(bundle), assignedStateData)); + assertTrue(!handledReleased.get()); + semaphore.release(); + + Awaitility.await() + .pollInterval(200, TimeUnit.MILLISECONDS) + .atMost(10, TimeUnit.SECONDS) + .untilAsserted(() -> assertEquals(slowTV.get(bundle), ownedStateData)); + + var newTv = pulsar.getClient().newTableView(schema) + .topic(topic) + .loadConf(Map.of( + strategyClassName, + ServiceUnitStateCompactionStrategy.class.getName())) + .create(); + Awaitility.await() + .pollInterval(200, TimeUnit.MILLISECONDS) + .atMost(10, TimeUnit.SECONDS) + .untilAsserted(() -> assertEquals(newTv.get(bundle), ownedStateData)); + + src = dst; + dst = "broker" + (i + 2); + newTv.close(); + } + + producer.close(); + slowTV.close(); + fastTV.close(); + pulsar2.close(); + + } - producer.newMessage().key("key0").value(testValue0( "content0")).send(); - producer.newMessage().key("key0").value(testValue0("content1")).send(); - producer.newMessage().key("key0").value(testValue0( "content2")).send(); + @Test + public void testBrokerRestartAfterCompaction() throws Exception { + String topic = "persistent://my-property/use/my-ns/my-topic1"; + + Producer producer = pulsarClient.newProducer(schema) + .topic(topic) + .enableBatching(true) + .create(); + String key = "key0"; + pulsarClient.newConsumer(schema).topic(topic).subscriptionName("sub1").readCompacted(true).subscribe().close(); + var testValues = Arrays.asList( + testValue("content0"), testValue("content1"), testValue("content2")); + for (var val : testValues) { + producer.newMessage().key(key).value(val).send(); + } StrategicTwoPhaseCompactor compactor = new StrategicTwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); compactor.compact(topic, strategy).get(); @@ -517,8 +636,8 @@ public void testBrokerRestartAfterCompaction() throws Exception { try (Consumer consumer = pulsarClient.newConsumer(schema).topic(topic).subscriptionName("sub1") .readCompacted(true).subscribe()) { Message m = consumer.receive(); - Assert.assertEquals(m.getKey(), "key0"); - Assert.assertEquals(m.getValue().broker(), "content2"); + Assert.assertEquals(m.getKey(), key); + Assert.assertEquals(m.getValue(), testValues.get(testValues.size() - 1)); } stopBroker(); @@ -534,8 +653,8 @@ public void testBrokerRestartAfterCompaction() throws Exception { try (Consumer consumer = pulsarClient.newConsumer(schema).topic(topic).subscriptionName("sub1") .readCompacted(true).subscribe()) { Message m = consumer.receive(); - Assert.assertEquals(m.getKey(), "key0"); - Assert.assertEquals(m.getValue().broker(), "content2"); + Assert.assertEquals(m.getKey(), key); + Assert.assertEquals(m.getValue(), testValues.get(testValues.size() - 1)); } } @@ -554,13 +673,14 @@ public void testCompactEmptyTopic() throws Exception { = new StrategicTwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); compactor.compact(topic, strategy).get(); - producer.newMessage().key("key0").value(testValue0( "content0")).send(); + var testValue = testValue( "content0"); + producer.newMessage().key("key0").value(testValue).send(); try (Consumer consumer = pulsarClient.newConsumer(schema).topic(topic).subscriptionName("sub1") .readCompacted(true).subscribe()) { Message m = consumer.receive(); Assert.assertEquals(m.getKey(), "key0"); - Assert.assertEquals(m.getValue().broker(), "content0"); + Assert.assertEquals(m.getValue(), testValue); } } @@ -583,10 +703,10 @@ public void testWholeBatchCompactedOut() throws Exception { .batchingMaxPublishDelay(1, TimeUnit.HOURS) .messageRoutingMode(MessageRoutingMode.SinglePartition) .create()) { - producerBatch.newMessage().key("key1").value(testValue1("my-message-1")).sendAsync(); - producerBatch.newMessage().key("key1").value(testValue1( "my-message-2")).sendAsync(); - producerBatch.newMessage().key("key1").value(testValue1("my-message-3")).sendAsync(); - producerNormal.newMessage().key("key1").value(testValue1( "my-message-4")).send(); + producerBatch.newMessage().key("key1").value(testValue("my-message-1")).sendAsync(); + producerBatch.newMessage().key("key1").value(testValue( "my-message-2")).sendAsync(); + producerBatch.newMessage().key("key1").value(testValue("my-message-3")).sendAsync(); + producerNormal.newMessage().key("key1").value(testValue( "my-message-4")).send(); } // compact the topic @@ -610,9 +730,9 @@ public void testCompactionWithLastDeletedKey() throws Exception { pulsarClient.newConsumer(schema).topic(topic).subscriptionName("sub1").readCompacted(true).subscribe().close(); - producer.newMessage().key("1").value(testValue(Owned, "1")).send(); - producer.newMessage().key("2").value(testValue(Owned, "3")).send(); - producer.newMessage().key("3").value(testValue(Owned, "5")).send(); + producer.newMessage().key("1").value(testValue("1")).send(); + producer.newMessage().key("2").value(testValue("3")).send(); + producer.newMessage().key("3").value(testValue( "5")).send(); producer.newMessage().key("1").value(null).send(); producer.newMessage().key("2").value(null).send(); @@ -707,7 +827,7 @@ public void testCompactMultipleTimesWithoutEmptyMessage() List> futures = new ArrayList<>(messages); for (int i = 0; i < messages; i++) { - futures.add(producer.newMessage().key(key).value(testValue0((i + ""))).sendAsync()); + futures.add(producer.newMessage().key(key).value(testValue((i + ""))).sendAsync()); } FutureUtil.waitForAll(futures).get(); @@ -720,7 +840,7 @@ public void testCompactMultipleTimesWithoutEmptyMessage() // 3. Send more ten messages futures.clear(); for (int i = 0; i < messages; i++) { - futures.add(producer.newMessage().key(key).value(testValue0((i + 10 + ""))).sendAsync()); + futures.add(producer.newMessage().key(key).value(testValue((i + 10 + ""))).sendAsync()); } FutureUtil.waitForAll(futures).get(); @@ -754,7 +874,7 @@ public void testReadUnCompacted() List> futures = new ArrayList<>(messages); for (int i = 0; i < messages; i++) { - futures.add(producer.newMessage().key(key).value(testValue0((i + ""))).sendAsync()); + futures.add(producer.newMessage().key(key).value(testValue((i + ""))).sendAsync()); } FutureUtil.waitForAll(futures).get(); @@ -767,7 +887,7 @@ public void testReadUnCompacted() // 3. Send more ten messages futures.clear(); for (int i = 0; i < messages; i++) { - futures.add(producer.newMessage().key(key).value(testValue0((i + 10 + ""))).sendAsync()); + futures.add(producer.newMessage().key(key).value(testValue((i + 10 + ""))).sendAsync()); } FutureUtil.waitForAll(futures).get(); try (Consumer consumer = pulsarClient.newConsumer(schema) @@ -788,9 +908,6 @@ public void testReadUnCompacted() } // 4.Send empty message to delete the key-value in the compacted topic. - for (ServiceUnitState state : nextStatesToNull(testState0)) { - producer.newMessage().key(key).value(new ServiceUnitStateData(state, "xx")).send(); - } producer.newMessage().key(key).value(null).send(); // 5.compact the topic. @@ -807,7 +924,7 @@ public void testReadUnCompacted() } for (int i = 0; i < messages; i++) { - futures.add(producer.newMessage().key(key).value(testValue0((i + 20 + ""))).sendAsync()); + futures.add(producer.newMessage().key(key).value(testValue((i + 20 + ""))).sendAsync()); } FutureUtil.waitForAll(futures).get(); From 870334029119dce41f2a6e986db5ca46d023220f Mon Sep 17 00:00:00 2001 From: Asaf Mesika Date: Fri, 24 Feb 2023 02:14:42 +0200 Subject: [PATCH 050/174] [improve][doc] Changing subject prefix for PIPs in the mailing list (#19617) --- wiki/proposals/PIP.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wiki/proposals/PIP.md b/wiki/proposals/PIP.md index afd0eac5fdfd8..d972956df0276 100644 --- a/wiki/proposals/PIP.md +++ b/wiki/proposals/PIP.md @@ -80,7 +80,7 @@ The process works in the following way: 1. The author(s) of the proposal will create a GitHub issue ticket choosing the template for PIP proposals. 2. The author(s) will send a note to the dev@pulsar.apache.org mailing list - to start the discussion, using subject prefix `[PIP] xxx`. The discussion + to start the discussion, using subject prefix `[DISCUSS] PIP-xxx: `. The discussion need to happen in the mailing list. Please avoid discussing it using GitHub comments in the PIP GitHub issue, as it creates two tracks of feedback. @@ -147,4 +147,4 @@ If there are alternatives that were already considered by the authors or, after the discussion, by the community, and were rejected, please list them here along with the reason why they were rejected. -``` \ No newline at end of file +``` From 3b075a60f04938fa1a90acc2fd856168ca0cadef Mon Sep 17 00:00:00 2001 From: Asaf Mesika Date: Fri, 24 Feb 2023 02:15:15 +0200 Subject: [PATCH 051/174] [improve][doc] Clarify where to grab the number for the PIP (#19610) --- wiki/proposals/PIP.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/wiki/proposals/PIP.md b/wiki/proposals/PIP.md index d972956df0276..f76c9f0f7a235 100644 --- a/wiki/proposals/PIP.md +++ b/wiki/proposals/PIP.md @@ -78,7 +78,9 @@ A PIP proposal can be in these states: The process works in the following way: 1. The author(s) of the proposal will create a GitHub issue ticket choosing the - template for PIP proposals. + template for PIP proposals. The issue title should be "PIP-xxx: title", where + the "xxx" number should be chosen to be the next number from the existing PIP + issues, listed [here]([url](https://github.com/apache/pulsar/labels/PIP)). 2. The author(s) will send a note to the dev@pulsar.apache.org mailing list to start the discussion, using subject prefix `[DISCUSS] PIP-xxx: `. The discussion need to happen in the mailing list. Please avoid discussing it using @@ -89,12 +91,12 @@ The process works in the following way: 4. Once some consensus is reached, there will be a vote to formally approve the proposal. The vote will be held on the dev@pulsar.apache.org mailing list. Everyone - is welcome to vote on the proposal, though it will considered to be binding + is welcome to vote on the proposal, though it will be considered to be binding only the vote of PMC members. I would be required to have a lazy majority of at least 3 binding +1s votes. The vote should stay open for at least 48 hours. 5. When the vote is closed, if the outcome is positive, the state of the - proposal is updated and the Pull Requests associated with this proposal can + proposal is updated, and the Pull Requests associated with this proposal can start to get merged into the master branch. All the Pull Requests that are created, should always reference the From bf982f4995e624659021191982f7fedc13fc3ba0 Mon Sep 17 00:00:00 2001 From: Neng Lu Date: Thu, 23 Feb 2023 18:44:09 -0800 Subject: [PATCH 052/174] [improve] configure whether function consumer should skip to latest (#17214) --- .../org/apache/pulsar/common/functions/FunctionConfig.java | 2 ++ .../java/org/apache/pulsar/admin/cli/CmdFunctions.java | 7 +++++++ .../pulsar/functions/instance/JavaInstanceRunnable.java | 5 +++++ .../pulsar/functions/source/MultiConsumerPulsarSource.java | 6 ++++++ .../apache/pulsar/functions/source/PulsarSourceConfig.java | 3 ++- .../functions/source/SingleConsumerPulsarSource.java | 6 ++++++ pulsar-functions/proto/src/main/proto/Function.proto | 1 + .../apache/pulsar/functions/utils/FunctionConfigUtils.java | 6 ++++++ 8 files changed, 35 insertions(+), 1 deletion(-) diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/FunctionConfig.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/FunctionConfig.java index 0b26e7e93b5f0..e304f25d5d373 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/FunctionConfig.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/FunctionConfig.java @@ -131,6 +131,8 @@ public enum Runtime { private Integer maxPendingAsyncRequests; // Whether the pulsar admin client exposed to function context, default is disabled. private Boolean exposePulsarAdminClientEnabled; + // Whether the consumer should skip to latest position in case of failure recovery + private Boolean skipToLatest; @Builder.Default private SubscriptionInitialPosition subscriptionPosition = SubscriptionInitialPosition.Latest; diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java index bc2585bc67bc9..05bab9c6f198b 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java @@ -325,6 +325,9 @@ abstract class FunctionDetailsCommand extends BaseCommand { @Parameter(names = "--subs-position", description = "Pulsar source subscription position if user wants to " + "consume messages from the specified location #Java") protected SubscriptionInitialPosition subsPosition; + @Parameter(names = "--skip-to-latest", description = "Whether or not the consumer skip to latest message " + + "upon function instance restart", arity = 1) + protected Boolean skipToLatest; @Parameter(names = "--parallelism", description = "The parallelism factor of a Pulsar Function " + "(i.e. the number of function instances to run) #Java") protected Integer parallelism; @@ -548,6 +551,10 @@ void processArguments() throws Exception { functionConfig.setSubscriptionPosition(subsPosition); } + if (null != skipToLatest) { + functionConfig.setSkipToLatest(skipToLatest); + } + if (null != userConfigString) { Type type = new TypeToken>() {}.getType(); Map userConfigMap = new Gson().fromJson(userConfigString, type); diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java index 5f82e93470089..0dbfa0945caa7 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java @@ -789,7 +789,12 @@ private void setupInput(ContextImpl contextImpl) throws Exception { convertFromFunctionDetailsSubscriptionPosition(sourceSpec.getSubscriptionPosition()) ); + pulsarSourceConfig.setSkipToLatest( + sourceSpec.getSkipToLatest() + ); + Objects.requireNonNull(contextImpl.getSubscriptionType()); + pulsarSourceConfig.setSubscriptionType(contextImpl.getSubscriptionType()); pulsarSourceConfig.setTypeClassName(sourceSpec.getTypeClassName()); diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/MultiConsumerPulsarSource.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/MultiConsumerPulsarSource.java index 61f5bfacb35a7..533e8d42c11ac 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/MultiConsumerPulsarSource.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/MultiConsumerPulsarSource.java @@ -27,6 +27,7 @@ import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.ConsumerBuilder; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.MessageListener; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; @@ -66,6 +67,11 @@ public void open(Map config, SourceContext sourceContext) throws cb.messageListener(this); Consumer consumer = cb.subscribeAsync().join(); + + if (pulsarSourceConfig.getSkipToLatest() != null && pulsarSourceConfig.getSkipToLatest()) { + consumer.seek(MessageId.latest); + } + inputConsumers.add(consumer); } } diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/PulsarSourceConfig.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/PulsarSourceConfig.java index 5315702d7c0e4..ad6ea5a877f30 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/PulsarSourceConfig.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/PulsarSourceConfig.java @@ -30,7 +30,8 @@ public abstract class PulsarSourceConfig { SubscriptionType subscriptionType; private String subscriptionName; private SubscriptionInitialPosition subscriptionPosition; - // Whether the subscriptions the functions created/used should be deleted when the functions is deleted + // Whether call consumer.seek(latest) to skip contents between last ask message and the latest message + private Boolean skipToLatest; private Integer maxMessageRetries = -1; private String deadLetterTopic; diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/SingleConsumerPulsarSource.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/SingleConsumerPulsarSource.java index 3e60111ddbe4b..426723804cad1 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/SingleConsumerPulsarSource.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/SingleConsumerPulsarSource.java @@ -27,6 +27,7 @@ import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.ConsumerBuilder; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.common.util.Reflections; import org.apache.pulsar.functions.api.Record; @@ -73,6 +74,11 @@ public void open(Map config, SourceContext sourceContext) throws ConsumerBuilder cb = createConsumeBuilder(topic, pulsarSourceConsumerConfig); consumer = cb.subscribeAsync().join(); + + if (this.pulsarSourceConfig.getSkipToLatest() != null && this.pulsarSourceConfig.getSkipToLatest()) { + consumer.seek(MessageId.latest); + } + inputConsumers.add(consumer); } diff --git a/pulsar-functions/proto/src/main/proto/Function.proto b/pulsar-functions/proto/src/main/proto/Function.proto index e67899abd223e..101d45bc59cd7 100644 --- a/pulsar-functions/proto/src/main/proto/Function.proto +++ b/pulsar-functions/proto/src/main/proto/Function.proto @@ -165,6 +165,7 @@ message SourceSpec { bool cleanupSubscription = 11; SubscriptionPosition subscriptionPosition = 12; uint64 negativeAckRedeliveryDelayMs = 13; + bool skipToLatest = 14; } message SinkSpec { diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java index 647210de33acf..d02fe5f788b5a 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java @@ -198,6 +198,12 @@ public static FunctionDetails convert(FunctionConfig functionConfig, ExtractedFu sourceSpecBuilder.setSubscriptionPosition(subPosition); } + if (functionConfig.getSkipToLatest() != null) { + sourceSpecBuilder.setSkipToLatest(functionConfig.getSkipToLatest()); + } else { + sourceSpecBuilder.setSkipToLatest(false); + } + if (extractedDetails.getTypeArg0() != null) { sourceSpecBuilder.setTypeClassName(extractedDetails.getTypeArg0()); } else if (StringUtils.isNotEmpty(functionConfig.getInputTypeClassName())) { From 8cc979de411f9bca05ea95b16807f1860b79382e Mon Sep 17 00:00:00 2001 From: jiangpengcheng Date: Fri, 24 Feb 2023 11:03:37 +0800 Subject: [PATCH 053/174] [improve][fn] Support e2e cryption in python instance (#18738) --- .../src/main/python/python_instance.py | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/pulsar-functions/instance/src/main/python/python_instance.py b/pulsar-functions/instance/src/main/python/python_instance.py index f77ef38f76e4d..ac723232f4839 100755 --- a/pulsar-functions/instance/src/main/python/python_instance.py +++ b/pulsar-functions/instance/src/main/python/python_instance.py @@ -22,6 +22,7 @@ """python_instance.py: Python Instance for running python functions """ import base64 +import json import os import signal import time @@ -188,13 +189,16 @@ def run(self): consumer_conf.schemaProperties) Log.debug("Setting up consumer for topic %s with subname %s" % (topic, subscription_name)) + crypto_key_reader = self.get_crypto_reader(consumer_conf.cryptoSpec) + consumer_args = { "consumer_type": mode, "schema": self.input_schema[topic], "message_listener": partial(self.message_listener, self.input_serdes[topic], self.input_schema[topic]), "unacked_messages_timeout_ms": int(self.timeout_ms) if self.timeout_ms else None, "initial_position": position, - "properties": properties + "properties": properties, + "crypto_key_reader": crypto_key_reader } if consumer_conf.HasField("receiverQueueSize"): consumer_args["receiver_queue_size"] = consumer_conf.receiverQueueSize.value @@ -343,6 +347,10 @@ def setup_producer(self): self.output_schema = self.get_schema(self.instance_config.function_details.sink.schemaType, self.instance_config.function_details.sink.typeClassName, self.instance_config.function_details.sink.schemaProperties) + crypto_key_reader = self.get_crypto_reader(self.instance_config.function_details.sink.producerSpec.cryptoSpec) + encryption_key = None + if crypto_key_reader is not None: + encryption_key = self.instance_config.function_details.sink.producerSpec.cryptoSpec.producerEncryptionKeyName[0] self.producer = self.pulsar_client.create_producer( str(self.instance_config.function_details.sink.topic), @@ -355,6 +363,9 @@ def setup_producer(self): # set send timeout to be infinity to prevent potential deadlock with consumer # that might happen when consumer is blocked due to unacked messages send_timeout_millis=0, + # python client only supports one key for encryption + encryption_key=encryption_key, + crypto_key_reader=crypto_key_reader, properties=util.get_properties(util.getFullyQualifiedFunctionName( self.instance_config.function_details.tenant, self.instance_config.function_details.namespace, @@ -484,7 +495,6 @@ def close(self): if self.pulsar_client: self.pulsar_client.close() - # TODO: support other schemas: PROTOBUF, PROTOBUF_NATIVE, and KeyValue def get_schema(self, schema_type, type_class_name, schema_properties): schema = DEFAULT_SCHEMA @@ -526,4 +536,16 @@ def get_record_class(self, class_name): record_kclass = util.import_class(os.path.dirname(self.user_code), class_name) except: pass - return record_kclass \ No newline at end of file + return record_kclass + def get_crypto_reader(self, crypto_spec): + crypto_key_reader = None + if crypto_spec is not None: + try: + crypto_config = json.loads(crypto_spec.cryptoKeyReaderConfig) + if crypto_spec.cryptoKeyReaderClassName == "" or crypto_spec.cryptoKeyReaderClassName is None: + crypto_key_reader = pulsar.CryptoKeyReader(**crypto_config) + else: + crypto_key_reader = util.import_class(os.path.dirname(self.user_code), crypto_spec.cryptoKeyReaderClassName)(**crypto_config) + except Exception as e: + Log.error("Failed to load the crypto key reader from spec: %s, error: %s" % (crypto_spec, e)) + return crypto_key_reader From a26240bd0ce2cc1f77a7612e05cc142785766717 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Fri, 24 Feb 2023 15:07:53 +0800 Subject: [PATCH 054/174] [fix][proxy] Fix JKS TLS transport (#19485) Signed-off-by: Zixuan Liu --- .../NettySSLContextAutoRefreshBuilder.java | 18 ++- .../proxy/server/DirectProxyHandler.java | 6 +- .../server/ProxyKeyStoreTlsTransportTest.java | 131 ++++++++++++++++++ 3 files changed, 145 insertions(+), 10 deletions(-) create mode 100644 pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyKeyStoreTlsTransportTest.java diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/NettySSLContextAutoRefreshBuilder.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/NettySSLContextAutoRefreshBuilder.java index 7b47eb9b8bb64..6d0cfb108bd0e 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/NettySSLContextAutoRefreshBuilder.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/NettySSLContextAutoRefreshBuilder.java @@ -47,7 +47,6 @@ public class NettySSLContextAutoRefreshBuilder extends SslContextAutoRefreshBuil protected String tlsKeyStorePassword; protected FileModifiedTimeUpdater tlsKeyStore; - protected AuthenticationDataProvider authData; protected final boolean isServer; // for server @@ -101,8 +100,14 @@ public NettySSLContextAutoRefreshBuilder(String sslProviderString, this.tlsAllowInsecureConnection = allowInsecureConnection; this.tlsProvider = sslProviderString; - this.authData = authData; - + if (authData != null) { + KeyStoreParams authParams = authData.getTlsKeyStoreParams(); + if (authParams != null) { + keyStoreTypeString = authParams.getKeyStoreType(); + keyStore = authParams.getKeyStorePath(); + keyStorePassword = authParams.getKeyStorePassword(); + } + } this.tlsKeyStoreType = keyStoreTypeString; this.tlsKeyStore = new FileModifiedTimeUpdater(keyStore); this.tlsKeyStorePassword = keyStorePassword; @@ -126,11 +131,10 @@ public synchronized KeyStoreSSLContext update() throws GeneralSecurityException, tlsTrustStoreType, tlsTrustStore.getFileName(), tlsTrustStorePassword, tlsRequireTrustedClientCertOnConnect, tlsCiphers, tlsProtocols); } else { - KeyStoreParams authParams = authData.getTlsKeyStoreParams(); this.keyStoreSSLContext = KeyStoreSSLContext.createClientKeyStoreSslContext(tlsProvider, - authParams != null ? authParams.getKeyStoreType() : tlsKeyStoreType, - authParams != null ? authParams.getKeyStorePath() : tlsKeyStore.getFileName(), - authParams != null ? authParams.getKeyStorePassword() : tlsKeyStorePassword, + tlsKeyStoreType, + tlsKeyStore.getFileName(), + tlsKeyStorePassword, tlsAllowInsecureConnection, tlsTrustStoreType, tlsTrustStore.getFileName(), tlsTrustStorePassword, tlsCiphers, tlsProtocols); diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java index 1e9fd676573fd..23c7faa2d4bb7 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java @@ -127,9 +127,9 @@ public DirectProxyHandler(ProxyService service, ProxyConnection proxyConnection) config.getBrokerClientTlsTrustStoreType(), config.getBrokerClientTlsTrustStore(), config.getBrokerClientTlsTrustStorePassword(), - null, - null, - null, + config.getBrokerClientTlsKeyStoreType(), + config.getBrokerClientTlsKeyStore(), + config.getBrokerClientTlsKeyStorePassword(), config.getBrokerClientTlsCiphers(), config.getBrokerClientTlsProtocols(), config.getTlsCertRefreshCheckDurationSec(), diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyKeyStoreTlsTransportTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyKeyStoreTlsTransportTest.java new file mode 100644 index 0000000000000..5c4e40ed65a70 --- /dev/null +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyKeyStoreTlsTransportTest.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.proxy.server; + +import static org.mockito.Mockito.doReturn; +import java.util.Optional; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.authentication.AuthenticationService; +import org.apache.pulsar.client.api.ClientBuilder; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; +import org.apache.pulsar.metadata.impl.ZKMetadataStore; +import org.mockito.Mockito; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "proxy") +public class ProxyKeyStoreTlsTransportTest extends MockedPulsarServiceBaseTest { + private ProxyService proxyService; + private ProxyConfiguration proxyConfig = new ProxyConfiguration(); + + @Override + @BeforeMethod + protected void setup() throws Exception { + + // broker with JKS + conf.setWebServicePortTls(Optional.of(0)); + conf.setBrokerServicePortTls(Optional.of(0)); + conf.setTlsEnabledWithKeyStore(true); + conf.setTlsKeyStoreType(KEYSTORE_TYPE); + conf.setTlsKeyStore(BROKER_KEYSTORE_FILE_PATH); + conf.setTlsKeyStorePassword(BROKER_KEYSTORE_PW); + conf.setTlsTrustStoreType(KEYSTORE_TYPE); + conf.setTlsTrustStore(CLIENT_TRUSTSTORE_FILE_PATH); + conf.setTlsTrustStorePassword(CLIENT_TRUSTSTORE_PW); + conf.setTlsRequireTrustedClientCertOnConnect(true); + + internalSetup(); + + // proxy with JKS + proxyConfig.setServicePort(Optional.of(0)); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); + proxyConfig.setServicePortTls(Optional.of(0)); + proxyConfig.setWebServicePort(Optional.of(0)); + proxyConfig.setWebServicePortTls(Optional.of(0)); + proxyConfig.setTlsEnabledWithBroker(true); + proxyConfig.setTlsEnabledWithKeyStore(true); + + proxyConfig.setTlsKeyStoreType(KEYSTORE_TYPE); + proxyConfig.setTlsKeyStore(BROKER_KEYSTORE_FILE_PATH); + proxyConfig.setTlsKeyStorePassword(BROKER_KEYSTORE_PW); + proxyConfig.setTlsTrustStoreType(KEYSTORE_TYPE); + proxyConfig.setTlsTrustStore(CLIENT_TRUSTSTORE_FILE_PATH); + proxyConfig.setTlsTrustStorePassword(CLIENT_TRUSTSTORE_PW); + + proxyConfig.setMetadataStoreUrl(DUMMY_VALUE); + proxyConfig.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE); + + proxyConfig.setTlsRequireTrustedClientCertOnConnect(false); + + proxyConfig.setBrokerClientTlsEnabledWithKeyStore(true); + proxyConfig.setBrokerClientTlsKeyStore(CLIENT_KEYSTORE_FILE_PATH); + proxyConfig.setBrokerClientTlsKeyStorePassword(CLIENT_KEYSTORE_PW); + proxyConfig.setBrokerClientTlsTrustStore(BROKER_TRUSTSTORE_FILE_PATH); + proxyConfig.setBrokerClientTlsTrustStorePassword(BROKER_TRUSTSTORE_PW); + + proxyService = Mockito.spy(new ProxyService(proxyConfig, + new AuthenticationService( + PulsarConfigurationLoader.convertFrom(proxyConfig)))); + doReturn(new ZKMetadataStore(mockZooKeeper)).when(proxyService).createLocalMetadataStore(); + doReturn(new ZKMetadataStore(mockZooKeeperGlobal)).when(proxyService).createConfigurationMetadataStore(); + + proxyService.start(); + } + + @Override + @AfterMethod(alwaysRun = true) + protected void cleanup() throws Exception { + internalCleanup(); + + proxyService.close(); + } + + protected PulsarClient newClient() throws Exception { + ClientBuilder clientBuilder = PulsarClient.builder() + .serviceUrl(proxyService.getServiceUrlTls()) + .useKeyStoreTls(true) + .tlsTrustStorePath(BROKER_TRUSTSTORE_FILE_PATH) + .tlsTrustStorePassword(BROKER_TRUSTSTORE_PW) + .tlsKeyStorePath(CLIENT_KEYSTORE_FILE_PATH) + .tlsKeyStorePassword(CLIENT_KEYSTORE_PW) + .allowTlsInsecureConnection(false); + return clientBuilder.build(); + } + + @Test + public void testProducer() throws Exception { + @Cleanup + PulsarClient client = newClient(); + @Cleanup + Producer producer = client.newProducer(Schema.BYTES) + .topic("persistent://sample/test/local/topic" + System.currentTimeMillis()) + .create(); + + for (int i = 0; i < 10; i++) { + producer.send("test".getBytes()); + } + } +} From 6d3e483bab0f960b21cb521fb3908eccd55993b6 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Fri, 24 Feb 2023 17:09:20 +0800 Subject: [PATCH 055/174] [fix][client] Fix load the trust store file (#19483) Signed-off-by: Zixuan Liu --- .../util/keystoretls/KeyStoreSSLContext.java | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/KeyStoreSSLContext.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/KeyStoreSSLContext.java index 4ed1826cfe9b1..c717127d085db 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/KeyStoreSSLContext.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/KeyStoreSSLContext.java @@ -34,6 +34,7 @@ import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLEngine; +import javax.net.ssl.TrustManager; import javax.net.ssl.TrustManagerFactory; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -150,25 +151,30 @@ public SSLContext createSSLContext() throws GeneralSecurityException, IOExceptio } // trust store - TrustManagerFactory trustManagerFactory; + TrustManagerFactory trustManagerFactory = null; if (this.allowInsecureConnection) { trustManagerFactory = InsecureTrustManagerFactory.INSTANCE; } else { - trustManagerFactory = provider != null - ? TrustManagerFactory.getInstance(tmfAlgorithm, provider) - : TrustManagerFactory.getInstance(tmfAlgorithm); - KeyStore trustStore = KeyStore.getInstance(trustStoreTypeString); - char[] passwordChars = trustStorePassword.toCharArray(); - try (FileInputStream inputStream = new FileInputStream(trustStorePath)) { - trustStore.load(inputStream, passwordChars); + if (!Strings.isNullOrEmpty(trustStorePath)) { + trustManagerFactory = provider != null + ? TrustManagerFactory.getInstance(tmfAlgorithm, provider) + : TrustManagerFactory.getInstance(tmfAlgorithm); + KeyStore trustStore = KeyStore.getInstance(trustStoreTypeString); + char[] passwordChars = trustStorePassword.toCharArray(); + try (FileInputStream inputStream = new FileInputStream(trustStorePath)) { + trustStore.load(inputStream, passwordChars); + } + trustManagerFactory.init(trustStore); } - trustManagerFactory.init(trustStore); + } + + TrustManager[] trustManagers = null; + if (trustManagerFactory != null) { + trustManagers = SecurityUtility.processConscryptTrustManagers(trustManagerFactory.getTrustManagers()); } // init - sslContext.init(keyManagers, SecurityUtility - .processConscryptTrustManagers(trustManagerFactory.getTrustManagers()), - new SecureRandom()); + sslContext.init(keyManagers, trustManagers, new SecureRandom()); this.sslContext = sslContext; return sslContext; } From 9e9ee02fe1d0e9434b5510c555a5906c9b427bdd Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Fri, 24 Feb 2023 11:30:47 -0600 Subject: [PATCH 056/174] [improve][broker] Store nonBlank clientVersions that have spaces (#19616) Relates to: https://github.com/apache/pulsar/pull/19540 ### Motivation We currently filter out the `clientVersion` when it has a `" "` in it. As a consequence, we filter out the go client's version because of how it is made: https://github.com/apache/pulsar-client-go/blob/dedbdc45c63b06e6a12356785418cd906d6bab3c/pulsar/internal/version.go#L43 Given that we do not have any documented restrictions on the `clientVersion`, I think we should not drop clients that have a space in their name. I propose that we remove the filter logic to store all "valid" versions supplied by the client. ### Modifications * Update `ServerCnx` to store `clientVersion` when it has a space in its name. ### Verifying this change A new test is added. ### Does this pull request potentially affect one of the following parts: This is a backwards compatible change. ### Documentation - [x] `doc-not-needed` ### Matching PR in forked repository PR in forked repository: skipping for this minor change that shouldn't make any tests fail --- .../pulsar/broker/service/ServerCnx.java | 2 +- .../pulsar/broker/service/ServerCnxTest.java | 29 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 242947f6a0fd6..2d6bed3eb7ee3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -690,7 +690,7 @@ private void completeConnect(int clientProtoVersion, String clientVersion) { log.debug("[{}] connect state change to : [{}]", remoteAddress, State.Connected.name()); } setRemoteEndpointProtocolVersion(clientProtoVersion); - if (isNotBlank(clientVersion) && !clientVersion.contains(" ") /* ignore default version: pulsar client */) { + if (isNotBlank(clientVersion)) { this.clientVersion = clientVersion.intern(); } if (brokerInterceptor != null) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java index a29d6dac72023..4580f028de2b0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java @@ -143,6 +143,7 @@ import org.mockito.stubbing.Answer; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @SuppressWarnings("unchecked") @@ -293,6 +294,34 @@ public void testConnectCommandWithProtocolVersion() throws Exception { channel.finish(); } + @DataProvider(name = "clientVersions") + public Object[][] clientVersions() { + return new Object[][]{ + {"Pulsar Client", true}, + {"Pulsar Go 0.2.1", true}, + {"Pulsar-Client-Java-v1.15.2", true}, + {"pulsar-java-3.0.0", true}, + {"", false}, + {" ", false} + }; + } + + @Test(dataProvider = "clientVersions") + public void testStoreClientVersionWhenNotBlank(String clientVersion, boolean expectSetToClientVersion) throws Exception { + resetChannel(); + assertTrue(channel.isActive()); + assertEquals(serverCnx.getState(), State.Start); + + ByteBuf clientCommand = Commands.newConnect("", "", clientVersion); + channel.writeInbound(clientCommand); + + assertEquals(serverCnx.getState(), State.Connected); + assertTrue(getResponse() instanceof CommandConnected); + + assertEquals(serverCnx.getClientVersion(), expectSetToClientVersion ? clientVersion : null); + channel.finish(); + } + @Test(timeOut = 30000) public void testKeepAlive() throws Exception { resetChannel(); From de43ad0340f9fa857394449b7bfdbc4a339dcc92 Mon Sep 17 00:00:00 2001 From: WangJialing <65590138+wangjialing218@users.noreply.github.com> Date: Sat, 25 Feb 2023 01:55:52 +0800 Subject: [PATCH 057/174] [improve][broker] Print stack trace when got excpetion in ServerCnx (#19593) ### Motivation I meet a issue that NPE happend in ServerCnx.handleSend() but the debug log does not print stack trace. ![servercnx](https://user-images.githubusercontent.com/65590138/220540592-77fd107a-99c4-4eb3-ae84-6e6ceba96dfd.png) ### Modifications print stack trace when got excpetion in ServerCnx ### Verifying this change - [ ] Make sure that the change passes the CI checks. This change is a trivial rework / code cleanup without any test coverage. ### Does this pull request potentially affect one of the following parts: *If the box was checked, please highlight the changes* - [ ] Dependencies (add or upgrade a dependency) - [ ] The public API - [ ] The schema - [ ] The default values of configurations - [ ] The threading model - [ ] The binary protocol - [ ] The REST endpoints - [ ] The admin CLI options - [ ] The metrics - [ ] Anything that affects deployment ### Documentation - [ ] `doc` - [ ] `doc-required` - [x] `doc-not-needed` - [ ] `doc-complete` --- .../main/java/org/apache/pulsar/broker/service/ServerCnx.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 2d6bed3eb7ee3..b0256cddf6351 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -397,7 +397,8 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws E // At default info level, suppress all subsequent exceptions that are thrown when the connection has already // failed if (log.isDebugEnabled()) { - log.debug("[{}] Got exception: {}", remoteAddress, cause); + log.debug("[{}] Got exception {}", remoteAddress, + ClientCnx.isKnownException(cause) ? cause : ExceptionUtils.getStackTrace(cause)); } } ctx.close(); From 939d065c9daeb5dced020ad838751b817dee022c Mon Sep 17 00:00:00 2001 From: feynmanlin <315157973@qq.com> Date: Sun, 26 Feb 2023 11:56:13 +0800 Subject: [PATCH 058/174] [fix][broker] Remove useless load balancer items about MemoryResourceWeight (#19559) Motivation Even for a Broker with a very low load, its memory will grow slowly until GC is triggered. If memory is used as a load calculation item, the Bundle will be unloaded by mistake Modifications Remove memory as load calculation item --- conf/broker.conf | 9 +++++---- .../org/apache/pulsar/broker/ServiceConfiguration.java | 4 +++- .../loadbalance/impl/LeastResourceUsageWithWeight.java | 1 - .../pulsar/broker/loadbalance/impl/ThresholdShedder.java | 2 +- .../policies/data/loadbalancer/LocalBrokerData.java | 9 ++++++++- .../policies/data/loadbalancer/LocalBrokerDataTest.java | 8 +++----- 6 files changed, 20 insertions(+), 13 deletions(-) diff --git a/conf/broker.conf b/conf/broker.conf index f64d08a1de88c..3183c83a837a8 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -1355,10 +1355,6 @@ loadBalancerBandwithOutResourceWeight=1.0 # It only takes effect in the ThresholdShedder strategy. loadBalancerCPUResourceWeight=1.0 -# The heap memory usage weight when calculating new resource usage. -# It only takes effect in the ThresholdShedder strategy. -loadBalancerMemoryResourceWeight=1.0 - # The direct memory usage weight when calculating new resource usage. # It only takes effect in the ThresholdShedder strategy. loadBalancerDirectMemoryResourceWeight=1.0 @@ -1669,6 +1665,11 @@ strictBookieAffinityEnabled=false # These settings are left here for compatibility +# The heap memory usage weight when calculating new resource usage. +# It only takes effect in the ThresholdShedder strategy. +# Deprecated: Memory is no longer used as a load balancing item +loadBalancerMemoryResourceWeight=1.0 + # Zookeeper quorum connection string # Deprecated: use metadataStoreUrl instead zookeeperServers= diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 4f2c8e72e131d..5a85be11745e2 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -2341,10 +2341,12 @@ The delayed message index bucket time step(in seconds) in per bucket snapshot se ) private double loadBalancerCPUResourceWeight = 1.0; + @Deprecated(since = "3.0.0") @FieldContext( dynamic = true, category = CATEGORY_LOAD_BALANCER, - doc = "Memory Resource Usage Weight" + doc = "Memory Resource Usage Weight. Deprecated: Memory is no longer used as a load balancing item.", + deprecated = true ) private double loadBalancerMemoryResourceWeight = 1.0; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LeastResourceUsageWithWeight.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LeastResourceUsageWithWeight.java index 35dd7282101b5..f50f9ed36538a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LeastResourceUsageWithWeight.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LeastResourceUsageWithWeight.java @@ -98,7 +98,6 @@ private double updateAndGetMaxResourceUsageWithWeight(String broker, BrokerData } double resourceUsage = brokerData.getLocalData().getMaxResourceUsageWithWeight( conf.getLoadBalancerCPUResourceWeight(), - conf.getLoadBalancerMemoryResourceWeight(), conf.getLoadBalancerDirectMemoryResourceWeight(), conf.getLoadBalancerBandwithInResourceWeight(), conf.getLoadBalancerBandwithOutResourceWeight()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedder.java index c75c9c8a3178c..e2f1a7808fe91 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedder.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedder.java @@ -168,7 +168,7 @@ private double updateAvgResourceUsage(String broker, LocalBrokerData localBroker brokerAvgResourceUsage.get(broker); double resourceUsage = localBrokerData.getMaxResourceUsageWithWeight( conf.getLoadBalancerCPUResourceWeight(), - conf.getLoadBalancerMemoryResourceWeight(), conf.getLoadBalancerDirectMemoryResourceWeight(), + conf.getLoadBalancerDirectMemoryResourceWeight(), conf.getLoadBalancerBandwithInResourceWeight(), conf.getLoadBalancerBandwithOutResourceWeight()); historyUsage = historyUsage == null diff --git a/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerData.java b/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerData.java index 030ecc63be545..8c0c008e0a555 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerData.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerData.java @@ -249,7 +249,7 @@ public String printResourceUsage() { cpu.percentUsage(), memory.percentUsage(), directMemory.percentUsage(), bandwidthIn.percentUsage(), bandwidthOut.percentUsage()); } - + @Deprecated public double getMaxResourceUsageWithWeight(final double cpuWeight, final double memoryWeight, final double directMemoryWeight, final double bandwidthInWeight, final double bandwidthOutWeight) { @@ -257,6 +257,13 @@ public double getMaxResourceUsageWithWeight(final double cpuWeight, final double directMemory.percentUsage() * directMemoryWeight, bandwidthIn.percentUsage() * bandwidthInWeight, bandwidthOut.percentUsage() * bandwidthOutWeight) / 100; } + public double getMaxResourceUsageWithWeight(final double cpuWeight, + final double directMemoryWeight, final double bandwidthInWeight, + final double bandwidthOutWeight) { + return max(cpu.percentUsage() * cpuWeight, + directMemory.percentUsage() * directMemoryWeight, bandwidthIn.percentUsage() * bandwidthInWeight, + bandwidthOut.percentUsage() * bandwidthOutWeight) / 100; + } public static double max(double... args) { double max = Double.NEGATIVE_INFINITY; diff --git a/pulsar-common/src/test/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerDataTest.java b/pulsar-common/src/test/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerDataTest.java index a95f019234caa..db55ecfe5035a 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerDataTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/policies/data/loadbalancer/LocalBrokerDataTest.java @@ -43,7 +43,6 @@ public void testLocalBrokerDataDeserialization() { public void testMaxResourceUsage() { LocalBrokerData data = new LocalBrokerData(); data.setCpu(new ResourceUsage(1.0, 100.0)); - data.setMemory(new ResourceUsage(800.0, 200.0)); data.setDirectMemory(new ResourceUsage(2.0, 100.0)); data.setBandwidthIn(new ResourceUsage(3.0, 100.0)); data.setBandwidthOut(new ResourceUsage(4.0, 100.0)); @@ -51,11 +50,10 @@ public void testMaxResourceUsage() { double epsilon = 0.00001; double weight = 0.5; // skips memory usage - assertEquals(data.getMaxResourceUsage(), 0.04, epsilon); + assertEquals(data.getMaxResourceUsage(), data.getBandwidthOut().percentUsage() / 100, epsilon); - assertEquals( - data.getMaxResourceUsageWithWeight( - weight, weight, weight, weight, weight), 2.0, epsilon); + assertEquals(data.getMaxResourceUsageWithWeight(weight, weight, weight, weight), + data.getBandwidthOut().percentUsage() * weight / 100, epsilon); } /* From 5d6932137d76d544f939bef27df25f61b4a4d00d Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Mon, 27 Feb 2023 08:56:47 +0800 Subject: [PATCH 059/174] [feat] PIP-242 part 1 Introduce configuration `strictTopicNameEnabled` (#19582) --- .../pulsar/broker/ServiceConfiguration.java | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 5a85be11745e2..c18367f265506 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -1399,6 +1399,31 @@ The delayed message index bucket time step(in seconds) in per bucket snapshot se doc = "Enable or disable system topic.") private boolean systemTopicEnabled = true; + @FieldContext( + category = CATEGORY_SERVER, + doc = "# Enable strict topic name check. Which includes two parts as follows:\n" + + "# 1. Mark `-partition-` as a keyword.\n" + + "# E.g.\n" + + " Create a non-partitioned topic.\n" + + " No corresponding partitioned topic\n" + + " - persistent://public/default/local-name (passed)\n" + + " - persistent://public/default/local-name-partition-z (rejected by keyword)\n" + + " - persistent://public/default/local-name-partition-0 (rejected by keyword)\n" + + " Has corresponding partitioned topic, partitions=2 and topic partition name " + + "is persistent://public/default/local-name\n" + + " - persistent://public/default/local-name-partition-0 (passed," + + " Because it is the partition topic's sub-partition)\n" + + " - persistent://public/default/local-name-partition-z (rejected by keyword)\n" + + " - persistent://public/default/local-name-partition-4 (rejected," + + " Because it exceeds the number of maximum partitions)\n" + + " Create a partitioned topic(topic metadata)\n" + + " - persistent://public/default/local-name (passed)\n" + + " - persistent://public/default/local-name-partition-z (rejected by keyword)\n" + + " - persistent://public/default/local-name-partition-0 (rejected by keyword)\n" + + "# 2. Allowed alphanumeric (a-zA-Z_0-9) and these special chars -=:. for topic name.\n" + + "# NOTE: This flag will be removed in some major releases in the future.\n") + private boolean strictTopicNameEnabled = false; + @FieldContext( category = CATEGORY_SCHEMA, doc = "The schema compatibility strategy to use for system topics" From dc02c404223db757702a8e1318e970adc37a660b Mon Sep 17 00:00:00 2001 From: Anonymitaet <50226895+Anonymitaet@users.noreply.github.com> Date: Mon, 27 Feb 2023 16:23:41 +0800 Subject: [PATCH 060/174] [fix][doc] update link for Pulsar PR Naming Convention Guide (#19642) --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 9dbe56a13717b..01ac26570b2d1 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,7 +1,7 @@ 4.1.12.1 5.1.0 - 4.1.87.Final - 0.0.17.Final + 4.1.89.Final + 0.0.18.Final 9.4.48.v20220622 2.5.2 2.34 diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index d11ab20397db7..64c488c38bbe9 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -231,21 +231,21 @@ The Apache Software License, Version 2.0 - commons-compress-1.21.jar - commons-lang3-3.11.jar * Netty - - netty-buffer-4.1.87.Final.jar - - netty-codec-4.1.87.Final.jar - - netty-codec-dns-4.1.87.Final.jar - - netty-codec-http-4.1.87.Final.jar - - netty-codec-haproxy-4.1.87.Final.jar - - netty-codec-socks-4.1.87.Final.jar - - netty-handler-proxy-4.1.87.Final.jar - - netty-common-4.1.87.Final.jar - - netty-handler-4.1.87.Final.jar + - netty-buffer-4.1.89.Final.jar + - netty-codec-4.1.89.Final.jar + - netty-codec-dns-4.1.89.Final.jar + - netty-codec-http-4.1.89.Final.jar + - netty-codec-haproxy-4.1.89.Final.jar + - netty-codec-socks-4.1.89.Final.jar + - netty-handler-proxy-4.1.89.Final.jar + - netty-common-4.1.89.Final.jar + - netty-handler-4.1.89.Final.jar - netty-reactive-streams-2.0.6.jar - - netty-resolver-4.1.87.Final.jar - - netty-resolver-dns-4.1.87.Final.jar - - netty-resolver-dns-classes-macos-4.1.87.Final.jar - - netty-resolver-dns-native-macos-4.1.87.Final-osx-aarch_64.jar - - netty-resolver-dns-native-macos-4.1.87.Final-osx-x86_64.jar + - netty-resolver-4.1.89.Final.jar + - netty-resolver-dns-4.1.89.Final.jar + - netty-resolver-dns-classes-macos-4.1.89.Final.jar + - netty-resolver-dns-native-macos-4.1.89.Final-osx-aarch_64.jar + - netty-resolver-dns-native-macos-4.1.89.Final-osx-x86_64.jar - netty-tcnative-boringssl-static-2.0.56.Final.jar - netty-tcnative-boringssl-static-2.0.56.Final-linux-aarch_64.jar - netty-tcnative-boringssl-static-2.0.56.Final-linux-x86_64.jar @@ -253,15 +253,15 @@ The Apache Software License, Version 2.0 - netty-tcnative-boringssl-static-2.0.56.Final-osx-x86_64.jar - netty-tcnative-boringssl-static-2.0.56.Final-windows-x86_64.jar - netty-tcnative-classes-2.0.56.Final.jar - - netty-transport-4.1.87.Final.jar - - netty-transport-classes-epoll-4.1.87.Final.jar - - netty-transport-native-epoll-4.1.87.Final-linux-x86_64.jar - - netty-transport-native-unix-common-4.1.87.Final.jar - - netty-transport-native-unix-common-4.1.87.Final-linux-x86_64.jar - - netty-codec-http2-4.1.87.Final.jar - - netty-incubator-transport-classes-io_uring-0.0.17.Final.jar - - netty-incubator-transport-native-io_uring-0.0.17.Final-linux-x86_64.jar - - netty-incubator-transport-native-io_uring-0.0.17.Final-linux-aarch_64.jar + - netty-transport-4.1.89.Final.jar + - netty-transport-classes-epoll-4.1.89.Final.jar + - netty-transport-native-epoll-4.1.89.Final-linux-x86_64.jar + - netty-transport-native-unix-common-4.1.89.Final.jar + - netty-transport-native-unix-common-4.1.89.Final-linux-x86_64.jar + - netty-codec-http2-4.1.89.Final.jar + - netty-incubator-transport-classes-io_uring-0.0.18.Final.jar + - netty-incubator-transport-native-io_uring-0.0.18.Final-linux-x86_64.jar + - netty-incubator-transport-native-io_uring-0.0.18.Final-linux-aarch_64.jar * GRPC - grpc-api-1.45.1.jar - grpc-context-1.45.1.jar From 69fb3c2ca3faa32ff12fd1270730b3517ea69220 Mon Sep 17 00:00:00 2001 From: Enrico Olivelli Date: Tue, 28 Feb 2023 12:17:33 +0100 Subject: [PATCH 067/174] [fix][client] Fix race condition that leads to caching failed CompletableFutures in ConnectionPool (#19661) --- .../apache/pulsar/client/impl/ConnectionPool.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java index 3a9a2b9b7ab94..1420d81c688ee 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java @@ -216,6 +216,15 @@ public CompletableFuture getConnection(InetSocketAddress logicalAddre pool.computeIfAbsent(logicalAddress, a -> new ConcurrentHashMap<>()); CompletableFuture completableFuture = innerPool .computeIfAbsent(randomKey, k -> createConnection(logicalAddress, physicalAddress, randomKey)); + if (completableFuture.isCompletedExceptionally()) { + // we cannot cache a failed connection, so we remove it from the pool + // there is a race condition in which + // cleanupConnection is called before caching this result + // and so the clean up fails + cleanupConnection(logicalAddress, randomKey, completableFuture); + return completableFuture; + } + return completableFuture.thenCompose(clientCnx -> { // If connection already release, create a new one. if (clientCnx.getIdleState().isReleased()) { @@ -274,6 +283,10 @@ private CompletableFuture createConnection(InetSocketAddress logicalA }).exceptionally(exception -> { log.warn("[{}] Connection handshake failed: {}", cnx.channel(), exception.getMessage()); cnxFuture.completeExceptionally(exception); + // this cleanupConnection may happen before that the + // CompletableFuture is cached into the "pool" map, + // it is not enough to clean it here, we need to clean it + // in the "pool" map when the CompletableFuture is cached cleanupConnection(logicalAddress, connectionKey, cnxFuture); cnx.ctx().close(); return null; From 7dad5791bddd25bbfc4b154056ac723bb5d64ede Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Wed, 1 Mar 2023 09:13:04 +0800 Subject: [PATCH 068/174] [fix][broker] Fix BucketDelayedDeliveryTracker merge issues (#19615) --- .../BucketDelayedDeliveryTrackerFactory.java | 3 +- .../bucket/BucketDelayedDeliveryTracker.java | 64 ++++++++++++++----- .../delayed/bucket/ImmutableBucket.java | 22 +++++-- .../BucketDelayedDeliveryTrackerTest.java | 9 +++ .../persistent/BucketDelayedDeliveryTest.java | 5 ++ 5 files changed, 80 insertions(+), 23 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerFactory.java index 16648d84e9fba..ae9cb23ceb922 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerFactory.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerFactory.java @@ -49,6 +49,7 @@ public class BucketDelayedDeliveryTrackerFactory implements DelayedDeliveryTrack public void initialize(PulsarService pulsarService) throws Exception { ServiceConfiguration config = pulsarService.getConfig(); bucketSnapshotStorage = new BookkeeperBucketSnapshotStorage(pulsarService); + bucketSnapshotStorage.start(); this.timer = new HashedWheelTimer(new DefaultThreadFactory("pulsar-delayed-delivery"), config.getDelayedDeliveryTickTimeMillis(), TimeUnit.MILLISECONDS); this.tickTimeMillis = config.getDelayedDeliveryTickTimeMillis(); @@ -63,7 +64,7 @@ public void initialize(PulsarService pulsarService) throws Exception { public DelayedDeliveryTracker newTracker(PersistentDispatcherMultipleConsumers dispatcher) { return new BucketDelayedDeliveryTracker(dispatcher, timer, tickTimeMillis, isDelayedDeliveryDeliverAtTimeStrict, bucketSnapshotStorage, delayedDeliveryMinIndexCountPerBucket, - delayedDeliveryMaxTimeStepPerBucketSnapshotSegmentSeconds, + TimeUnit.SECONDS.toMillis(delayedDeliveryMaxTimeStepPerBucketSnapshotSegmentSeconds), delayedDeliveryMaxNumBuckets); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index 77c1dfb1eea14..bd4ef92cc7320 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -55,6 +55,7 @@ import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue; +import org.roaringbitmap.RoaringBitmap; @Slf4j @ThreadSafe @@ -64,7 +65,7 @@ public class BucketDelayedDeliveryTracker extends AbstractDelayedDeliveryTracker private final long minIndexCountPerBucket; - private final long timeStepPerBucketSnapshotSegment; + private final long timeStepPerBucketSnapshotSegmentInMillis; private final int maxNumBuckets; @@ -84,21 +85,21 @@ public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispat Timer timer, long tickTimeMillis, boolean isDelayedDeliveryDeliverAtTimeStrict, BucketSnapshotStorage bucketSnapshotStorage, - long minIndexCountPerBucket, long timeStepPerBucketSnapshotSegment, + long minIndexCountPerBucket, long timeStepPerBucketSnapshotSegmentInMillis, int maxNumBuckets) { this(dispatcher, timer, tickTimeMillis, Clock.systemUTC(), isDelayedDeliveryDeliverAtTimeStrict, - bucketSnapshotStorage, minIndexCountPerBucket, timeStepPerBucketSnapshotSegment, maxNumBuckets); + bucketSnapshotStorage, minIndexCountPerBucket, timeStepPerBucketSnapshotSegmentInMillis, maxNumBuckets); } public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, Timer timer, long tickTimeMillis, Clock clock, boolean isDelayedDeliveryDeliverAtTimeStrict, BucketSnapshotStorage bucketSnapshotStorage, - long minIndexCountPerBucket, long timeStepPerBucketSnapshotSegment, + long minIndexCountPerBucket, long timeStepPerBucketSnapshotSegmentInMillis, int maxNumBuckets) { super(dispatcher, timer, tickTimeMillis, clock, isDelayedDeliveryDeliverAtTimeStrict); this.minIndexCountPerBucket = minIndexCountPerBucket; - this.timeStepPerBucketSnapshotSegment = timeStepPerBucketSnapshotSegment; + this.timeStepPerBucketSnapshotSegmentInMillis = timeStepPerBucketSnapshotSegmentInMillis; this.maxNumBuckets = maxNumBuckets; this.sharedBucketPriorityQueue = new TripleLongPriorityQueue(); this.immutableBuckets = TreeRangeMap.create(); @@ -255,7 +256,7 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver && lastMutableBucket.size() >= minIndexCountPerBucket && !lastMutableBucket.isEmpty()) { Pair immutableBucketDelayedIndexPair = - lastMutableBucket.sealBucketAndAsyncPersistent(this.timeStepPerBucketSnapshotSegment, + lastMutableBucket.sealBucketAndAsyncPersistent(this.timeStepPerBucketSnapshotSegmentInMillis, this.sharedBucketPriorityQueue); afterCreateImmutableBucket(immutableBucketDelayedIndexPair); lastMutableBucket.resetLastMutableBucketRange(); @@ -303,17 +304,21 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot() { long numberMessages = bucketL.numberBucketDelayedMessages + bucketR.numberBucketDelayedMessages; if (numberMessages < minNumberMessages) { minNumberMessages = (int) numberMessages; - minIndex = i; + if (bucketL.lastSegmentEntryId > bucketL.getCurrentSegmentEntryId()) { + minIndex = i; + } } } + + if (minIndex == -1) { + log.warn("[{}] Can't find able merged bucket", dispatcher.getName()); + return CompletableFuture.completedFuture(null); + } return asyncMergeBucketSnapshot(values.get(minIndex), values.get(minIndex + 1)); } private synchronized CompletableFuture asyncMergeBucketSnapshot(ImmutableBucket bucketA, ImmutableBucket bucketB) { - immutableBuckets.remove(Range.closed(bucketA.startLedgerId, bucketA.endLedgerId)); - immutableBuckets.remove(Range.closed(bucketB.startLedgerId, bucketB.endLedgerId)); - CompletableFuture snapshotCreateFutureA = bucketA.getSnapshotCreateFuture().orElse(CompletableFuture.completedFuture(null)); CompletableFuture snapshotCreateFutureB = @@ -328,16 +333,41 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot(ImmutableB .thenAccept(combinedDelayedIndexQueue -> { Pair immutableBucketDelayedIndexPair = lastMutableBucket.createImmutableBucketAndAsyncPersistent( - timeStepPerBucketSnapshotSegment, sharedBucketPriorityQueue, + timeStepPerBucketSnapshotSegmentInMillis, sharedBucketPriorityQueue, combinedDelayedIndexQueue, bucketA.startLedgerId, bucketB.endLedgerId); + + // Merge bit map to new bucket + Map delayedIndexBitMapA = bucketA.getDelayedIndexBitMap(); + Map delayedIndexBitMapB = bucketB.getDelayedIndexBitMap(); + Map delayedIndexBitMap = new HashMap<>(delayedIndexBitMapA); + delayedIndexBitMapB.forEach((ledgerId, bitMapB) -> { + delayedIndexBitMap.compute(ledgerId, (k, bitMapA) -> { + if (bitMapA == null) { + return bitMapB; + } + + bitMapA.or(bitMapB); + return bitMapA; + }); + }); + immutableBucketDelayedIndexPair.getLeft().setDelayedIndexBitMap(delayedIndexBitMap); + afterCreateImmutableBucket(immutableBucketDelayedIndexPair); - immutableBucketDelayedIndexPair.getLeft().getSnapshotCreateFuture() - .orElse(CompletableFuture.completedFuture(null)).thenCompose(___ -> { - CompletableFuture removeAFuture = bucketA.asyncDeleteBucketSnapshot(); - CompletableFuture removeBFuture = bucketB.asyncDeleteBucketSnapshot(); - return CompletableFuture.allOf(removeAFuture, removeBFuture); - }); + CompletableFuture snapshotCreateFuture = CompletableFuture.completedFuture(null); + if (immutableBucketDelayedIndexPair != null) { + snapshotCreateFuture = immutableBucketDelayedIndexPair.getLeft().getSnapshotCreateFuture() + .orElse(CompletableFuture.completedFuture(null)); + } + + snapshotCreateFuture.thenCompose(___ -> { + CompletableFuture removeAFuture = bucketA.asyncDeleteBucketSnapshot(); + CompletableFuture removeBFuture = bucketB.asyncDeleteBucketSnapshot(); + return CompletableFuture.allOf(removeAFuture, removeBFuture); + }); + + immutableBuckets.remove(Range.closed(bucketA.startLedgerId, bucketA.endLedgerId)); + immutableBuckets.remove(Range.closed(bucketB.startLedgerId, bucketB.endLedgerId)); }); }); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java index 8348b4999ed80..3e9c577454fed 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java @@ -24,9 +24,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.function.Supplier; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedCursor; @@ -134,7 +132,11 @@ private void recoverDelayedIndexBitMapAndNumber(int startSnapshotIndex, } CompletableFuture> getRemainSnapshotSegment() { - return bucketSnapshotStorage.getBucketSnapshotSegment(getAndUpdateBucketId(), currentSegmentEntryId, + int nextSegmentEntryId = currentSegmentEntryId + 1; + if (nextSegmentEntryId > lastSegmentEntryId) { + return CompletableFuture.completedFuture(Collections.emptyList()); + } + return bucketSnapshotStorage.getBucketSnapshotSegment(getAndUpdateBucketId(), nextSegmentEntryId, lastSegmentEntryId); } @@ -155,9 +157,19 @@ void clear(boolean delete) { getSnapshotCreateFuture().ifPresent(snapshotGenerateFuture -> { if (delete) { snapshotGenerateFuture.cancel(true); + String bucketKey = bucketKey(); + long bucketId = getAndUpdateBucketId(); try { - asyncDeleteBucketSnapshot().get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS); - } catch (InterruptedException | ExecutionException | TimeoutException e) { + // Because bucketSnapshotStorage.deleteBucketSnapshot may be use the same thread with clear, + // so we can't block deleteBucketSnapshot when clearing the bucket snapshot. + removeBucketCursorProperty(bucketKey()) + .thenApply(__ -> bucketSnapshotStorage.deleteBucketSnapshot(bucketId).exceptionally(ex -> { + log.error("Failed to delete bucket snapshot, bucketId: {}, bucketKey: {}", + bucketId, bucketKey, ex); + return null; + })).get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS); + } catch (Exception e) { + log.error("Failed to delete bucket snapshot, bucketId: {}, bucketKey: {}", bucketId, bucketKey, e); if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java index 0a2a76ec339eb..920f2cf2b64b3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java @@ -36,6 +36,7 @@ import java.util.Arrays; import java.util.List; import java.util.NavigableMap; +import java.util.NavigableSet; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.TimeUnit; @@ -253,5 +254,13 @@ public void testMergeSnapshot(BucketDelayedDeliveryTracker tracker) { int size = tracker.getImmutableBuckets().asMapOfRanges().size(); assertEquals(10, size); + + clockTime.set(110 * 10); + + NavigableSet scheduledMessages = tracker.getScheduledMessages(110); + for (int i = 1; i <= 110; i++) { + PositionImpl position = scheduledMessages.pollFirst(); + assertEquals(position, PositionImpl.get(i, i)); + } } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java index a54c0ce794cf0..5d81ba8bc0261 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java @@ -30,6 +30,11 @@ public class BucketDelayedDeliveryTest extends DelayedDeliveryTest { @Override public void setup() throws Exception { conf.setDelayedDeliveryTrackerFactoryClassName(BucketDelayedDeliveryTrackerFactory.class.getName()); + conf.setDelayedDeliveryMaxNumBuckets(10); + conf.setDelayedDeliveryMaxTimeStepPerBucketSnapshotSegmentSeconds(1); + conf.setDelayedDeliveryMinIndexCountPerBucket(50); + conf.setManagedLedgerMaxEntriesPerLedger(50); + conf.setManagedLedgerMinLedgerRolloverTimeMinutes(0); super.setup(); } From 145e985f7b7ef981d33036b79b24fd5a4e27d43c Mon Sep 17 00:00:00 2001 From: houxiaoyu Date: Wed, 1 Mar 2023 09:43:13 +0800 Subject: [PATCH 069/174] [improve][broker] Replace ScheduledExecutorService to ExecutorService in ModularLoadManagerImpl (#19656) --- .../loadbalance/impl/ModularLoadManagerImpl.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java index 7a933908962ec..c1afe6007f6bb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java @@ -33,9 +33,9 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; @@ -174,8 +174,8 @@ public class ModularLoadManagerImpl implements ModularLoadManager { // Pulsar service used to initialize this. private PulsarService pulsar; - // Executor service used to regularly update broker data. - private final ScheduledExecutorService scheduler; + // Executor service used to update broker data. + private final ExecutorService executors; // check if given broker can load persistent/non-persistent topic private final BrokerTopicLoadingPredicate brokerTopicLoadingPredicate; @@ -215,7 +215,7 @@ public ModularLoadManagerImpl() { loadData = new LoadData(); loadSheddingPipeline = new ArrayList<>(); preallocatedBundleToBroker = new ConcurrentHashMap<>(); - scheduler = Executors.newSingleThreadScheduledExecutor( + executors = Executors.newSingleThreadExecutor( new ExecutorProvider.ExtendedThreadFactory("pulsar-modular-load-manager")); this.brokerToFailureDomainMap = new HashMap<>(); this.bundleBrokerAffinityMap = new ConcurrentHashMap<>(); @@ -276,7 +276,7 @@ public void initialize(final PulsarService pulsar) { // register listeners for domain changes pulsar.getPulsarResources().getClusterResources().getFailureDomainResources() .registerListener(__ -> { - scheduler.execute(() -> refreshBrokerToFailureDomainMap()); + executors.execute(() -> refreshBrokerToFailureDomainMap()); }); loadSheddingPipeline.add(createLoadSheddingStrategy()); @@ -290,7 +290,7 @@ public void handleDataNotification(Notification t) { }); try { - scheduler.execute(ModularLoadManagerImpl.this::updateAll); + executors.execute(ModularLoadManagerImpl.this::updateAll); } catch (RejectedExecutionException e) { // Executor is shutting down } @@ -982,7 +982,7 @@ public void start() throws PulsarServerException { */ @Override public void stop() throws PulsarServerException { - scheduler.shutdownNow(); + executors.shutdownNow(); try { brokersData.close(); From a12f5541cee0d51fad654e97487edf1140de3286 Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Wed, 1 Mar 2023 00:58:33 -0800 Subject: [PATCH 070/174] [improve][broker] PIP-192: Added VersionId in ServiceUnitStateData (#19620) --- .../channel/ServiceUnitStateChannelImpl.java | 59 +++-- .../ServiceUnitStateCompactionStrategy.java | 11 +- .../channel/ServiceUnitStateData.java | 15 +- .../channel/ServiceUnitStateChannelTest.java | 12 +- ...erviceUnitStateCompactionStrategyTest.java | 241 +++++++++++------- .../channel/ServiceUnitStateDataTest.java | 20 +- .../ServiceUnitStateCompactionTest.java | 36 ++- 7 files changed, 251 insertions(+), 143 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index 9f205f85c5454..4819a54fcad73 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -103,7 +103,7 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel { NamespaceName.SYSTEM_NAMESPACE, "loadbalancer-service-unit-state").toString(); private static final long MAX_IN_FLIGHT_STATE_WAITING_TIME_IN_MILLIS = 30 * 1000; // 30sec - + public static final long VERSION_ID_INIT = 1; // initial versionId private static final long OWNERSHIP_MONITOR_DELAY_TIME_IN_SECS = 60; public static final long MAX_CLEAN_UP_DELAY_TIME_IN_SECS = 3 * 60; // 3 mins private static final long MIN_CLEAN_UP_DELAY_TIME_IN_SECS = 0; // 0 secs to clean immediately @@ -451,11 +451,25 @@ public CompletableFuture> getOwnerAsync(String serviceUnit) { } } + private long getNextVersionId(String serviceUnit) { + var data = tableview.get(serviceUnit); + return getNextVersionId(data); + } + + private long getNextVersionId(ServiceUnitStateData data) { + return data == null ? VERSION_ID_INIT : data.versionId() + 1; + } + public CompletableFuture publishAssignEventAsync(String serviceUnit, String broker) { + if (!validateChannelState(Started, true)) { + return CompletableFuture.failedFuture( + new IllegalStateException("Invalid channel state:" + channelState.name())); + } EventType eventType = Assign; eventCounters.get(eventType).getTotal().incrementAndGet(); CompletableFuture getOwnerRequest = deferGetOwnerRequest(serviceUnit); - pubAsync(serviceUnit, new ServiceUnitStateData(Assigning, broker)) + + pubAsync(serviceUnit, new ServiceUnitStateData(Assigning, broker, getNextVersionId(serviceUnit))) .whenComplete((__, ex) -> { if (ex != null) { getOwnerRequests.remove(serviceUnit, getOwnerRequest); @@ -469,16 +483,20 @@ public CompletableFuture publishAssignEventAsync(String serviceUnit, Str } public CompletableFuture publishUnloadEventAsync(Unload unload) { + if (!validateChannelState(Started, true)) { + return CompletableFuture.failedFuture( + new IllegalStateException("Invalid channel state:" + channelState.name())); + } EventType eventType = Unload; eventCounters.get(eventType).getTotal().incrementAndGet(); String serviceUnit = unload.serviceUnit(); CompletableFuture future; if (isTransferCommand(unload)) { future = pubAsync(serviceUnit, new ServiceUnitStateData( - Assigning, unload.destBroker().get(), unload.sourceBroker())); + Assigning, unload.destBroker().get(), unload.sourceBroker(), getNextVersionId(serviceUnit))); } else { future = pubAsync(serviceUnit, new ServiceUnitStateData( - Releasing, unload.sourceBroker())); + Releasing, unload.sourceBroker(), getNextVersionId(serviceUnit))); } return future.whenComplete((__, ex) -> { @@ -489,10 +507,15 @@ public CompletableFuture publishUnloadEventAsync(Unload unload) { } public CompletableFuture publishSplitEventAsync(Split split) { + if (!validateChannelState(Started, true)) { + return CompletableFuture.failedFuture( + new IllegalStateException("Invalid channel state:" + channelState.name())); + } EventType eventType = Split; eventCounters.get(eventType).getTotal().incrementAndGet(); String serviceUnit = split.serviceUnit(); - ServiceUnitStateData next = new ServiceUnitStateData(Splitting, split.sourceBroker()); + ServiceUnitStateData next = + new ServiceUnitStateData(Splitting, split.sourceBroker(), getNextVersionId(serviceUnit)); return pubAsync(serviceUnit, next).whenComplete((__, ex) -> { if (ex != null) { eventCounters.get(eventType).getFailure().incrementAndGet(); @@ -599,7 +622,8 @@ private void handleOwnEvent(String serviceUnit, ServiceUnitStateData data) { private void handleAssignEvent(String serviceUnit, ServiceUnitStateData data) { if (isTargetBroker(data.broker())) { ServiceUnitStateData next = new ServiceUnitStateData( - isTransferCommand(data) ? Releasing : Owned, data.broker(), data.sourceBroker()); + isTransferCommand(data) ? Releasing : Owned, data.broker(), data.sourceBroker(), + getNextVersionId(data)); pubAsync(serviceUnit, next) .whenComplete((__, e) -> log(e, serviceUnit, data, next)); } @@ -608,7 +632,8 @@ private void handleAssignEvent(String serviceUnit, ServiceUnitStateData data) { private void handleReleaseEvent(String serviceUnit, ServiceUnitStateData data) { if (isTransferCommand(data)) { if (isTargetBroker(data.sourceBroker())) { - ServiceUnitStateData next = new ServiceUnitStateData(Owned, data.broker(), data.sourceBroker()); + ServiceUnitStateData next = + new ServiceUnitStateData(Owned, data.broker(), data.sourceBroker(), getNextVersionId(data)); // TODO: when close, pass message to clients to connect to the new broker closeServiceUnit(serviceUnit) .thenCompose(__ -> pubAsync(serviceUnit, next)) @@ -616,7 +641,7 @@ private void handleReleaseEvent(String serviceUnit, ServiceUnitStateData data) { } } else { if (isTargetBroker(data.broker())) { - ServiceUnitStateData next = new ServiceUnitStateData(Free, data.broker()); + ServiceUnitStateData next = new ServiceUnitStateData(Free, data.broker(), getNextVersionId(data)); closeServiceUnit(serviceUnit) .thenCompose(__ -> pubAsync(serviceUnit, next)) .whenComplete((__, e) -> log(e, serviceUnit, data, next)); @@ -660,10 +685,6 @@ private void handleInitEvent(String serviceUnit) { } private CompletableFuture pubAsync(String serviceUnit, ServiceUnitStateData data) { - if (!validateChannelState(Started, true)) { - return CompletableFuture.failedFuture( - new IllegalStateException("Invalid channel state:" + channelState.name())); - } CompletableFuture future = new CompletableFuture<>(); producer.newMessage() .key(serviceUnit) @@ -774,7 +795,7 @@ protected void splitServiceUnitOnceAndRetry(NamespaceService namespaceService, updateFuture.completeExceptionally(new BrokerServiceException.ServiceUnitNotReadyException(msg)); return; } - ServiceUnitStateData next = new ServiceUnitStateData(Owned, data.broker()); + ServiceUnitStateData next = new ServiceUnitStateData(Owned, data.broker(), VERSION_ID_INIT); NamespaceBundles targetNsBundle = splitBundlesPair.getLeft(); List splitBundles = Collections.unmodifiableList(splitBundlesPair.getRight()); List successPublishedBundles = @@ -812,7 +833,8 @@ protected void splitServiceUnitOnceAndRetry(NamespaceService namespaceService, updateFuture.thenAccept(r -> { // Delete the old bundle - pubAsync(serviceUnit, new ServiceUnitStateData(Deleted, data.broker())).thenRun(() -> { + pubAsync(serviceUnit, new ServiceUnitStateData(Deleted, data.broker(), getNextVersionId(data))) + .thenRun(() -> { // Update bundled_topic cache for load-report-generation pulsar.getBrokerService().refreshTopicToStatsMaps(bundle); // TODO: Update the load data immediately if needed. @@ -938,7 +960,7 @@ private void overrideOwnership(String serviceUnit, ServiceUnitStateData orphanDa Optional selectedBroker = brokerSelector.select(availableBrokers, null, getContext()); if (selectedBroker.isPresent()) { - var override = new ServiceUnitStateData(Owned, selectedBroker.get(), true); + var override = new ServiceUnitStateData(Owned, selectedBroker.get(), true, getNextVersionId(orphanData)); log.info("Overriding ownership serviceUnit:{} from orphanData:{} to overrideData:{}", serviceUnit, orphanData, override); pubAsync(serviceUnit, override).whenComplete((__, e) -> { @@ -1007,19 +1029,20 @@ private void doCleanup(String broker) throws ExecutionException, InterruptedExce private Optional getOverrideStateData(String serviceUnit, ServiceUnitStateData orphanData, Set availableBrokers, LoadManagerContext context) { + long nextVersionId = getNextVersionId(orphanData); if (isTransferCommand(orphanData)) { // rollback to the src - return Optional.of(new ServiceUnitStateData(Owned, orphanData.sourceBroker(), true)); + return Optional.of(new ServiceUnitStateData(Owned, orphanData.sourceBroker(), true, nextVersionId)); } else if (orphanData.state() == Assigning) { // assign // roll-forward to another broker Optional selectedBroker = brokerSelector.select(availableBrokers, null, context); if (selectedBroker.isEmpty()) { return Optional.empty(); } - return Optional.of(new ServiceUnitStateData(Owned, selectedBroker.get(), true)); + return Optional.of(new ServiceUnitStateData(Owned, selectedBroker.get(), true, nextVersionId)); } else if (orphanData.state() == Splitting || orphanData.state() == Releasing) { // rollback to the target broker for split and unload - return Optional.of(new ServiceUnitStateData(Owned, orphanData.broker(), true)); + return Optional.of(new ServiceUnitStateData(Owned, orphanData.broker(), true, nextVersionId)); } else { var msg = String.format("Failed to get the overrideStateData from serviceUnit=%s, orphanData=%s", serviceUnit, orphanData); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java index d2a585af9d9d5..8af0f0c027da4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java @@ -50,14 +50,19 @@ public void checkBrokers(boolean check) { public boolean shouldKeepLeft(ServiceUnitStateData from, ServiceUnitStateData to) { if (to == null) { return false; - } else if (to.force()) { - return false; } + // Skip the compaction case where from = null and to.versionId > 1 + if (from != null && from.versionId() + 1 != to.versionId()) { + return true; + } + + if (to.force()) { + return false; + } ServiceUnitState prevState = state(from); ServiceUnitState state = state(to); - if (!ServiceUnitState.isValidTransition(prevState, state)) { return true; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java index 6a04431de64d5..ef25acff10a4b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.channel; - import java.util.Objects; import org.apache.commons.lang3.StringUtils; @@ -28,7 +27,7 @@ */ public record ServiceUnitStateData( - ServiceUnitState state, String broker, String sourceBroker, boolean force, long timestamp) { + ServiceUnitState state, String broker, String sourceBroker, boolean force, long timestamp, long versionId) { public ServiceUnitStateData { Objects.requireNonNull(state); @@ -37,16 +36,16 @@ public record ServiceUnitStateData( } } - public ServiceUnitStateData(ServiceUnitState state, String broker, String sourceBroker) { - this(state, broker, sourceBroker, false, System.currentTimeMillis()); + public ServiceUnitStateData(ServiceUnitState state, String broker, String sourceBroker, long versionId) { + this(state, broker, sourceBroker, false, System.currentTimeMillis(), versionId); } - public ServiceUnitStateData(ServiceUnitState state, String broker) { - this(state, broker, null, false, System.currentTimeMillis()); + public ServiceUnitStateData(ServiceUnitState state, String broker, long versionId) { + this(state, broker, null, false, System.currentTimeMillis(), versionId); } - public ServiceUnitStateData(ServiceUnitState state, String broker, boolean force) { - this(state, broker, null, force, System.currentTimeMillis()); + public ServiceUnitStateData(ServiceUnitState state, String broker, boolean force, long versionId) { + this(state, broker, null, force, System.currentTimeMillis(), versionId); } public static ServiceUnitState state(ServiceUnitStateData data) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java index 49eee6ecb7aef..6aa8fe387605b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java @@ -924,26 +924,26 @@ public void conflictAndCompactionTest() throws ExecutionException, InterruptedEx @Test(priority = 11) public void ownerLookupCountTests() throws IllegalAccessException { - overrideTableView(channel1, bundle, new ServiceUnitStateData(Assigning, "b1")); + overrideTableView(channel1, bundle, new ServiceUnitStateData(Assigning, "b1", 1)); channel1.getOwnerAsync(bundle); channel1.getOwnerAsync(bundle); - overrideTableView(channel1, bundle, new ServiceUnitStateData(Owned, "b1")); + overrideTableView(channel1, bundle, new ServiceUnitStateData(Owned, "b1", 1)); channel1.getOwnerAsync(bundle); channel1.getOwnerAsync(bundle); channel1.getOwnerAsync(bundle); - overrideTableView(channel1, bundle, new ServiceUnitStateData(Releasing, "b1")); + overrideTableView(channel1, bundle, new ServiceUnitStateData(Releasing, "b1", 1)); channel1.getOwnerAsync(bundle); channel1.getOwnerAsync(bundle); - overrideTableView(channel1, bundle, new ServiceUnitStateData(Splitting, "b1")); + overrideTableView(channel1, bundle, new ServiceUnitStateData(Splitting, "b1", 1)); channel1.getOwnerAsync(bundle); - overrideTableView(channel1, bundle, new ServiceUnitStateData(Free, "b1")); + overrideTableView(channel1, bundle, new ServiceUnitStateData(Free, "b1", 1)); channel1.getOwnerAsync(bundle); - overrideTableView(channel1, bundle, new ServiceUnitStateData(Deleted, "b1")); + overrideTableView(channel1, bundle, new ServiceUnitStateData(Deleted, "b1", 1)); channel1.getOwnerAsync(bundle); channel1.getOwnerAsync(bundle); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java index 1a4aba15f9e6f..0cd05d8bd7559 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java @@ -32,110 +32,173 @@ @Test(groups = "broker") public class ServiceUnitStateCompactionStrategyTest { ServiceUnitStateCompactionStrategy strategy = new ServiceUnitStateCompactionStrategy(); + String dst = "dst"; + String src = "src"; ServiceUnitStateData data(ServiceUnitState state) { - return new ServiceUnitStateData(state, "broker"); + return new ServiceUnitStateData(state, "broker", 1); } ServiceUnitStateData data(ServiceUnitState state, String dst) { - return new ServiceUnitStateData(state, dst, null); + return new ServiceUnitStateData(state, dst, null, 1); } + ServiceUnitStateData data(ServiceUnitState state, String src, String dst) { - return new ServiceUnitStateData(state, dst, src); + return new ServiceUnitStateData(state, dst, src, 1); + } + + ServiceUnitStateData data2(ServiceUnitState state) { + return new ServiceUnitStateData(state, "broker", 2); + } + + ServiceUnitStateData data2(ServiceUnitState state, String dst) { + return new ServiceUnitStateData(state, dst, null, 2); + } + + ServiceUnitStateData data2(ServiceUnitState state, String src, String dst) { + return new ServiceUnitStateData(state, dst, src, 2); } @Test - public void test() throws InterruptedException { - String dst = "dst"; - String src = "src"; + public void testVersionId(){ + assertTrue(strategy.shouldKeepLeft( + new ServiceUnitStateData(Assigning, dst, 1), + new ServiceUnitStateData(Assigning, dst, 1))); + + assertTrue(strategy.shouldKeepLeft( + new ServiceUnitStateData(Assigning, dst, 1), + new ServiceUnitStateData(Assigning, dst, 2))); + + assertFalse(strategy.shouldKeepLeft( + new ServiceUnitStateData(Owned, dst, src, 10), + new ServiceUnitStateData(Assigning, "broker2", dst, 11))); + + assertFalse(strategy.shouldKeepLeft( + new ServiceUnitStateData(Owned, dst, src, Long.MAX_VALUE), + new ServiceUnitStateData(Assigning, "broker2", dst, Long.MAX_VALUE + 1))); + + assertFalse(strategy.shouldKeepLeft( + new ServiceUnitStateData(Owned, dst, src, Long.MAX_VALUE + 1), + new ServiceUnitStateData(Assigning, "broker2", dst, Long.MAX_VALUE + 2))); + assertTrue(strategy.shouldKeepLeft( + new ServiceUnitStateData(Owned, dst, src, 10), + new ServiceUnitStateData(Assigning, "broker2", dst, 5))); + + } + + @Test + public void testForce(){ assertFalse(strategy.shouldKeepLeft( - new ServiceUnitStateData(Init, dst), - new ServiceUnitStateData(Init, dst, true))); + new ServiceUnitStateData(Init, dst, 1), + new ServiceUnitStateData(Init, dst, true, 2))); + assertTrue(strategy.shouldKeepLeft( + new ServiceUnitStateData(Init, dst, 1), + new ServiceUnitStateData(Init, dst, true, 1))); + } + + @Test + public void testTombstone() { + assertFalse(strategy.shouldKeepLeft( + data(Init), null)); + assertFalse(strategy.shouldKeepLeft( + data(Assigning), null)); assertFalse(strategy.shouldKeepLeft( data(Owned), null)); + assertFalse(strategy.shouldKeepLeft( + data(Releasing), null)); + assertFalse(strategy.shouldKeepLeft( + data(Splitting), null)); + assertFalse(strategy.shouldKeepLeft( + data(Free), null)); + assertFalse(strategy.shouldKeepLeft( + data(Deleted), null)); + } + + @Test + public void testTransitionsAndBrokers() { + + assertTrue(strategy.shouldKeepLeft(data(Init), data2(Init))); + assertFalse(strategy.shouldKeepLeft(data(Init), data2(Free))); + assertFalse(strategy.shouldKeepLeft(data(Init), data2(Assigning))); + assertFalse(strategy.shouldKeepLeft(data(Init), data2(Owned))); + assertFalse(strategy.shouldKeepLeft(data(Init), data2(Releasing))); + assertFalse(strategy.shouldKeepLeft(data(Init), data2(Splitting))); + assertFalse(strategy.shouldKeepLeft(data(Init), data2(Deleted))); + + assertTrue(strategy.shouldKeepLeft(data(Assigning), data2(Init))); + assertTrue(strategy.shouldKeepLeft(data(Assigning), data2(Free))); + assertTrue(strategy.shouldKeepLeft(data(Assigning), data2(Assigning))); + assertTrue(strategy.shouldKeepLeft(data(Assigning, "dst1"), data2(Owned, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Assigning, dst), data2(Owned, src, dst))); + assertFalse(strategy.shouldKeepLeft(data(Assigning, dst), data2(Owned, dst))); + assertTrue(strategy.shouldKeepLeft(data(Assigning, src, dst), data2(Releasing, dst))); + assertTrue(strategy.shouldKeepLeft(data(Assigning, src, "dst1"), data2(Releasing, src, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Assigning, "src1", dst), data2(Releasing, "src2", dst))); + assertFalse(strategy.shouldKeepLeft(data(Assigning, src, dst), data2(Releasing, src, dst))); + assertTrue(strategy.shouldKeepLeft(data(Assigning), data2(Splitting, dst))); + assertTrue(strategy.shouldKeepLeft(data(Assigning), data2(Deleted, dst))); + + assertTrue(strategy.shouldKeepLeft(data(Owned), data2(Init))); + assertTrue(strategy.shouldKeepLeft(data(Owned), data2(Free))); + assertTrue(strategy.shouldKeepLeft(data(Owned, src, "dst1"), data2(Assigning, src, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Assigning, dst))); + assertTrue(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Assigning, src, dst))); + assertTrue(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Assigning, dst, dst))); + assertFalse(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Assigning, dst, "dst1"))); + assertTrue(strategy.shouldKeepLeft(data(Owned), data2(Owned))); + assertTrue(strategy.shouldKeepLeft(data(Owned), data2(Releasing, dst))); + assertTrue(strategy.shouldKeepLeft(data(Owned, src, "dst1"), data2(Releasing, src, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Owned, "dst1"), data2(Releasing, "dst2"))); + assertFalse(strategy.shouldKeepLeft(data(Owned, dst), data2(Releasing, dst))); + assertFalse(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Releasing, dst))); + assertTrue(strategy.shouldKeepLeft(data(Owned, src, "dst1"), data2(Splitting, src, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Owned, "dst1"), data2(Splitting, "dst2"))); + assertFalse(strategy.shouldKeepLeft(data(Owned, dst), data2(Splitting, dst))); + assertFalse(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Splitting, dst))); + assertTrue(strategy.shouldKeepLeft(data(Owned), data2(Deleted, dst))); + + assertTrue(strategy.shouldKeepLeft(data(Releasing), data2(Init))); + assertFalse(strategy.shouldKeepLeft(data(Releasing), data2(Free))); + assertTrue(strategy.shouldKeepLeft(data(Releasing, "dst1"), data2(Free, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Releasing, "src1", dst), data2(Free, "src2", dst))); + assertTrue(strategy.shouldKeepLeft(data(Releasing), data2(Assigning))); + assertTrue(strategy.shouldKeepLeft(data(Releasing, "dst1"), data2(Owned, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Releasing, src, "dst1"), data2(Owned, src, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Releasing, "src1", dst), data2(Owned, "src2", dst))); + assertFalse(strategy.shouldKeepLeft(data(Releasing, src, dst), data2(Owned, src, dst))); + assertTrue(strategy.shouldKeepLeft(data(Releasing), data2(Releasing))); + assertTrue(strategy.shouldKeepLeft(data(Releasing), data2(Splitting))); + assertTrue(strategy.shouldKeepLeft(data(Releasing), data2(Deleted, dst))); + + assertTrue(strategy.shouldKeepLeft(data(Splitting), data2(Init))); + assertTrue(strategy.shouldKeepLeft(data(Splitting), data2(Free))); + assertTrue(strategy.shouldKeepLeft(data(Splitting), data2(Assigning))); + assertTrue(strategy.shouldKeepLeft(data(Splitting), data2(Owned))); + assertTrue(strategy.shouldKeepLeft(data(Splitting), data2(Releasing))); + assertTrue(strategy.shouldKeepLeft(data(Splitting), data2(Splitting))); + assertTrue(strategy.shouldKeepLeft(data(Splitting, src, "dst1"), data2(Deleted, src, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Splitting, "dst1"), data2(Deleted, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Splitting, "src1", dst), data2(Deleted, "src2", dst))); + assertFalse(strategy.shouldKeepLeft(data(Splitting, dst), data2(Deleted, dst))); + assertFalse(strategy.shouldKeepLeft(data(Splitting, src, dst), data2(Deleted, src, dst))); + + assertFalse(strategy.shouldKeepLeft(data(Deleted), data2(Init))); + assertTrue(strategy.shouldKeepLeft(data(Deleted), data2(Free))); + assertTrue(strategy.shouldKeepLeft(data(Deleted), data2(Assigning))); + assertTrue(strategy.shouldKeepLeft(data(Deleted), data2(Owned))); + assertTrue(strategy.shouldKeepLeft(data(Deleted), data2(Releasing))); + assertTrue(strategy.shouldKeepLeft(data(Deleted), data2(Splitting))); + assertTrue(strategy.shouldKeepLeft(data(Deleted), data2(Deleted))); - assertTrue(strategy.shouldKeepLeft(data(Init), data(Init))); - assertFalse(strategy.shouldKeepLeft(data(Init), data(Free))); - assertFalse(strategy.shouldKeepLeft(data(Init), data(Assigning))); - assertFalse(strategy.shouldKeepLeft(data(Init), data(Owned))); - assertFalse(strategy.shouldKeepLeft(data(Init), data(Releasing))); - assertFalse(strategy.shouldKeepLeft(data(Init), data(Splitting))); - assertFalse(strategy.shouldKeepLeft(data(Init), data(Deleted))); - - assertTrue(strategy.shouldKeepLeft(data(Assigning), data(Init))); - assertTrue(strategy.shouldKeepLeft(data(Assigning), data(Free))); - assertTrue(strategy.shouldKeepLeft(data(Assigning), data(Assigning))); - assertTrue(strategy.shouldKeepLeft(data(Assigning, "dst1"), data(Owned, "dst2"))); - assertTrue(strategy.shouldKeepLeft(data(Assigning, dst), data(Owned, src, dst))); - assertFalse(strategy.shouldKeepLeft(data(Assigning, dst), data(Owned, dst))); - assertTrue(strategy.shouldKeepLeft(data(Assigning, src, dst), data(Releasing, dst))); - assertTrue(strategy.shouldKeepLeft(data(Assigning, src, "dst1"), data(Releasing, src, "dst2"))); - assertTrue(strategy.shouldKeepLeft(data(Assigning, "src1", dst), data(Releasing, "src2", dst))); - assertFalse(strategy.shouldKeepLeft(data(Assigning, src, dst), data(Releasing, src, dst))); - assertTrue(strategy.shouldKeepLeft(data(Assigning), data(Splitting, dst))); - assertTrue(strategy.shouldKeepLeft(data(Assigning), data(Deleted, dst))); - - assertTrue(strategy.shouldKeepLeft(data(Owned), data(Init))); - assertTrue(strategy.shouldKeepLeft(data(Owned), data(Free))); - assertTrue(strategy.shouldKeepLeft(data(Owned, src, "dst1"), data(Assigning, src, "dst2"))); - assertTrue(strategy.shouldKeepLeft(data(Owned, src, dst), data(Assigning, dst))); - assertTrue(strategy.shouldKeepLeft(data(Owned, src, dst), data(Assigning, src, dst))); - assertTrue(strategy.shouldKeepLeft(data(Owned, src, dst), data(Assigning, dst, dst))); - assertFalse(strategy.shouldKeepLeft(data(Owned, src, dst), data(Assigning, dst, "dst1"))); - assertTrue(strategy.shouldKeepLeft(data(Owned), data(Owned))); - assertTrue(strategy.shouldKeepLeft(data(Owned), data(Releasing, dst))); - assertTrue(strategy.shouldKeepLeft(data(Owned, src, "dst1"), data(Releasing, src, "dst2"))); - assertTrue(strategy.shouldKeepLeft(data(Owned, "dst1"), data(Releasing, "dst2"))); - assertFalse(strategy.shouldKeepLeft(data(Owned, dst), data(Releasing, dst))); - assertFalse(strategy.shouldKeepLeft(data(Owned, src, dst), data(Releasing, dst))); - assertTrue(strategy.shouldKeepLeft(data(Owned, src, "dst1"), data(Splitting, src, "dst2"))); - assertTrue(strategy.shouldKeepLeft(data(Owned, "dst1"), data(Splitting, "dst2"))); - assertFalse(strategy.shouldKeepLeft(data(Owned, dst), data(Splitting, dst))); - assertFalse(strategy.shouldKeepLeft(data(Owned, src, dst), data(Splitting, dst))); - assertTrue(strategy.shouldKeepLeft(data(Owned), data(Deleted, dst))); - - assertTrue(strategy.shouldKeepLeft(data(Releasing), data(Init))); - assertFalse(strategy.shouldKeepLeft(data(Releasing), data(Free))); - assertTrue(strategy.shouldKeepLeft(data(Releasing, "dst1"), data(Free, "dst2"))); - assertTrue(strategy.shouldKeepLeft(data(Releasing, "src1", dst), data(Free, "src2", dst))); - assertTrue(strategy.shouldKeepLeft(data(Releasing), data(Assigning))); - assertTrue(strategy.shouldKeepLeft(data(Releasing, "dst1"), data(Owned, "dst2"))); - assertTrue(strategy.shouldKeepLeft(data(Releasing, src, "dst1"), data(Owned, src, "dst2"))); - assertTrue(strategy.shouldKeepLeft(data(Releasing, "src1", dst), data(Owned, "src2", dst))); - assertFalse(strategy.shouldKeepLeft(data(Releasing, src, dst), data(Owned, src, dst))); - assertTrue(strategy.shouldKeepLeft(data(Releasing), data(Releasing))); - assertTrue(strategy.shouldKeepLeft(data(Releasing), data(Splitting))); - assertTrue(strategy.shouldKeepLeft(data(Releasing), data(Deleted, dst))); - - assertTrue(strategy.shouldKeepLeft(data(Splitting), data(Init))); - assertTrue(strategy.shouldKeepLeft(data(Splitting), data(Free))); - assertTrue(strategy.shouldKeepLeft(data(Splitting), data(Assigning))); - assertTrue(strategy.shouldKeepLeft(data(Splitting), data(Owned))); - assertTrue(strategy.shouldKeepLeft(data(Splitting), data(Releasing))); - assertTrue(strategy.shouldKeepLeft(data(Splitting), data(Splitting))); - assertTrue(strategy.shouldKeepLeft(data(Splitting, src, "dst1"), data(Deleted, src, "dst2"))); - assertTrue(strategy.shouldKeepLeft(data(Splitting, "dst1"), data(Deleted, "dst2"))); - assertTrue(strategy.shouldKeepLeft(data(Splitting, "src1", dst), data(Deleted, "src2", dst))); - assertFalse(strategy.shouldKeepLeft(data(Splitting, dst), data(Deleted, dst))); - assertFalse(strategy.shouldKeepLeft(data(Splitting, src, dst), data(Deleted, src, dst))); - - assertFalse(strategy.shouldKeepLeft(data(Deleted), data(Init))); - assertTrue(strategy.shouldKeepLeft(data(Deleted), data(Free))); - assertTrue(strategy.shouldKeepLeft(data(Deleted), data(Assigning))); - assertTrue(strategy.shouldKeepLeft(data(Deleted), data(Owned))); - assertTrue(strategy.shouldKeepLeft(data(Deleted), data(Releasing))); - assertTrue(strategy.shouldKeepLeft(data(Deleted), data(Splitting))); - assertTrue(strategy.shouldKeepLeft(data(Deleted), data(Deleted))); - - assertFalse(strategy.shouldKeepLeft(data(Free), data(Init))); - assertTrue(strategy.shouldKeepLeft(data(Free), data(Free))); - assertFalse(strategy.shouldKeepLeft(data(Free), data(Assigning))); - assertTrue(strategy.shouldKeepLeft(data(Free), data(Assigning, src, dst))); - assertTrue(strategy.shouldKeepLeft(data(Free), data(Owned))); - assertTrue(strategy.shouldKeepLeft(data(Free), data(Releasing))); - assertTrue(strategy.shouldKeepLeft(data(Free), data(Splitting))); - assertTrue(strategy.shouldKeepLeft(data(Free), data(Deleted))); + assertFalse(strategy.shouldKeepLeft(data(Free), data2(Init))); + assertTrue(strategy.shouldKeepLeft(data(Free), data2(Free))); + assertFalse(strategy.shouldKeepLeft(data(Free), data2(Assigning))); + assertTrue(strategy.shouldKeepLeft(data(Free), data2(Assigning, src, dst))); + assertTrue(strategy.shouldKeepLeft(data(Free), data2(Owned))); + assertTrue(strategy.shouldKeepLeft(data(Free), data2(Releasing))); + assertTrue(strategy.shouldKeepLeft(data(Free), data2(Splitting))); + assertTrue(strategy.shouldKeepLeft(data(Free), data2(Deleted))); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataTest.java index 9617c8a8c2bd0..a48e2a4db8b37 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataTest.java @@ -33,15 +33,16 @@ public class ServiceUnitStateDataTest { @Test public void testConstructors() throws InterruptedException { - ServiceUnitStateData data1 = new ServiceUnitStateData(Owned, "A"); + ServiceUnitStateData data1 = new ServiceUnitStateData(Owned, "A", 1); assertEquals(data1.state(), Owned); assertEquals(data1.broker(), "A"); assertNull(data1.sourceBroker()); - assertThat(data1.timestamp()).isGreaterThan(0);; + assertThat(data1.timestamp()).isGreaterThan(0); + ; Thread.sleep(10); - ServiceUnitStateData data2 = new ServiceUnitStateData(Assigning, "A", "B"); + ServiceUnitStateData data2 = new ServiceUnitStateData(Assigning, "A", "B", 1); assertEquals(data2.state(), Assigning); assertEquals(data2.broker(), "A"); assertEquals(data2.sourceBroker(), "B"); @@ -50,23 +51,28 @@ public void testConstructors() throws InterruptedException { @Test(expectedExceptions = NullPointerException.class) public void testNullState() { - new ServiceUnitStateData(null, "A"); + new ServiceUnitStateData(null, "A", 1); } @Test(expectedExceptions = IllegalArgumentException.class) public void testNullBroker() { - new ServiceUnitStateData(Owned, null); + new ServiceUnitStateData(Owned, null, 1); } @Test(expectedExceptions = IllegalArgumentException.class) public void testEmptyBroker() { - new ServiceUnitStateData(Owned, ""); + new ServiceUnitStateData(Owned, "", 1); + } + + @Test + public void testZeroVersionId() { + new ServiceUnitStateData(Owned, "A", Long.MAX_VALUE + 1); } @Test public void jsonWriteAndReadTest() throws JsonProcessingException { ObjectMapper mapper = ObjectMapperFactory.create(); - final ServiceUnitStateData src = new ServiceUnitStateData(Assigning, "A", "B"); + final ServiceUnitStateData src = new ServiceUnitStateData(Assigning, "A", "B", 1); String json = mapper.writeValueAsString(src); ServiceUnitStateData dst = mapper.readValue(json, ServiceUnitStateData.class); assertEquals(dst, src); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java index 4c1d4f7d2a89d..543b7c629ac5f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java @@ -54,6 +54,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateCompactionStrategy; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData; import org.apache.pulsar.client.admin.PulsarAdminException; @@ -88,20 +89,24 @@ public class ServiceUnitStateCompactionTest extends MockedPulsarServiceBaseTest private ServiceUnitState testState = Init; + private ServiceUnitStateData testData = null; + private static Random RANDOM = new Random(); private ServiceUnitStateData testValue(ServiceUnitState state, String broker) { if (state == Init) { - return null; + testData = null; + } else { + testData = new ServiceUnitStateData(state, broker, versionId(testData) + 1); } - return new ServiceUnitStateData(state, broker); + + return testData; } private ServiceUnitStateData testValue(String broker) { - ServiceUnitState to = nextValidStateNonSplit(testState); - testState = to; - return testValue(to, broker); + testState = nextValidStateNonSplit(testState); + return testValue(testState, broker); } private ServiceUnitState nextValidState(ServiceUnitState from) { @@ -149,6 +154,7 @@ public void setup() throws Exception { strategy.checkBrokers(false); testState = Init; + testData = null; } @@ -202,13 +208,14 @@ TestData generateTestData() throws PulsarAdminException, PulsarClientException { ServiceUnitState state = invalid ? nextInvalidState(prevState) : nextValidState(prevState); ServiceUnitStateData value; + long versionId = versionId(prev) + 1; if (invalid) { - value = new ServiceUnitStateData(state, key + ":" + j, false); + value = new ServiceUnitStateData(state, key + ":" + j, false, versionId); } else { if (state == Init) { - value = new ServiceUnitStateData(state, key + ":" + j, true); + value = new ServiceUnitStateData(state, key + ":" + j, true, versionId); } else { - value = new ServiceUnitStateData(state, key + ":" + j, false); + value = new ServiceUnitStateData(state, key + ":" + j, false, versionId); } } @@ -560,15 +567,16 @@ public void testSlowTableviewAfterCompaction() throws Exception { String bundle = "bundle1"; String src = "broker0"; String dst = "broker1"; - producer.newMessage().key(bundle).value(new ServiceUnitStateData(Owned, src)).send(); + long versionId = 1; + producer.newMessage().key(bundle).value(new ServiceUnitStateData(Owned, src, versionId++)).send(); for (int i = 0; i < 3; i++) { - var assignedStateData = new ServiceUnitStateData(Assigning, dst, src); + var assignedStateData = new ServiceUnitStateData(Assigning, dst, src, versionId++); producer.newMessage().key(bundle).value(assignedStateData).send(); producer.newMessage().key(bundle).value(assignedStateData).send(); - var releasedStateData = new ServiceUnitStateData(Releasing, dst, src); + var releasedStateData = new ServiceUnitStateData(Releasing, dst, src, versionId++); producer.newMessage().key(bundle).value(releasedStateData).send(); producer.newMessage().key(bundle).value(releasedStateData).send(); - var ownedStateData = new ServiceUnitStateData(Owned, dst, src); + var ownedStateData = new ServiceUnitStateData(Owned, dst, src, versionId++); producer.newMessage().key(bundle).value(ownedStateData).send(); producer.newMessage().key(bundle).value(ownedStateData).send(); compactor.compact(topic, strategy).get(); @@ -945,4 +953,8 @@ public void testReadUnCompacted() assertNull(none); } } + + public static long versionId(ServiceUnitStateData data) { + return data == null ? ServiceUnitStateChannelImpl.VERSION_ID_INIT - 1 : data.versionId(); + } } From 579f22c8449be287ee1209a477aeaad346495289 Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Wed, 1 Mar 2023 01:02:11 -0800 Subject: [PATCH 071/174] [improve][broker] PIP-192 Added --extensions option in BrokerMonitor (#19654) --- .../pulsar/testclient/BrokerMonitor.java | 94 +++++++++++++++++-- 1 file changed, 88 insertions(+), 6 deletions(-) diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/BrokerMonitor.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/BrokerMonitor.java index c209c34a3d76e..3f8969860163d 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/BrokerMonitor.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/BrokerMonitor.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.testclient; +import static org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.BROKER_LOAD_DATA_STORE_TOPIC; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import com.beust.jcommander.ParameterException; @@ -31,7 +32,13 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SizeUnit; +import org.apache.pulsar.client.api.TableView; import org.apache.pulsar.policies.data.loadbalancer.LoadReport; import org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData; import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; @@ -54,7 +61,7 @@ public class BrokerMonitor { private static final String BROKER_ROOT = "/loadbalance/brokers"; private static final int ZOOKEEPER_TIMEOUT_MILLIS = 30000; private static final int GLOBAL_STATS_PRINT_PERIOD_MILLIS = 60000; - private final ZooKeeper zkClient; + private ZooKeeper zkClient; private static final Gson gson = new Gson(); // Fields common for message rows. @@ -77,7 +84,7 @@ public class BrokerMonitor { private static final Object[] ALLOC_MESSAGE_ROW = makeMessageRow("ALLOC MSG"); private static final Object[] GLOBAL_HEADER = { "BROKER", "BUNDLE", "MSG/S", "LONG/S", "KB/S", "MAX %" }; - private final Map loadData; + private Map loadData; private static final FixedColumnLengthTableMaker localTableMaker = new FixedColumnLengthTableMaker(); static { @@ -434,8 +441,11 @@ private static class Arguments { @Parameter(names = { "-h", "--help" }, description = "Help message", help = true) boolean help; - @Parameter(names = { "--connect-string" }, description = "Zookeeper connect string", required = true) + @Parameter(names = { "--connect-string" }, description = "Zookeeper or broker connect string", required = true) public String connectString = null; + + @Parameter(names = { "--extensions" }, description = "true to monitor Load Balance Extensions.") + boolean extensions = false; } /** @@ -464,6 +474,71 @@ public void start() { } } + private TableView brokerLoadDataTableView; + + private BrokerMonitor(String brokerServiceUrl) { + try { + PulsarClient client = PulsarClient.builder() + .memoryLimit(0, SizeUnit.BYTES) + .serviceUrl(brokerServiceUrl) + .connectionsPerBroker(4) + .ioThreads(Runtime.getRuntime().availableProcessors()) + .statsInterval(0, TimeUnit.SECONDS) + .build(); + this.brokerLoadDataTableView = client + .newTableView(Schema.JSON(BrokerLoadData.class)) + .topic(BROKER_LOAD_DATA_STORE_TOPIC).create(); + } catch (Throwable e) { + log.info("Failed to start BrokerMonitor", e); + throw new RuntimeException(e); + } + } + + private synchronized void printBrokerLoadData(final String broker, final BrokerLoadData brokerLoadData) { + + // Initialize the constant rows. + final Object[][] rows = new Object[6][]; + rows[0] = SYSTEM_ROW; + rows[2] = COUNT_ROW; + rows[4] = LATEST_ROW; + + // First column is a label, so start at the second column at index 1. + // System row. + rows[1] = new Object[SYSTEM_ROW.length]; + initRow(rows[1], brokerLoadData.getCpu().percentUsage(), brokerLoadData.getMemory().percentUsage(), + brokerLoadData.getDirectMemory().percentUsage(), brokerLoadData.getBandwidthIn().percentUsage(), + brokerLoadData.getBandwidthOut().percentUsage(), brokerLoadData.getMaxResourceUsage() * 100); + + // Count row. + rows[3] = new Object[COUNT_ROW.length]; + initRow(rows[3], null, brokerLoadData.getBundleCount(), + null, null, + null, null); + + // Latest message data row. + rows[5] = new Object[LATEST_ROW.length]; + initMessageRow(rows[5], brokerLoadData.getMsgRateIn(), brokerLoadData.getMsgRateOut(), + brokerLoadData.getMsgThroughputIn(), brokerLoadData.getMsgThroughputOut()); + + final String table = localTableMaker.make(rows); + log.info("\nBroker Data for {}:\n{}\n", broker, table); + } + + private synchronized void printBrokerLoadDataStore() { + brokerLoadDataTableView.forEach(this::printBrokerLoadData); + } + + private void startBrokerLoadDataStoreMonitor() { + try { + while (true) { + Thread.sleep(GLOBAL_STATS_PRINT_PERIOD_MILLIS); + printBrokerLoadDataStore(); + } + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + /** * Run a monitor from command line arguments. * @@ -481,8 +556,15 @@ public static void main(String[] args) throws Exception { jc.usage(); PerfClientUtils.exit(1); } - final ZooKeeper zkClient = new ZooKeeper(arguments.connectString, ZOOKEEPER_TIMEOUT_MILLIS, null); - final BrokerMonitor monitor = new BrokerMonitor(zkClient); - monitor.start(); + + + if (arguments.extensions) { + final BrokerMonitor monitor = new BrokerMonitor(arguments.connectString); + monitor.startBrokerLoadDataStoreMonitor(); + } else { + final ZooKeeper zkClient = new ZooKeeper(arguments.connectString, ZOOKEEPER_TIMEOUT_MILLIS, null); + final BrokerMonitor monitor = new BrokerMonitor(zkClient); + monitor.start(); + } } } From 7e0a6c48d23a3706182977c2e47eea211306ab5a Mon Sep 17 00:00:00 2001 From: feynmanlin <315157973@qq.com> Date: Wed, 1 Mar 2023 22:33:59 +0800 Subject: [PATCH 072/174] [fix][broker] Fixed history load not releasing (#19560) When a broker in the cluster is not active, its load still remain in the memory of the Leader. If deployed in a container, brokers will scale up and down frequently, and the memory of the Leader gradually increase --- .../loadbalance/LoadSheddingStrategy.java | 8 ++++++++ .../impl/ModularLoadManagerImpl.java | 3 +++ .../loadbalance/impl/ThresholdShedder.java | 9 +++++++-- .../loadbalance/impl/ThresholdShedderTest.java | 18 ++++++++++++++++++ 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadSheddingStrategy.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadSheddingStrategy.java index ee136d6e9b851..3cad4b74d9903 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadSheddingStrategy.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadSheddingStrategy.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.loadbalance; import com.google.common.collect.Multimap; +import java.util.Set; import org.apache.pulsar.broker.ServiceConfiguration; /** @@ -36,4 +37,11 @@ public interface LoadSheddingStrategy { * @return A map from all selected bundles to the brokers on which they reside. */ Multimap findBundlesForUnloading(LoadData loadData, ServiceConfiguration conf); + + /** + * Triggered when active broker changes. + * + * @param activeBrokers active Brokers + */ + default void onActiveBrokersChange(Set activeBrokers) {} } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java index c1afe6007f6bb..ea2472eb199d4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java @@ -491,6 +491,9 @@ private void cleanupDeadBrokersData() { if (pulsar.getLeaderElectionService() != null && pulsar.getLeaderElectionService().isLeader()) { deadBrokers.forEach(this::deleteTimeAverageDataFromMetadataStoreAsync); + for (LoadSheddingStrategy loadSheddingStrategy : loadSheddingPipeline) { + loadSheddingStrategy.onActiveBrokersChange(activeBrokers); + } } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedder.java index e2f1a7808fe91..86df49f952674 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedder.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedder.java @@ -22,6 +22,7 @@ import com.google.common.collect.Multimap; import java.util.HashMap; import java.util.Map; +import java.util.Set; import org.apache.commons.lang3.mutable.MutableBoolean; import org.apache.commons.lang3.mutable.MutableDouble; import org.apache.commons.lang3.tuple.Pair; @@ -59,7 +60,8 @@ public class ThresholdShedder implements LoadSheddingStrategy { private final Map brokerAvgResourceUsage = new HashMap<>(); @Override - public Multimap findBundlesForUnloading(final LoadData loadData, final ServiceConfiguration conf) { + public synchronized Multimap findBundlesForUnloading(final LoadData loadData, + final ServiceConfiguration conf) { selectedBundlesCache.clear(); final double threshold = conf.getLoadBalancerBrokerThresholdShedderPercentage() / 100.0; final Map recentlyUnloadedBundles = loadData.getRecentlyUnloadedBundles(); @@ -75,7 +77,6 @@ public Multimap findBundlesForUnloading(final LoadData loadData, loadData.getBrokerData().forEach((broker, brokerData) -> { final LocalBrokerData localData = brokerData.getLocalData(); final double currentUsage = brokerAvgResourceUsage.getOrDefault(broker, 0.0); - if (currentUsage < avgUsage + threshold) { if (log.isDebugEnabled()) { log.debug("[{}] broker is not overloaded, ignoring at this point", broker); @@ -233,5 +234,9 @@ private Pair getMaxUsageBroker( } return Pair.of(hasBrokerBelowLowerBound, maxUsageBrokerName); } + @Override + public synchronized void onActiveBrokersChange(Set newBrokers) { + brokerAvgResourceUsage.keySet().removeIf((key) -> !newBrokers.contains(key)); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedderTest.java index 4747471eab09f..16a15f5a7c7f5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ThresholdShedderTest.java @@ -23,7 +23,10 @@ import static org.testng.Assert.assertTrue; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; +import java.lang.reflect.Field; +import java.util.HashSet; import java.util.List; +import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.LoadData; @@ -57,6 +60,21 @@ public void testNoBrokers() { assertTrue(thresholdShedder.findBundlesForUnloading(loadData, conf).isEmpty()); } + @Test + public void testCleanCache() throws Exception { + testBrokerReachThreshold(); + Field field = ThresholdShedder.class.getDeclaredField("brokerAvgResourceUsage"); + field.setAccessible(true); + Map map = (Map) field.get(thresholdShedder); + assertFalse(map.isEmpty()); + HashSet activeBrokers = new HashSet<>(); + activeBrokers.add("leader"); + thresholdShedder.onActiveBrokersChange(activeBrokers); + thresholdShedder.findBundlesForUnloading(new LoadData(), conf); + map = (Map) field.get(thresholdShedder); + assertTrue(map.isEmpty()); + } + @Test public void testBrokersWithNoBundles() { LoadData loadData = new LoadData(); From 6621fd3451a52bb7447ebeb845c83ac2f310b718 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Wed, 1 Mar 2023 11:45:26 -0600 Subject: [PATCH 073/174] [improve] Simplify enabling Broker, WS Proxy hostname verification (#19674) ### Motivation When we merged #15818 in order to make the broker's client configurable, we did not add an explicit config for hostname verification. This PR adds that config to the broker and the websocket proxy. I chose the name `tlsHostnameVerificationEnabled` because that is what is already used in the proxy. It diverges from the function worker's config of `tlsEnableHostnameVerification`. Before this PR, you would have enabled hostname verification by configuring `brokerClient_tlsHostnameVerificationEnable=true` in the broker and WS proxy configs. (Note that the variable name is slightly different because the `ClientConfiguration` does not have a `d` at the end of its name. The remaining follow up work will be to update the `ClusterData` objects to configure hostname verification there to make it easier to configure hostname verification for remote clusters. ### Modifications * Add `tlsHostnameVerificationEnabled` to the `broker.conf` and the `proxy.conf` * Update all of the relevant locations that were previously only relying on `brokerClient_tlsHostnameVerificationEnable` ### Verifying this change I added a single test to ensure that the `WebSocketProxyConfiguration` properly converts to the `ServiceConfiguration` object. Otherwise, I verified that anywhere we are using `"brokerClient_"`, this PR also adds the right configuration. ### Does this pull request potentially affect one of the following parts: This PR introduces a "new" configuration key, but not a new concept. All underlying behaviors are unchanged. ### Documentation - [x] `doc-not-needed` Docs are automatically updated by these changes. --- conf/broker.conf | 6 +++++ conf/websocket.conf | 3 +++ .../pulsar/broker/ServiceConfiguration.java | 5 ++++ .../apache/pulsar/broker/PulsarService.java | 6 +++-- .../broker/namespace/NamespaceService.java | 3 ++- .../pulsar/broker/service/BrokerService.java | 24 ++++++++++++------- .../pulsar/compaction/CompactorTool.java | 3 ++- .../pulsar/websocket/WebSocketService.java | 1 + .../service/WebSocketProxyConfiguration.java | 6 ++++- .../WebSocketProxyConfigurationTest.java | 16 +++++++++++++ 10 files changed, 60 insertions(+), 13 deletions(-) diff --git a/conf/broker.conf b/conf/broker.conf index 3183c83a837a8..4b7c108be5f11 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -680,6 +680,9 @@ tlsTrustCertsFilePath= # though the cert will not be used for client authentication. tlsAllowInsecureConnection=false +# Whether the hostname is validated when the broker creates a TLS connection with other brokers +tlsHostnameVerificationEnabled=false + # Specify the tls protocols the broker will use to negotiate during TLS handshake # (a comma-separated list of protocol names). # Examples: @@ -1055,6 +1058,9 @@ bookkeeperTLSTrustCertsFilePath= # Tls cert refresh duration at bookKeeper-client in seconds (0 to disable check) bookkeeperTlsCertFilesRefreshDurationSeconds=300 +# Whether the hostname is validated when the broker creates a TLS connection to a bookkeeper +bookkeeper_tlsHostnameVerificationEnabled=false + # Enable/disable disk weight based placement. Default is false bookkeeperDiskWeightBasedPlacementEnabled=false diff --git a/conf/websocket.conf b/conf/websocket.conf index a966f05c71935..2e2824a838c6f 100644 --- a/conf/websocket.conf +++ b/conf/websocket.conf @@ -108,6 +108,9 @@ brokerClientAuthenticationPlugin= brokerClientAuthenticationParameters= brokerClientTrustCertsFilePath= +# Whether the hostname is validated when connecting to the broker. +tlsHostnameVerificationEnabled=false + # You can add extra configuration options for the Pulsar Client # by prefixing them with "brokerClient_". These configurations are applied after hard coded configuration # and before the above brokerClient configurations named above. diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index c18367f265506..ec5b0d4042bcd 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -1490,6 +1490,11 @@ The delayed message index bucket time step(in seconds) in per bucket snapshot se doc = "Accept untrusted TLS certificate from client" ) private boolean tlsAllowInsecureConnection = false; + @FieldContext( + category = CATEGORY_TLS, + doc = "Whether the hostname is validated when the broker creates a TLS connection with other brokers" + ) + private boolean tlsHostnameVerificationEnabled = false; @FieldContext( category = CATEGORY_TLS, doc = "Specify the tls protocols the broker will use to negotiate during TLS Handshake.\n\n" diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index ae0fb1a9f283c..4af94c339c82f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -1551,6 +1551,7 @@ public synchronized PulsarClient getClient() throws PulsarServerException { conf.setTlsCiphers(this.getConfiguration().getBrokerClientTlsCiphers()); conf.setTlsProtocols(this.getConfiguration().getBrokerClientTlsProtocols()); conf.setTlsAllowInsecureConnection(this.getConfiguration().isTlsAllowInsecureConnection()); + conf.setTlsHostnameVerificationEnable(this.getConfiguration().isTlsHostnameVerificationEnabled()); if (this.getConfiguration().isBrokerClientTlsEnabledWithKeyStore()) { conf.setUseKeyStoreTls(true); conf.setTlsTrustStoreType(this.getConfiguration().getBrokerClientTlsTrustStoreType()); @@ -1622,7 +1623,8 @@ public synchronized PulsarAdmin getAdminClient() throws PulsarServerException { .tlsKeyFilePath(conf.getBrokerClientKeyFilePath()) .tlsCertificateFilePath(conf.getBrokerClientCertificateFilePath()); } - builder.allowTlsInsecureConnection(conf.isTlsAllowInsecureConnection()); + builder.allowTlsInsecureConnection(conf.isTlsAllowInsecureConnection()) + .enableTlsHostnameVerification(conf.isTlsHostnameVerificationEnabled()); } // most of the admin request requires to make zk-call so, keep the max read-timeout based on @@ -1849,7 +1851,7 @@ public static WorkerConfig initializeWorkerConfigFromBrokerConfig(ServiceConfigu workerConfig.setMetadataStoreCacheExpirySeconds(brokerConfig.getMetadataStoreCacheExpirySeconds()); workerConfig.setTlsAllowInsecureConnection(brokerConfig.isTlsAllowInsecureConnection()); workerConfig.setTlsEnabled(brokerConfig.isTlsEnabled()); - workerConfig.setTlsEnableHostnameVerification(false); + workerConfig.setTlsEnableHostnameVerification(brokerConfig.isTlsHostnameVerificationEnabled()); workerConfig.setBrokerClientTrustCertsFilePath(brokerConfig.getTlsTrustCertsFilePath()); // client in worker will use this config to authenticate with broker diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index 245c3f896af1f..33b15926c3c33 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -1420,7 +1420,8 @@ public PulsarClientImpl getNamespaceClient(ClusterDataImpl cluster) { ? cluster.getBrokerServiceUrlTls() : cluster.getServiceUrlTls()) .enableTls(true) .tlsTrustCertsFilePath(pulsar.getConfiguration().getBrokerClientTrustCertsFilePath()) - .allowTlsInsecureConnection(pulsar.getConfiguration().isTlsAllowInsecureConnection()); + .allowTlsInsecureConnection(pulsar.getConfiguration().isTlsAllowInsecureConnection()) + .enableTlsHostnameVerification(pulsar.getConfiguration().isTlsHostnameVerificationEnabled()); } else { clientBuilder.serviceUrl(isNotBlank(cluster.getBrokerServiceUrl()) ? cluster.getBrokerServiceUrl() : cluster.getServiceUrl()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index b9b63427b370e..15d48e59849b6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -1307,7 +1307,8 @@ public PulsarClient getReplicationClient(String cluster, Optional c data.getBrokerClientTlsKeyStorePassword(), data.getBrokerClientTrustCertsFilePath(), data.getBrokerClientKeyFilePath(), - data.getBrokerClientCertificateFilePath() + data.getBrokerClientCertificateFilePath(), + pulsar.getConfiguration().isTlsHostnameVerificationEnabled() ); } else if (pulsar.getConfiguration().isBrokerClientTlsEnabled()) { configTlsSettings(clientBuilder, serviceUrlTls, @@ -1321,7 +1322,8 @@ public PulsarClient getReplicationClient(String cluster, Optional c pulsar.getConfiguration().getBrokerClientTlsKeyStorePassword(), pulsar.getConfiguration().getBrokerClientTrustCertsFilePath(), pulsar.getConfiguration().getBrokerClientKeyFilePath(), - pulsar.getConfiguration().getBrokerClientCertificateFilePath() + pulsar.getConfiguration().getBrokerClientCertificateFilePath(), + pulsar.getConfiguration().isTlsHostnameVerificationEnabled() ); } else { clientBuilder.serviceUrl( @@ -1351,10 +1353,12 @@ private void configTlsSettings(ClientBuilder clientBuilder, String serviceUrl, String brokerClientTlsTrustStorePassword, String brokerClientTlsKeyStoreType, String brokerClientTlsKeyStore, String brokerClientTlsKeyStorePassword, String brokerClientTrustCertsFilePath, - String brokerClientKeyFilePath, String brokerClientCertificateFilePath) { + String brokerClientKeyFilePath, String brokerClientCertificateFilePath, + boolean isTlsHostnameVerificationEnabled) { clientBuilder .serviceUrl(serviceUrl) - .allowTlsInsecureConnection(isTlsAllowInsecureConnection); + .allowTlsInsecureConnection(isTlsAllowInsecureConnection) + .enableTlsHostnameVerification(isTlsHostnameVerificationEnabled); if (brokerClientTlsEnabledWithKeyStore) { clientBuilder.useKeyStoreTls(true) .tlsTrustStoreType(brokerClientTlsTrustStoreType) @@ -1376,7 +1380,8 @@ private void configAdminTlsSettings(PulsarAdminBuilder adminBuilder, boolean bro String brokerClientTlsTrustStorePassword, String brokerClientTlsKeyStoreType, String brokerClientTlsKeyStore, String brokerClientTlsKeyStorePassword, String brokerClientTrustCertsFilePath, - String brokerClientKeyFilePath, String brokerClientCertificateFilePath) { + String brokerClientKeyFilePath, String brokerClientCertificateFilePath, + boolean isTlsHostnameVerificationEnabled) { if (brokerClientTlsEnabledWithKeyStore) { adminBuilder.useKeyStoreTls(true) .tlsTrustStoreType(brokerClientTlsTrustStoreType) @@ -1390,7 +1395,8 @@ private void configAdminTlsSettings(PulsarAdminBuilder adminBuilder, boolean bro .tlsKeyFilePath(brokerClientKeyFilePath) .tlsCertificateFilePath(brokerClientCertificateFilePath); } - adminBuilder.allowTlsInsecureConnection(isTlsAllowInsecureConnection); + adminBuilder.allowTlsInsecureConnection(isTlsAllowInsecureConnection) + .enableTlsHostnameVerification(isTlsHostnameVerificationEnabled); } public PulsarAdmin getClusterPulsarAdmin(String cluster, Optional clusterDataOp) { @@ -1438,7 +1444,8 @@ public PulsarAdmin getClusterPulsarAdmin(String cluster, Optional c data.getBrokerClientTlsKeyStorePassword(), data.getBrokerClientTrustCertsFilePath(), data.getBrokerClientKeyFilePath(), - data.getBrokerClientCertificateFilePath() + data.getBrokerClientCertificateFilePath(), + pulsar.getConfiguration().isTlsHostnameVerificationEnabled() ); } else if (conf.isBrokerClientTlsEnabled()) { configAdminTlsSettings(builder, @@ -1452,7 +1459,8 @@ public PulsarAdmin getClusterPulsarAdmin(String cluster, Optional c conf.getBrokerClientTlsKeyStorePassword(), conf.getBrokerClientTrustCertsFilePath(), conf.getBrokerClientKeyFilePath(), - conf.getBrokerClientCertificateFilePath() + conf.getBrokerClientCertificateFilePath(), + pulsar.getConfiguration().isTlsHostnameVerificationEnabled() ); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java index c359823549fcd..2539c306500a2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java @@ -82,7 +82,8 @@ public static PulsarClient createClient(ServiceConfiguration brokerConfig) throw AdvertisedListener internalListener = ServiceConfigurationUtils.getInternalListener(brokerConfig, "pulsar+ssl"); if (internalListener.getBrokerServiceUrlTls() != null && brokerConfig.isBrokerClientTlsEnabled()) { clientBuilder.serviceUrl(internalListener.getBrokerServiceUrlTls().toString()) - .allowTlsInsecureConnection(brokerConfig.isTlsAllowInsecureConnection()); + .allowTlsInsecureConnection(brokerConfig.isTlsAllowInsecureConnection()) + .enableTlsHostnameVerification(brokerConfig.isTlsHostnameVerificationEnabled()); if (brokerConfig.isBrokerClientTlsEnabledWithKeyStore()) { clientBuilder.useKeyStoreTls(true) .tlsKeyStoreType(brokerConfig.getBrokerClientTlsKeyStoreType()) diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/WebSocketService.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/WebSocketService.java index 9a8653029ce4c..66b2a0075ec2d 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/WebSocketService.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/WebSocketService.java @@ -199,6 +199,7 @@ private PulsarClient createClientInstance(ClusterData clusterData) throws IOExce .statsInterval(0, TimeUnit.SECONDS) // .enableTls(config.isTlsEnabled()) // .allowTlsInsecureConnection(config.isTlsAllowInsecureConnection()) // + .enableTlsHostnameVerification(config.isTlsHostnameVerificationEnabled()) .tlsTrustCertsFilePath(config.getBrokerClientTrustCertsFilePath()) // .ioThreads(config.getWebSocketNumIoThreads()) // .connectionsPerBroker(config.getWebSocketConnectionsPerBroker()); diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketProxyConfiguration.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketProxyConfiguration.java index 7dba9e719ad5a..7acfd4a64ad35 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketProxyConfiguration.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketProxyConfiguration.java @@ -129,6 +129,10 @@ public class WebSocketProxyConfiguration implements PulsarConfiguration { @FieldContext(doc = "Path for the trusted TLS certificate file for outgoing connection to a server (broker)") private String brokerClientTrustCertsFilePath = ""; + // Note: name matches the ServiceConfiguration name to ensure correct mapping + @FieldContext(doc = "Enable TLS hostname verification when connecting to broker") + private boolean tlsHostnameVerificationEnabled = false; + @FieldContext(doc = "Number of IO threads in Pulsar client used in WebSocket proxy") private int webSocketNumIoThreads = Runtime.getRuntime().availableProcessors(); @@ -183,7 +187,7 @@ public class WebSocketProxyConfiguration implements PulsarConfiguration { @FieldContext(doc = "Path for the trusted TLS certificate file") private String tlsTrustCertsFilePath = ""; - @FieldContext(doc = "Accept untrusted TLS certificate from client") + @FieldContext(doc = "Accept untrusted TLS certificate from client and broker") private boolean tlsAllowInsecureConnection = false; @FieldContext(doc = "Specify whether client certificates are required for " diff --git a/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/WebSocketProxyConfigurationTest.java b/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/WebSocketProxyConfigurationTest.java index e3a303003b841..92b4238cddd7d 100644 --- a/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/WebSocketProxyConfigurationTest.java +++ b/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/WebSocketProxyConfigurationTest.java @@ -19,9 +19,11 @@ package org.apache.pulsar.websocket; +import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; import org.apache.pulsar.websocket.service.WebSocketProxyConfiguration; import org.testng.annotations.Test; +import org.testng.Assert; import java.io.File; import java.io.FileInputStream; @@ -87,4 +89,18 @@ public void testBackwardCompatibility() throws IOException { assertEquals(serviceConfig.getMetadataStoreSessionTimeoutMillis(), 100); assertEquals(serviceConfig.getMetadataStoreCacheExpirySeconds(), 300); } + + @Test + public void testConfigurationConversionUsedByWebSocketProxyStarter() { + WebSocketProxyConfiguration config = new WebSocketProxyConfiguration(); + // Use non-default values for testing + config.setTlsAllowInsecureConnection(true); + Assert.assertFalse(config.isTlsHostnameVerificationEnabled(), "Update me when default changes."); + config.setTlsHostnameVerificationEnabled(true); + ServiceConfiguration brokerConf = PulsarConfigurationLoader.convertFrom(config); + Assert.assertTrue(brokerConf.isTlsAllowInsecureConnection(), + "TlsAllowInsecureConnection should convert correctly"); + Assert.assertTrue(brokerConf.isTlsHostnameVerificationEnabled(), + "TlsHostnameVerificationEnabled should convert correctly"); + } } From 5122f7721b236e763c72193fabc064ace8ab7399 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Wed, 1 Mar 2023 19:31:56 +0100 Subject: [PATCH 074/174] [fix][broker] Fix LedgerOffloaderStatsImpl singleton close method (#19666) --- .../impl/LedgerOffloaderStatsImpl.java | 6 +++-- .../stats/LedgerOffloaderMetricsTest.java | 23 +++++++++---------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/LedgerOffloaderStatsImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/LedgerOffloaderStatsImpl.java index be2895bf81867..5e05e4c8137cd 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/LedgerOffloaderStatsImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/LedgerOffloaderStatsImpl.java @@ -276,17 +276,19 @@ public void run() { } @Override - public void close() throws Exception { + public synchronized void close() throws Exception { if (instance == this && this.closed.compareAndSet(false, true)) { CollectorRegistry.defaultRegistry.unregister(this.offloadError); CollectorRegistry.defaultRegistry.unregister(this.offloadRate); CollectorRegistry.defaultRegistry.unregister(this.readLedgerLatency); CollectorRegistry.defaultRegistry.unregister(this.writeStorageError); CollectorRegistry.defaultRegistry.unregister(this.readOffloadError); + CollectorRegistry.defaultRegistry.unregister(this.readOffloadBytes); CollectorRegistry.defaultRegistry.unregister(this.readOffloadRate); CollectorRegistry.defaultRegistry.unregister(this.readOffloadIndexLatency); CollectorRegistry.defaultRegistry.unregister(this.readOffloadDataLatency); - this.offloadAndReadOffloadBytesMap.clear(); + CollectorRegistry.defaultRegistry.unregister(this.deleteOffloadOps); + instance = null; } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/LedgerOffloaderMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/LedgerOffloaderMetricsTest.java index a285974f13dcc..effb12a548e3b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/LedgerOffloaderMetricsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/LedgerOffloaderMetricsTest.java @@ -19,7 +19,6 @@ package org.apache.pulsar.broker.stats; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertTrue; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; @@ -32,7 +31,7 @@ import org.testng.annotations.AfterMethod; import org.testng.annotations.Test; -public class LedgerOffloaderMetricsTest extends BrokerTestBase { +public class LedgerOffloaderMetricsTest extends BrokerTestBase { @Override protected void setup() throws Exception { @@ -60,7 +59,7 @@ public void testTopicLevelMetrics() throws Exception { String ns1 = "prop/ns-abc1"; admin.namespaces().createNamespace(ns1); - String []topics = new String[3]; + String[] topics = new String[3]; LedgerOffloaderStatsImpl offloaderStats = (LedgerOffloaderStatsImpl) pulsar.getOffloaderStats(); for (int i = 0; i < 3; i++) { @@ -80,10 +79,10 @@ public void testTopicLevelMetrics() throws Exception { for (String topicName : topics) { assertEquals(offloaderStats.getOffloadError(topicName), 2); - assertEquals(offloaderStats.getOffloadBytes(topicName) , 100); + assertEquals(offloaderStats.getOffloadBytes(topicName), 100); assertEquals((long) offloaderStats.getReadLedgerLatency(topicName).sum, 1); assertEquals(offloaderStats.getReadOffloadError(topicName), 2); - assertEquals((long) offloaderStats.getReadOffloadIndexLatency(topicName).sum ,1000); + assertEquals((long) offloaderStats.getReadOffloadIndexLatency(topicName).sum, 1000); assertEquals(offloaderStats.getReadOffloadBytes(topicName), 100000); assertEquals(offloaderStats.getWriteStorageError(topicName), 2); } @@ -126,13 +125,13 @@ public void testNamespaceLevelMetrics() throws Exception { List topics = entry.getValue(); String topicName = topics.get(0); - assertTrue(offloaderStats.getOffloadError(topicName) >= 1); - assertTrue(offloaderStats.getOffloadBytes(topicName) >= 100); - assertTrue((long) offloaderStats.getReadLedgerLatency(topicName).sum >= 1); - assertTrue(offloaderStats.getReadOffloadError(topicName) >= 1); - assertTrue((long) offloaderStats.getReadOffloadIndexLatency(topicName).sum >= 1000); - assertTrue(offloaderStats.getReadOffloadBytes(topicName) >= 100000); - assertTrue(offloaderStats.getWriteStorageError(topicName) >= 1); + assertEquals(offloaderStats.getOffloadError(topicName), 6); + assertEquals(offloaderStats.getOffloadBytes(topicName), 600); + assertEquals((long) offloaderStats.getReadLedgerLatency(topicName).sum, 6); + assertEquals(offloaderStats.getReadOffloadError(topicName), 6); + assertEquals((long) offloaderStats.getReadOffloadIndexLatency(topicName).sum, 6000); + assertEquals(offloaderStats.getReadOffloadBytes(topicName), 600000); + assertEquals(offloaderStats.getWriteStorageError(topicName), 6); } } From 7f37670fc96e4b629234c242257fe1d42480e5cd Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Wed, 1 Mar 2023 13:36:12 -0600 Subject: [PATCH 075/174] [fix][client] Fix typo in Java doc for ClientConfigurationData (#19672) ### Motivation A minor typo fix. ### Modifications Update `ClientConfigurationData` by replacing `proxy` with `client`. ### Documentation - [x] `doc` This is a doc change. --- .../apache/pulsar/client/impl/conf/ClientConfigurationData.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ClientConfigurationData.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ClientConfigurationData.java index 481bdc99f9a99..1e7bc6f8221cf 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ClientConfigurationData.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ClientConfigurationData.java @@ -170,7 +170,7 @@ public class ClientConfigurationData implements Serializable, Cloneable { @ApiModelProperty( name = "tlsHostnameVerificationEnable", - value = "Whether the hostname is validated when the proxy creates a TLS connection with brokers." + value = "Whether the hostname is validated when the client creates a TLS connection with brokers." ) private boolean tlsHostnameVerificationEnable = false; @ApiModelProperty( From 5e717c68e061562fd44f2c44c4a41667abaf4df8 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Thu, 2 Mar 2023 09:48:22 +0800 Subject: [PATCH 076/174] [improve][broker] Make BucketDelayedDeliveryTracker can retry snapshot operation & improve logs (#19577) --- .../bookkeeper/mledger/util/Futures.java | 4 +- .../BookkeeperBucketSnapshotStorage.java | 12 +- .../pulsar/broker/delayed/bucket/Bucket.java | 25 +- .../bucket/BucketDelayedDeliveryTracker.java | 220 ++++++++++++------ .../delayed/bucket/ImmutableBucket.java | 152 +++++++----- .../broker/delayed/bucket/MutableBucket.java | 20 +- .../delayed/MockBucketSnapshotStorage.java | 41 ++++ .../broker/delayed/MockManagedCursor.java | 2 +- .../BucketDelayedDeliveryTrackerTest.java | 117 +++++++++- 9 files changed, 433 insertions(+), 160 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/Futures.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/Futures.java index dc1d1eb6c9ac5..f5ad77a71d8c4 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/Futures.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/Futures.java @@ -24,6 +24,7 @@ import java.util.function.Supplier; import org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback; import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.pulsar.common.util.FutureUtil; /** * Conveniences to use with {@link CompletableFuture}. @@ -78,7 +79,8 @@ public static CompletableFuture executeWithRetry(Supplier 0) { + Throwable throwable = FutureUtil.unwrapCompletionException(ex); + if (needRetryExceptionClass.isAssignableFrom(throwable.getClass()) && maxRetryTimes > 0) { executeWithRetry(op, needRetryExceptionClass, maxRetryTimes - 1).whenComplete((res2, ex2) -> { if (ex2 == null) { resultFuture.complete(res2); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java index 1cadc6d98e268..7dd6266e2115c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java @@ -154,7 +154,7 @@ private CompletableFuture createLedger(String bucketKey) { LedgerPassword, (rc, handle, ctx) -> { if (rc != BKException.Code.OK) { - future.completeExceptionally(bkException("Failed to create ledger", rc, -1)); + future.completeExceptionally(bkException("Create ledger", rc, -1)); } else { future.complete(handle); } @@ -170,7 +170,7 @@ private CompletableFuture openLedger(Long ledgerId) { LedgerPassword, (rc, handle, ctx) -> { if (rc != BKException.Code.OK) { - future.completeExceptionally(bkException("Failed to open ledger", rc, ledgerId)); + future.completeExceptionally(bkException("Open ledger", rc, ledgerId)); } else { future.complete(handle); } @@ -184,7 +184,7 @@ private CompletableFuture closeLedger(LedgerHandle ledgerHandle) { ledgerHandle.asyncClose((rc, handle, ctx) -> { if (rc != BKException.Code.OK) { log.warn("Failed to close a Ledger Handle: {}", ledgerHandle.getId()); - future.completeExceptionally(bkException("Failed to close ledger", rc, ledgerHandle.getId())); + future.completeExceptionally(bkException("Close ledger", rc, ledgerHandle.getId())); } else { future.complete(null); } @@ -197,7 +197,7 @@ private CompletableFuture addEntry(LedgerHandle ledgerHandle, byte[] data) ledgerHandle.asyncAddEntry(data, (rc, handle, entryId, ctx) -> { if (rc != BKException.Code.OK) { - future.completeExceptionally(bkException("Failed to add entry", rc, ledgerHandle.getId())); + future.completeExceptionally(bkException("Add entry", rc, ledgerHandle.getId())); } else { future.complete(null); } @@ -217,7 +217,7 @@ CompletableFuture> getLedgerEntryThenCloseLedger(Ledger ledger.asyncReadEntries(firstEntryId, lastEntryId, (rc, handle, entries, ctx) -> { if (rc != BKException.Code.OK) { - future.completeExceptionally(bkException("Failed to read entry", rc, ledger.getId())); + future.completeExceptionally(bkException("Read entry", rc, ledger.getId())); } else { future.complete(entries); } @@ -231,7 +231,7 @@ private CompletableFuture deleteLedger(long ledgerId) { CompletableFuture future = new CompletableFuture<>(); bookKeeper.asyncDeleteLedger(ledgerId, (int rc, Object cnx) -> { if (rc != BKException.Code.OK) { - future.completeExceptionally(bkException("Failed to delete ledger", rc, ledgerId)); + future.completeExceptionally(bkException("Delete ledger", rc, ledgerId)); } else { future.complete(null); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java index 5d2a556337a6e..50b5cd12ead07 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java @@ -42,6 +42,8 @@ abstract class Bucket { static final String DELIMITER = "_"; static final int MaxRetryTimes = 3; + protected final String dispatcherName; + protected final ManagedCursor cursor; protected final BucketSnapshotStorage bucketSnapshotStorage; @@ -54,17 +56,18 @@ abstract class Bucket { int lastSegmentEntryId; - int currentSegmentEntryId; + volatile int currentSegmentEntryId; - long snapshotLength; + volatile long snapshotLength; private volatile Long bucketId; private volatile CompletableFuture snapshotCreateFuture; - Bucket(ManagedCursor cursor, BucketSnapshotStorage storage, long startLedgerId, long endLedgerId) { - this(cursor, storage, startLedgerId, endLedgerId, new HashMap<>(), -1, -1, 0, 0, null, null); + Bucket(String dispatcherName, ManagedCursor cursor, + BucketSnapshotStorage storage, long startLedgerId, long endLedgerId) { + this(dispatcherName, cursor, storage, startLedgerId, endLedgerId, new HashMap<>(), -1, -1, 0, 0, null, null); } boolean containsMessage(long ledgerId, long entryId) { @@ -126,13 +129,19 @@ CompletableFuture asyncSaveBucketSnapshot( ImmutableBucket bucket, DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata, List bucketSnapshotSegments) { final String bucketKey = bucket.bucketKey(); - return bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata, bucketSnapshotSegments, bucketKey) - .thenCompose(newBucketId -> { + return executeWithRetry( + () -> bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata, bucketSnapshotSegments, bucketKey) + .whenComplete((__, ex) -> { + if (ex != null) { + log.warn("[{}] Failed to create bucket snapshot, bucketKey: {}", + dispatcherName, bucketKey, ex); + } + }), BucketSnapshotPersistenceException.class, MaxRetryTimes).thenCompose(newBucketId -> { bucket.setBucketId(newBucketId); return putBucketKeyId(bucketKey, newBucketId).exceptionally(ex -> { - log.warn("Failed to record bucketId to cursor property, bucketKey: {}, bucketId: {}", - bucketKey, bucketId); + log.warn("[{}] Failed to record bucketId to cursor property, bucketKey: {}, bucketId: {}", + dispatcherName, bucketKey, newBucketId, ex); return null; }).thenApply(__ -> newBucketId); }); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index bd4ef92cc7320..a77b272297be4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -21,6 +21,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static org.apache.pulsar.broker.delayed.bucket.Bucket.DELAYED_BUCKET_KEY_PREFIX; import static org.apache.pulsar.broker.delayed.bucket.Bucket.DELIMITER; +import static org.apache.pulsar.broker.delayed.bucket.Bucket.MaxRetryTimes; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.HashBasedTable; import com.google.common.collect.Range; @@ -61,7 +62,9 @@ @ThreadSafe public class BucketDelayedDeliveryTracker extends AbstractDelayedDeliveryTracker { - static final int AsyncOperationTimeoutSeconds = 30; + static final CompletableFuture NULL_LONG_PROMISE = CompletableFuture.completedFuture(null); + + static final int AsyncOperationTimeoutSeconds = 60; private final long minIndexCountPerBucket; @@ -104,20 +107,19 @@ public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispat this.sharedBucketPriorityQueue = new TripleLongPriorityQueue(); this.immutableBuckets = TreeRangeMap.create(); this.snapshotSegmentLastIndexTable = HashBasedTable.create(); - ManagedCursor cursor = dispatcher.getCursor(); - this.lastMutableBucket = new MutableBucket(cursor, bucketSnapshotStorage); + this.lastMutableBucket = new MutableBucket(dispatcher.getName(), dispatcher.getCursor(), bucketSnapshotStorage); this.numberDelayedMessages = recoverBucketSnapshot(); } private synchronized long recoverBucketSnapshot() throws RuntimeException { - ManagedCursor cursor = this.lastMutableBucket.cursor; + ManagedCursor cursor = this.lastMutableBucket.getCursor(); Map, ImmutableBucket> toBeDeletedBucketMap = new HashMap<>(); cursor.getCursorProperties().keySet().forEach(key -> { if (key.startsWith(DELAYED_BUCKET_KEY_PREFIX)) { String[] keys = key.split(DELIMITER); checkArgument(keys.length == 3); ImmutableBucket immutableBucket = - new ImmutableBucket(cursor, this.lastMutableBucket.bucketSnapshotStorage, + new ImmutableBucket(dispatcher.getName(), cursor, this.lastMutableBucket.bucketSnapshotStorage, Long.parseLong(keys[1]), Long.parseLong(keys[2])); putAndCleanOverlapRange(Range.closed(immutableBucket.startLedgerId, immutableBucket.endLedgerId), immutableBucket, toBeDeletedBucketMap); @@ -126,6 +128,8 @@ private synchronized long recoverBucketSnapshot() throws RuntimeException { Map, ImmutableBucket> immutableBucketMap = immutableBuckets.asMapOfRanges(); if (immutableBucketMap.isEmpty()) { + log.info("[{}] Recover delayed message index bucket snapshot finish, don't find bucket snapshot", + dispatcher.getName()); return 0; } @@ -232,10 +236,42 @@ private void afterCreateImmutableBucket(Pair immu DelayedIndex lastDelayedIndex = immutableBucketDelayedIndexPair.getRight(); snapshotSegmentLastIndexTable.put(lastDelayedIndex.getLedgerId(), lastDelayedIndex.getEntryId(), immutableBucket); - if (log.isDebugEnabled()) { - log.debug("[{}] Create bucket snapshot, bucket: {}", dispatcher.getName(), - lastMutableBucket); - } + + immutableBucket.getSnapshotCreateFuture().ifPresent(createFuture -> { + CompletableFuture future = createFuture.whenComplete((__, ex) -> { + if (ex == null) { + immutableBucket.setSnapshotSegments(null); + log.info("[{}] Creat bucket snapshot finish, bucketKey: {}", dispatcher.getName(), + immutableBucket.bucketKey()); + return; + } + + //TODO Record create snapshot failed + log.error("[{}] Failed to create bucket snapshot, bucketKey: {}", + dispatcher.getName(), immutableBucket.bucketKey(), ex); + + // Put indexes back into the shared queue and downgrade to memory mode + synchronized (BucketDelayedDeliveryTracker.this) { + immutableBucket.getSnapshotSegments().ifPresent(snapshotSegments -> { + for (DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment : + snapshotSegments) { + for (DelayedIndex delayedIndex : snapshotSegment.getIndexesList()) { + sharedBucketPriorityQueue.add(delayedIndex.getTimestamp(), + delayedIndex.getLedgerId(), delayedIndex.getEntryId()); + } + } + immutableBucket.setSnapshotSegments(null); + }); + + immutableBucket.setCurrentSegmentEntryId(immutableBucket.lastSegmentEntryId); + immutableBuckets.remove( + Range.closed(immutableBucket.startLedgerId, immutableBucket.endLedgerId)); + snapshotSegmentLastIndexTable.remove(lastDelayedIndex.getLedgerId(), + lastDelayedIndex.getTimestamp()); + } + }); + immutableBucket.setSnapshotCreateFuture(future); + }); } } @@ -263,12 +299,10 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver if (immutableBuckets.asMapOfRanges().size() > maxNumBuckets) { try { - asyncMergeBucketSnapshot().get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS); - } catch (InterruptedException | ExecutionException | TimeoutException e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - throw new RuntimeException(e); + asyncMergeBucketSnapshot().get(2 * AsyncOperationTimeoutSeconds * MaxRetryTimes, TimeUnit.SECONDS); + } catch (Exception e) { + // Ignore exception to merge bucket on the next schedule. + log.error("[{}] An exception occurs when merge bucket snapshot.", dispatcher.getName(), e); } } } @@ -304,7 +338,10 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot() { long numberMessages = bucketL.numberBucketDelayedMessages + bucketR.numberBucketDelayedMessages; if (numberMessages < minNumberMessages) { minNumberMessages = (int) numberMessages; - if (bucketL.lastSegmentEntryId > bucketL.getCurrentSegmentEntryId()) { + if (bucketL.lastSegmentEntryId > bucketL.getCurrentSegmentEntryId() + && bucketR.lastSegmentEntryId > bucketR.getCurrentSegmentEntryId() + && bucketL.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE).isDone() + && bucketR.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE).isDone()) { minIndex = i; } } @@ -314,62 +351,66 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot() { log.warn("[{}] Can't find able merged bucket", dispatcher.getName()); return CompletableFuture.completedFuture(null); } - return asyncMergeBucketSnapshot(values.get(minIndex), values.get(minIndex + 1)); + + ImmutableBucket immutableBucketA = values.get(minIndex); + ImmutableBucket immutableBucketB = values.get(minIndex + 1); + + if (log.isDebugEnabled()) { + log.info("[{}] Merging bucket snapshot, bucketAKey: {}, bucketBKey: {}", dispatcher.getName(), + immutableBucketA.bucketKey(), immutableBucketB.bucketKey()); + } + return asyncMergeBucketSnapshot(immutableBucketA, immutableBucketB).whenComplete((__, ex) -> { + if (ex != null) { + log.error("[{}] Failed to merge bucket snapshot, bucketAKey: {}, bucketBKey: {}", + dispatcher.getName(), immutableBucketA.bucketKey(), immutableBucketB.bucketKey(), ex); + } else { + log.info("[{}] Merge bucket snapshot finish, bucketAKey: {}, bucketBKey: {}", + dispatcher.getName(), immutableBucketA.bucketKey(), immutableBucketB.bucketKey()); + } + }); } private synchronized CompletableFuture asyncMergeBucketSnapshot(ImmutableBucket bucketA, ImmutableBucket bucketB) { - CompletableFuture snapshotCreateFutureA = - bucketA.getSnapshotCreateFuture().orElse(CompletableFuture.completedFuture(null)); - CompletableFuture snapshotCreateFutureB = - bucketB.getSnapshotCreateFuture().orElse(CompletableFuture.completedFuture(null)); - - return CompletableFuture.allOf(snapshotCreateFutureA, snapshotCreateFutureB).thenCompose(__ -> { - CompletableFuture> futureA = - bucketA.getRemainSnapshotSegment(); - CompletableFuture> futureB = - bucketB.getRemainSnapshotSegment(); - return futureA.thenCombine(futureB, CombinedSegmentDelayedIndexQueue::wrap) - .thenAccept(combinedDelayedIndexQueue -> { - Pair immutableBucketDelayedIndexPair = - lastMutableBucket.createImmutableBucketAndAsyncPersistent( - timeStepPerBucketSnapshotSegmentInMillis, sharedBucketPriorityQueue, - combinedDelayedIndexQueue, bucketA.startLedgerId, bucketB.endLedgerId); - - // Merge bit map to new bucket - Map delayedIndexBitMapA = bucketA.getDelayedIndexBitMap(); - Map delayedIndexBitMapB = bucketB.getDelayedIndexBitMap(); - Map delayedIndexBitMap = new HashMap<>(delayedIndexBitMapA); - delayedIndexBitMapB.forEach((ledgerId, bitMapB) -> { - delayedIndexBitMap.compute(ledgerId, (k, bitMapA) -> { - if (bitMapA == null) { - return bitMapB; - } - - bitMapA.or(bitMapB); - return bitMapA; - }); + CompletableFuture> futureA = + bucketA.getRemainSnapshotSegment(); + CompletableFuture> futureB = + bucketB.getRemainSnapshotSegment(); + return futureA.thenCombine(futureB, CombinedSegmentDelayedIndexQueue::wrap) + .thenAccept(combinedDelayedIndexQueue -> { + Pair immutableBucketDelayedIndexPair = + lastMutableBucket.createImmutableBucketAndAsyncPersistent( + timeStepPerBucketSnapshotSegmentInMillis, sharedBucketPriorityQueue, + combinedDelayedIndexQueue, bucketA.startLedgerId, bucketB.endLedgerId); + + // Merge bit map to new bucket + Map delayedIndexBitMapA = bucketA.getDelayedIndexBitMap(); + Map delayedIndexBitMapB = bucketB.getDelayedIndexBitMap(); + Map delayedIndexBitMap = new HashMap<>(delayedIndexBitMapA); + delayedIndexBitMapB.forEach((ledgerId, bitMapB) -> { + delayedIndexBitMap.compute(ledgerId, (k, bitMapA) -> { + if (bitMapA == null) { + return bitMapB; + } + + bitMapA.or(bitMapB); + return bitMapA; }); - immutableBucketDelayedIndexPair.getLeft().setDelayedIndexBitMap(delayedIndexBitMap); - - afterCreateImmutableBucket(immutableBucketDelayedIndexPair); - - CompletableFuture snapshotCreateFuture = CompletableFuture.completedFuture(null); - if (immutableBucketDelayedIndexPair != null) { - snapshotCreateFuture = immutableBucketDelayedIndexPair.getLeft().getSnapshotCreateFuture() - .orElse(CompletableFuture.completedFuture(null)); - } + }); + immutableBucketDelayedIndexPair.getLeft().setDelayedIndexBitMap(delayedIndexBitMap); - snapshotCreateFuture.thenCompose(___ -> { - CompletableFuture removeAFuture = bucketA.asyncDeleteBucketSnapshot(); - CompletableFuture removeBFuture = bucketB.asyncDeleteBucketSnapshot(); - return CompletableFuture.allOf(removeAFuture, removeBFuture); - }); + afterCreateImmutableBucket(immutableBucketDelayedIndexPair); - immutableBuckets.remove(Range.closed(bucketA.startLedgerId, bucketA.endLedgerId)); - immutableBuckets.remove(Range.closed(bucketB.startLedgerId, bucketB.endLedgerId)); + immutableBucketDelayedIndexPair.getLeft().getSnapshotCreateFuture() + .orElse(NULL_LONG_PROMISE).thenCompose(___ -> { + CompletableFuture removeAFuture = bucketA.asyncDeleteBucketSnapshot(); + CompletableFuture removeBFuture = bucketB.asyncDeleteBucketSnapshot(); + return CompletableFuture.allOf(removeAFuture, removeBFuture); }); - }); + + immutableBuckets.remove(Range.closed(bucketA.startLedgerId, bucketA.endLedgerId)); + immutableBuckets.remove(Range.closed(bucketB.startLedgerId, bucketB.endLedgerId)); + }); } @Override @@ -422,19 +463,31 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa long ledgerId = sharedBucketPriorityQueue.peekN2(); long entryId = sharedBucketPriorityQueue.peekN3(); - positions.add(new PositionImpl(ledgerId, entryId)); - sharedBucketPriorityQueue.pop(); - removeIndexBit(ledgerId, entryId); - - ImmutableBucket bucket = snapshotSegmentLastIndexTable.remove(ledgerId, entryId); + ImmutableBucket bucket = snapshotSegmentLastIndexTable.get(ledgerId, entryId); if (bucket != null && immutableBuckets.asMapOfRanges().containsValue(bucket)) { + final int preSegmentEntryId = bucket.currentSegmentEntryId; if (log.isDebugEnabled()) { - log.debug("[{}] Load next snapshot segment, bucket: {}", dispatcher.getName(), bucket); + log.debug("[{}] Loading next bucket snapshot segment, bucketKey: {}, nextSegmentEntryId: {}", + dispatcher.getName(), bucket.bucketKey(), preSegmentEntryId + 1); } // All message of current snapshot segment are scheduled, load next snapshot segment // TODO make it asynchronous and not blocking this process try { + boolean createFutureDone = bucket.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE).isDone(); + + if (!createFutureDone) { + log.info("[{}] Skip load to wait for bucket snapshot create finish, bucketKey:{}", + dispatcher.getName(), bucket.bucketKey()); + break; + } + + if (bucket.currentSegmentEntryId == bucket.lastSegmentEntryId) { + immutableBuckets.remove(Range.closed(bucket.startLedgerId, bucket.endLedgerId)); + bucket.asyncDeleteBucketSnapshot(); + continue; + } + bucket.asyncLoadNextBucketSnapshotEntry().thenAccept(indexList -> { if (CollectionUtils.isEmpty(indexList)) { immutableBuckets.remove(Range.closed(bucket.startLedgerId, bucket.endLedgerId)); @@ -449,13 +502,32 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa sharedBucketPriorityQueue.add(index.getTimestamp(), index.getLedgerId(), index.getEntryId()); } - }).get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS); - } catch (InterruptedException | ExecutionException | TimeoutException e) { - // TODO make this segment load again - throw new RuntimeException(e); + }).whenComplete((__, ex) -> { + if (ex != null) { + // Back bucket state + bucket.setCurrentSegmentEntryId(preSegmentEntryId); + + log.error("[{}] Failed to load bucket snapshot segment, bucketKey: {}", + dispatcher.getName(), bucket.bucketKey(), ex); + } else { + log.info("[{}] Load next bucket snapshot segment finish, bucketKey: {}, segmentEntryId: {}", + dispatcher.getName(), bucket.bucketKey(), bucket.currentSegmentEntryId); + } + }).get(AsyncOperationTimeoutSeconds * MaxRetryTimes, TimeUnit.SECONDS); + snapshotSegmentLastIndexTable.remove(ledgerId, entryId); + } catch (Exception e) { + // Ignore exception to reload this segment on the next schedule. + log.error("[{}] An exception occurs when load next bucket snapshot, bucketKey:{}", + dispatcher.getName(), bucket.bucketKey(), e); + break; } } + positions.add(new PositionImpl(ledgerId, entryId)); + + sharedBucketPriorityQueue.pop(); + removeIndexBit(ledgerId, entryId); + --n; --numberDelayedMessages; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java index 3e9c577454fed..c947e53843a85 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java @@ -18,14 +18,17 @@ */ package org.apache.pulsar.broker.delayed.bucket; +import static org.apache.bookkeeper.mledger.util.Futures.executeWithRetry; import static org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker.AsyncOperationTimeoutSeconds; import com.google.protobuf.ByteString; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.commons.collections4.CollectionUtils; @@ -38,8 +41,17 @@ @Slf4j class ImmutableBucket extends Bucket { - ImmutableBucket(ManagedCursor cursor, BucketSnapshotStorage storage, long startLedgerId, long endLedgerId) { - super(cursor, storage, startLedgerId, endLedgerId); + + @Setter + private volatile List snapshotSegments; + + ImmutableBucket(String dispatcherName, ManagedCursor cursor, + BucketSnapshotStorage storage, long startLedgerId, long endLedgerId) { + super(dispatcherName, cursor, storage, startLedgerId, endLedgerId); + } + + public Optional> getSnapshotSegments() { + return Optional.ofNullable(snapshotSegments); } CompletableFuture> asyncLoadNextBucketSnapshotEntry() { @@ -52,61 +64,65 @@ CompletableFuture> asyncRecoverBucketSnapshotEntry(Supplier> asyncLoadNextBucketSnapshotEntry(boolean isRecover, Supplier cutoffTimeSupplier) { - if (log.isDebugEnabled()) { - log.debug("[{}] Load next bucket snapshot data, bucket: {}", cursor.getName(), this); + final long bucketId = getAndUpdateBucketId(); + final CompletableFuture loadMetaDataFuture; + if (isRecover) { + final long cutoffTime = cutoffTimeSupplier.get(); + // Load Metadata of bucket snapshot + final String bucketKey = bucketKey(); + loadMetaDataFuture = executeWithRetry(() -> bucketSnapshotStorage.getBucketSnapshotMetadata(bucketId) + .whenComplete((___, ex) -> { + if (ex != null) { + log.warn("[{}] Failed to get bucket snapshot metadata," + + " bucketKey: {}, bucketId: {}", + dispatcherName, bucketKey, bucketId, ex); + } + }), BucketSnapshotPersistenceException.class, MaxRetryTimes) + .thenApply(snapshotMetadata -> { + List metadataList = + snapshotMetadata.getMetadataListList(); + + // Skip all already reach schedule time snapshot segments + int nextSnapshotEntryIndex = 0; + while (nextSnapshotEntryIndex < metadataList.size() + && metadataList.get(nextSnapshotEntryIndex).getMaxScheduleTimestamp() <= cutoffTime) { + nextSnapshotEntryIndex++; + } + + this.setLastSegmentEntryId(metadataList.size()); + this.recoverDelayedIndexBitMapAndNumber(nextSnapshotEntryIndex, metadataList); + + return nextSnapshotEntryIndex + 1; + }); + } else { + loadMetaDataFuture = CompletableFuture.completedFuture(currentSegmentEntryId + 1); } - // Wait bucket snapshot create finish - CompletableFuture snapshotCreateFuture = - getSnapshotCreateFuture().orElseGet(() -> CompletableFuture.completedFuture(null)) - .thenApply(__ -> null); - - return snapshotCreateFuture.thenCompose(__ -> { - final long bucketId = getAndUpdateBucketId(); - final CompletableFuture loadMetaDataFuture; - if (isRecover) { - final long cutoffTime = cutoffTimeSupplier.get(); - // Load Metadata of bucket snapshot - loadMetaDataFuture = bucketSnapshotStorage.getBucketSnapshotMetadata(bucketId) - .thenApply(snapshotMetadata -> { - List metadataList = - snapshotMetadata.getMetadataListList(); - - // Skip all already reach schedule time snapshot segments - int nextSnapshotEntryIndex = 0; - while (nextSnapshotEntryIndex < metadataList.size() - && metadataList.get(nextSnapshotEntryIndex).getMaxScheduleTimestamp() <= cutoffTime) { - nextSnapshotEntryIndex++; - } - - this.setLastSegmentEntryId(metadataList.size()); - this.recoverDelayedIndexBitMapAndNumber(nextSnapshotEntryIndex, metadataList); - - return nextSnapshotEntryIndex + 1; - }); - } else { - loadMetaDataFuture = CompletableFuture.completedFuture(currentSegmentEntryId + 1); + return loadMetaDataFuture.thenCompose(nextSegmentEntryId -> { + if (nextSegmentEntryId > lastSegmentEntryId) { + return CompletableFuture.completedFuture(null); } - return loadMetaDataFuture.thenCompose(nextSegmentEntryId -> { - if (nextSegmentEntryId > lastSegmentEntryId) { - return CompletableFuture.completedFuture(null); - } - - return bucketSnapshotStorage.getBucketSnapshotSegment(bucketId, nextSegmentEntryId, nextSegmentEntryId) - .thenApply(bucketSnapshotSegments -> { - if (CollectionUtils.isEmpty(bucketSnapshotSegments)) { - return Collections.emptyList(); - } - - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment = - bucketSnapshotSegments.get(0); - List indexList = - snapshotSegment.getIndexesList(); - this.setCurrentSegmentEntryId(nextSegmentEntryId); - return indexList; - }); - }); + return executeWithRetry( + () -> bucketSnapshotStorage.getBucketSnapshotSegment(bucketId, nextSegmentEntryId, + nextSegmentEntryId).whenComplete((___, ex) -> { + if (ex != null) { + log.warn("[{}] Failed to get bucket snapshot segment. bucketKey: {}, bucketId: {}", + dispatcherName, bucketKey(), bucketId, ex); + } + }), BucketSnapshotPersistenceException.class, MaxRetryTimes) + .thenApply(bucketSnapshotSegments -> { + if (CollectionUtils.isEmpty(bucketSnapshotSegments)) { + return Collections.emptyList(); + } + + DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment = + bucketSnapshotSegments.get(0); + List indexList = + snapshotSegment.getIndexesList(); + this.setCurrentSegmentEntryId(nextSegmentEntryId); + return indexList; + }); }); } @@ -136,18 +152,34 @@ CompletableFuture> if (nextSegmentEntryId > lastSegmentEntryId) { return CompletableFuture.completedFuture(Collections.emptyList()); } - return bucketSnapshotStorage.getBucketSnapshotSegment(getAndUpdateBucketId(), nextSegmentEntryId, - lastSegmentEntryId); + return executeWithRetry(() -> { + return bucketSnapshotStorage.getBucketSnapshotSegment(getAndUpdateBucketId(), nextSegmentEntryId, + lastSegmentEntryId).whenComplete((__, ex) -> { + if (ex != null) { + log.warn("[{}] Failed to get remain bucket snapshot segment, bucketKey: {}.", + dispatcherName, bucketKey(), ex); + } + }); + }, BucketSnapshotPersistenceException.class, MaxRetryTimes); } CompletableFuture asyncDeleteBucketSnapshot() { String bucketKey = bucketKey(); long bucketId = getAndUpdateBucketId(); return removeBucketCursorProperty(bucketKey).thenCompose(__ -> - bucketSnapshotStorage.deleteBucketSnapshot(bucketId)).whenComplete((__, ex) -> { + executeWithRetry(() -> bucketSnapshotStorage.deleteBucketSnapshot(bucketId).whenComplete((___, ex) -> { + if (ex != null) { + log.warn("[{}] Failed to delete bucket snapshot. bucketKey: {}, bucketId: {}", + dispatcherName, bucketKey, bucketId, ex); + } + }), + BucketSnapshotPersistenceException.class, MaxRetryTimes)).whenComplete((__, ex) -> { if (ex != null) { - log.warn("Failed to delete bucket snapshot, bucketId: {}, bucketKey: {}", - bucketId, bucketKey, ex); + log.warn("[{}] Failed to delete bucket snapshot, bucketId: {}, bucketKey: {}", + dispatcherName, bucketId, bucketKey, ex); + } else { + log.info("[{}] Delete bucket snapshot finish, bucketId: {}, bucketKey: {}", + dispatcherName, bucketId, bucketKey); } }); } @@ -179,8 +211,8 @@ void clear(boolean delete) { try { snapshotGenerateFuture.get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS); } catch (Exception e) { - log.warn("Failed wait to snapshot generate, bucketId: {}, bucketKey: {}", getBucketId(), - bucketKey()); + log.warn("[{}] Failed wait to snapshot generate, bucketId: {}, bucketKey: {}", dispatcherName, + getBucketId(), bucketKey()); } } }); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java index ad457329c427f..40ba8f4c4b593 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java @@ -42,9 +42,9 @@ class MutableBucket extends Bucket implements AutoCloseable { private final TripleLongPriorityQueue priorityQueue; - MutableBucket(ManagedCursor cursor, + MutableBucket(String dispatcherName, ManagedCursor cursor, BucketSnapshotStorage bucketSnapshotStorage) { - super(cursor, bucketSnapshotStorage, -1L, -1L); + super(dispatcherName, cursor, bucketSnapshotStorage, -1L, -1L); this.priorityQueue = new TripleLongPriorityQueue(); } @@ -59,6 +59,9 @@ Pair createImmutableBucketAndAsyncPersistent( final long timeStepPerBucketSnapshotSegment, TripleLongPriorityQueue sharedQueue, DelayedIndexQueue delayedIndexQueue, final long startLedgerId, final long endLedgerId) { + log.info("[{}] Creating bucket snapshot, startLedgerId: {}, endLedgerId: {}", dispatcherName, + startLedgerId, endLedgerId); + if (delayedIndexQueue.isEmpty()) { return null; } @@ -122,11 +125,16 @@ Pair createImmutableBucketAndAsyncPersistent( final int lastSegmentEntryId = segmentMetadataList.size(); - ImmutableBucket bucket = new ImmutableBucket(cursor, bucketSnapshotStorage, startLedgerId, endLedgerId); + ImmutableBucket bucket = new ImmutableBucket(dispatcherName, cursor, bucketSnapshotStorage, + startLedgerId, endLedgerId); bucket.setCurrentSegmentEntryId(1); bucket.setNumberBucketDelayedMessages(numMessages); bucket.setLastSegmentEntryId(lastSegmentEntryId); + // Skip first segment, because it has already been loaded + List snapshotSegments = bucketSnapshotSegments.subList(1, bucketSnapshotSegments.size()); + bucket.setSnapshotSegments(snapshotSegments); + // Add the first snapshot segment last message to snapshotSegmentLastMessageTable checkArgument(!bucketSnapshotSegments.isEmpty()); SnapshotSegment snapshotSegment = bucketSnapshotSegments.get(0); @@ -136,12 +144,6 @@ Pair createImmutableBucketAndAsyncPersistent( CompletableFuture future = asyncSaveBucketSnapshot(bucket, bucketSnapshotMetadata, bucketSnapshotSegments); bucket.setSnapshotCreateFuture(future); - future.whenComplete((__, ex) -> { - if (ex != null) { - //TODO Record create snapshot failed - log.error("Failed to create snapshot: ", ex); - } - }); return result; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java index 9b2fbda4195da..b106642915587 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java @@ -23,8 +23,10 @@ import io.netty.buffer.PooledByteBufAllocator; import io.netty.util.concurrent.DefaultThreadFactory; import java.util.ArrayList; +import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Queue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; @@ -36,6 +38,7 @@ import org.apache.pulsar.broker.delayed.bucket.BucketSnapshotStorage; import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata; import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; +import org.apache.pulsar.common.util.FutureUtil; @Slf4j public class MockBucketSnapshotStorage implements BucketSnapshotStorage { @@ -53,9 +56,35 @@ public MockBucketSnapshotStorage() { this.maxBucketId = new AtomicLong(); } + public Queue createExceptionQueue = new LinkedList<>(); + public Queue getMetaDataExceptionQueue = new LinkedList<>(); + public Queue getSegmentExceptionQueue = new LinkedList<>(); + public Queue deleteExceptionQueue = new LinkedList<>(); + + + public void injectCreateException(Throwable throwable) { + createExceptionQueue.add(throwable); + } + + public void injectGetMetaDataException(Throwable throwable) { + getMetaDataExceptionQueue.add(throwable); + } + + public void injectGetSegmentException(Throwable throwable) { + getSegmentExceptionQueue.add(throwable); + } + + public void injectDeleteException(Throwable throwable) { + deleteExceptionQueue.add(throwable); + } + @Override public CompletableFuture createBucketSnapshot( SnapshotMetadata snapshotMetadata, List bucketSnapshotSegments, String bucketKey) { + Throwable throwable = createExceptionQueue.poll(); + if (throwable != null) { + return FutureUtil.failedFuture(throwable); + } return CompletableFuture.supplyAsync(() -> { long bucketId = maxBucketId.getAndIncrement(); List entries = new ArrayList<>(); @@ -81,6 +110,10 @@ public CompletableFuture createBucketSnapshot( @Override public CompletableFuture getBucketSnapshotMetadata(long bucketId) { + Throwable throwable = getMetaDataExceptionQueue.poll(); + if (throwable != null) { + return FutureUtil.failedFuture(throwable); + } return CompletableFuture.supplyAsync(() -> { ByteBuf byteBuf = this.bucketSnapshots.get(bucketId).get(0); SnapshotMetadata snapshotMetadata; @@ -96,6 +129,10 @@ public CompletableFuture getBucketSnapshotMetadata(long bucket @Override public CompletableFuture> getBucketSnapshotSegment(long bucketId, long firstSegmentEntryId, long lastSegmentEntryId) { + Throwable throwable = getSegmentExceptionQueue.poll(); + if (throwable != null) { + return FutureUtil.failedFuture(throwable); + } return CompletableFuture.supplyAsync(() -> { List snapshotSegments = new ArrayList<>(); long lastEntryId = Math.min(lastSegmentEntryId, this.bucketSnapshots.get(bucketId).size()); @@ -115,6 +152,10 @@ public CompletableFuture> getBucketSnapshotSegment(long bu @Override public CompletableFuture deleteBucketSnapshot(long bucketId) { + Throwable throwable = deleteExceptionQueue.poll(); + if (throwable != null) { + return FutureUtil.failedFuture(throwable); + } return CompletableFuture.supplyAsync(() -> { List remove = this.bucketSnapshots.remove(bucketId); if (remove != null) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java index efb0fa7ab7ba2..499262c1e60b9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java @@ -46,7 +46,7 @@ public MockManagedCursor(String name) { @Override public String getName() { - return null; + return this.name; } @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java index 920f2cf2b64b3..08e1f78725bf4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java @@ -50,6 +50,8 @@ import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; import org.roaringbitmap.RoaringBitmap; import org.roaringbitmap.buffer.ImmutableRoaringBitmap; +import org.testcontainers.shaded.org.apache.commons.lang3.mutable.MutableLong; +import org.testcontainers.shaded.org.awaitility.Awaitility; import org.testng.annotations.AfterMethod; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -77,6 +79,7 @@ public Object[][] provider(Method method) throws Exception { bucketSnapshotStorage.start(); ManagedCursor cursor = new MockManagedCursor("my_test_cursor"); doReturn(cursor).when(dispatcher).getCursor(); + doReturn(cursor.getName()).when(dispatcher).getName(); final String methodName = method.getName(); return switch (methodName) { @@ -136,7 +139,7 @@ public Object[][] provider(Method method) throws Exception { new BucketDelayedDeliveryTracker(dispatcher, timer, 500, clock, true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50) }}; - case "testMergeSnapshot" -> new Object[][]{{ + case "testMergeSnapshot", "testWithBkException", "testWithCreateFailDowngrade" -> new Object[][]{{ new BucketDelayedDeliveryTracker(dispatcher, timer, 100000, clock, true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 10) }}; @@ -255,6 +258,24 @@ public void testMergeSnapshot(BucketDelayedDeliveryTracker tracker) { assertEquals(10, size); + tracker.addMessage(111, 1011, 111 * 10); + + MutableLong delayedMessagesInSnapshot = new MutableLong(); + tracker.getImmutableBuckets().asMapOfRanges().forEach((k, v) -> { + delayedMessagesInSnapshot.add(v.getNumberBucketDelayedMessages()); + }); + + tracker.close(); + + tracker = new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock, + true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 10); + + assertEquals(tracker.getNumberOfDelayedMessages(), delayedMessagesInSnapshot.getValue()); + + for (int i = 1; i <= 110; i++) { + tracker.addMessage(i, i, i * 10); + } + clockTime.set(110 * 10); NavigableSet scheduledMessages = tracker.getScheduledMessages(110); @@ -263,4 +284,98 @@ public void testMergeSnapshot(BucketDelayedDeliveryTracker tracker) { assertEquals(position, PositionImpl.get(i, i)); } } + + @Test(dataProvider = "delayedTracker") + public void testWithBkException(BucketDelayedDeliveryTracker tracker) { + MockBucketSnapshotStorage mockBucketSnapshotStorage = (MockBucketSnapshotStorage) bucketSnapshotStorage; + mockBucketSnapshotStorage.injectCreateException( + new BucketSnapshotPersistenceException("Bookie operation timeout, op: Create entry")); + mockBucketSnapshotStorage.injectGetMetaDataException( + new BucketSnapshotPersistenceException("Bookie operation timeout, op: Get entry")); + mockBucketSnapshotStorage.injectGetSegmentException( + new BucketSnapshotPersistenceException("Bookie operation timeout, op: Get entry")); + mockBucketSnapshotStorage.injectDeleteException( + new BucketSnapshotPersistenceException("Bookie operation timeout, op: Delete entry")); + + assertEquals(1, mockBucketSnapshotStorage.createExceptionQueue.size()); + assertEquals(1, mockBucketSnapshotStorage.getMetaDataExceptionQueue.size()); + assertEquals(1, mockBucketSnapshotStorage.getSegmentExceptionQueue.size()); + assertEquals(1, mockBucketSnapshotStorage.deleteExceptionQueue.size()); + + for (int i = 1; i <= 110; i++) { + tracker.addMessage(i, i, i * 10); + } + + assertEquals(110, tracker.getNumberOfDelayedMessages()); + + int size = tracker.getImmutableBuckets().asMapOfRanges().size(); + + assertEquals(10, size); + + tracker.addMessage(111, 1011, 111 * 10); + + MutableLong delayedMessagesInSnapshot = new MutableLong(); + tracker.getImmutableBuckets().asMapOfRanges().forEach((k, v) -> { + delayedMessagesInSnapshot.add(v.getNumberBucketDelayedMessages()); + }); + + tracker.close(); + + tracker = new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock, + true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 10); + + Long delayedMessagesInSnapshotValue = delayedMessagesInSnapshot.getValue(); + assertEquals(tracker.getNumberOfDelayedMessages(), delayedMessagesInSnapshotValue); + + clockTime.set(110 * 10); + + mockBucketSnapshotStorage.injectGetSegmentException( + new BucketSnapshotPersistenceException("Bookie operation timeout1, op: Get entry")); + mockBucketSnapshotStorage.injectGetSegmentException( + new BucketSnapshotPersistenceException("Bookie operation timeout2, op: Get entry")); + mockBucketSnapshotStorage.injectGetSegmentException( + new BucketSnapshotPersistenceException("Bookie operation timeout3, op: Get entry")); + mockBucketSnapshotStorage.injectGetSegmentException( + new BucketSnapshotPersistenceException("Bookie operation timeout4, op: Get entry")); + + assertEquals(tracker.getScheduledMessages(100).size(), 0); + + assertEquals(tracker.getScheduledMessages(100).size(), delayedMessagesInSnapshotValue); + + assertTrue(mockBucketSnapshotStorage.createExceptionQueue.isEmpty()); + assertTrue(mockBucketSnapshotStorage.getMetaDataExceptionQueue.isEmpty()); + assertTrue(mockBucketSnapshotStorage.getSegmentExceptionQueue.isEmpty()); + assertTrue(mockBucketSnapshotStorage.deleteExceptionQueue.isEmpty()); + } + + @Test(dataProvider = "delayedTracker") + public void testWithCreateFailDowngrade(BucketDelayedDeliveryTracker tracker) { + MockBucketSnapshotStorage mockBucketSnapshotStorage = (MockBucketSnapshotStorage) bucketSnapshotStorage; + mockBucketSnapshotStorage.injectCreateException( + new BucketSnapshotPersistenceException("Bookie operation timeout, op: Create entry")); + mockBucketSnapshotStorage.injectCreateException( + new BucketSnapshotPersistenceException("Bookie operation timeout, op: Create entry")); + mockBucketSnapshotStorage.injectCreateException( + new BucketSnapshotPersistenceException("Bookie operation timeout, op: Create entry")); + mockBucketSnapshotStorage.injectCreateException( + new BucketSnapshotPersistenceException("Bookie operation timeout, op: Create entry")); + + assertEquals(4, mockBucketSnapshotStorage.createExceptionQueue.size()); + + for (int i = 1; i <= 6; i++) { + tracker.addMessage(i, i, i * 10); + } + + Awaitility.await().untilAsserted(() -> assertEquals(0, tracker.getImmutableBuckets().asMapOfRanges().size())); + + clockTime.set(5 * 10); + + assertEquals(6, tracker.getNumberOfDelayedMessages()); + + NavigableSet scheduledMessages = tracker.getScheduledMessages(5); + for (int i = 1; i <= 5; i++) { + PositionImpl position = scheduledMessages.pollFirst(); + assertEquals(position, PositionImpl.get(i, i)); + } + } } From fec4578e8a5cfeb7e5ad53a0ce47b3e116e94a4d Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Thu, 2 Mar 2023 14:29:48 +0800 Subject: [PATCH 077/174] [improve][broker] PIP-192: Add large topic count filter (#19613) --- .../extensions/ExtensibleLoadManagerImpl.java | 2 + .../extensions/data/BrokerLoadData.java | 4 + .../filter/BrokerMaxTopicCountFilter.java | 49 ++++++++ .../reporter/BrokerLoadDataReporter.java | 2 +- .../pulsar/broker/service/PulsarStats.java | 3 +- .../pulsar/broker/stats/BrokerStats.java | 2 + .../ExtensibleLoadManagerImplTest.java | 2 +- .../extensions/data/BrokerLoadDataTest.java | 8 +- .../filter/BrokerFilterTestBase.java | 109 ++++++++++++++++++ .../filter/BrokerMaxTopicCountFilterTest.java | 68 +++++++++++ .../filter/BrokerVersionFilterTest.java | 26 +---- .../reporter/BrokerLoadDataReporterTest.java | 7 +- .../scheduler/TransferShedderTest.java | 2 +- .../LeastResourceUsageWithWeightTest.java | 4 +- 14 files changed, 251 insertions(+), 37 deletions(-) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilterTestBase.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index 59c6674676101..650c12af57391 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -41,6 +41,7 @@ import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData; import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerFilter; +import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerMaxTopicCountFilter; import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerVersionFilter; import org.apache.pulsar.broker.loadbalance.extensions.models.AssignCounter; import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter; @@ -132,6 +133,7 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager { */ public ExtensibleLoadManagerImpl() { this.brokerFilterPipeline = new ArrayList<>(); + this.brokerFilterPipeline.add(new BrokerMaxTopicCountFilter()); this.brokerFilterPipeline.add(new BrokerVersionFilter()); // TODO: Make brokerSelectionStrategy configurable. this.brokerSelectionStrategy = new LeastResourceUsageWithWeight(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadData.java index dbd17152d26e4..8b373bc5954b2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadData.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadData.java @@ -57,6 +57,7 @@ public class BrokerLoadData { private double msgRateIn; // messages/sec private double msgRateOut; // messages/sec private int bundleCount; + private int topics; // Load data features computed from the above resources. private double maxResourceUsage; // max of resource usages @@ -113,6 +114,7 @@ public void update(final SystemResourceUsage usage, double msgRateIn, double msgRateOut, int bundleCount, + int topics, ServiceConfiguration conf) { updateSystemResourceUsage(usage.cpu, usage.memory, usage.directMemory, usage.bandwidthIn, usage.bandwidthOut); this.msgThroughputIn = msgThroughputIn; @@ -120,6 +122,7 @@ public void update(final SystemResourceUsage usage, this.msgRateIn = msgRateIn; this.msgRateOut = msgRateOut; this.bundleCount = bundleCount; + this.topics = topics; updateFeatures(conf); updatedAt = System.currentTimeMillis(); } @@ -137,6 +140,7 @@ public void update(final BrokerLoadData other) { msgRateIn = other.msgRateIn; msgRateOut = other.msgRateOut; bundleCount = other.bundleCount; + topics = other.topics; weightedMaxEMA = other.weightedMaxEMA; maxResourceUsage = other.maxResourceUsage; updatedAt = other.updatedAt; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java new file mode 100644 index 0000000000000..e3f8faca32468 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.filter; + +import java.util.Map; +import java.util.Optional; +import org.apache.pulsar.broker.loadbalance.BrokerFilterException; +import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; + +public class BrokerMaxTopicCountFilter implements BrokerFilter { + + public static final String FILTER_NAME = "broker_max_topic_count_filter"; + + @Override + public String name() { + return FILTER_NAME; + } + + @Override + public Map filter(Map brokers, + LoadManagerContext context) throws BrokerFilterException { + int loadBalancerBrokerMaxTopics = context.brokerConfiguration().getLoadBalancerBrokerMaxTopics(); + brokers.keySet().removeIf(broker -> { + Optional brokerLoadDataOpt = context.brokerLoadDataStore().get(broker); + long topics = brokerLoadDataOpt.map(BrokerLoadData::getTopics).orElse(0); + // TODO: The broker load data might be delayed, so the max topic check might not accurate. + return topics >= loadBalancerBrokerMaxTopics; + }); + return brokers; + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporter.java index cf50f942e11b4..256e52c4554d1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporter.java @@ -83,8 +83,8 @@ public BrokerLoadData generateLoadData() { brokerStats.msgRateIn, brokerStats.msgRateOut, brokerStats.bundleCount, + brokerStats.topics, pulsar.getConfiguration()); - } return this.localData; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java index 045bb336d62e2..9cdf9d1dfc68d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java @@ -132,6 +132,7 @@ public synchronized void updateStats( k -> new NamespaceBundleStats()); currentBundleStats.reset(); currentBundleStats.topics = topics.size(); + brokerStats.topics += topics.size(); topicStatsStream.startObject(NamespaceBundle.getBundleRange(bundle)); @@ -280,4 +281,4 @@ public void recordConnectionCreateSuccess() { public void recordConnectionCreateFail() { brokerOperabilityMetrics.recordConnectionCreateFail(); } -} \ No newline at end of file +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerStats.java index d0be71167002d..84d5432fb9e19 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerStats.java @@ -21,6 +21,7 @@ public class BrokerStats extends NamespaceStats { public int bundleCount; + public int topics; public BrokerStats(int ratePeriodInSeconds) { super(ratePeriodInSeconds); } @@ -29,5 +30,6 @@ public BrokerStats(int ratePeriodInSeconds) { public void reset() { super.reset(); bundleCount = 0; + topics = 0; } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index 001aac34a4ba2..d1bf29725d004 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -324,7 +324,7 @@ public void testGetMetrics() throws Exception { usage.setDirectMemory(directMemory); usage.setBandwidthIn(bandwidthIn); usage.setBandwidthOut(bandwidthOut); - loadData.update(usage, 1, 2, 3, 4, 5, conf); + loadData.update(usage, 1, 2, 3, 4, 5, 6, conf); brokerLoadMetrics.set(loadData.toMetrics(pulsar.getAdvertisedAddress())); } { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadDataTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadDataTest.java index c85fa4ce9d2cd..5ba7629dd1132 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadDataTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadDataTest.java @@ -59,7 +59,7 @@ public void testUpdateBySystemResourceUsage() { usage1.setDirectMemory(directMemory); usage1.setBandwidthIn(bandwidthIn); usage1.setBandwidthOut(bandwidthOut); - data.update(usage1, 1, 2, 3, 4, 5, conf); + data.update(usage1, 1, 2, 3, 4, 5, 6, conf); assertEquals(data.getCpu(), cpu); assertEquals(data.getMemory(), memory); @@ -71,6 +71,7 @@ public void testUpdateBySystemResourceUsage() { assertEquals(data.getMsgRateIn(), 3.0); assertEquals(data.getMsgRateOut(), 4.0); assertEquals(data.getBundleCount(), 5); + assertEquals(data.getTopics(), 6); assertEquals(data.getMaxResourceUsage(), 0.04); // skips memory usage assertEquals(data.getWeightedMaxEMA(), 2); assertThat(data.getUpdatedAt(), greaterThanOrEqualTo(now)); @@ -87,7 +88,7 @@ public void testUpdateBySystemResourceUsage() { usage2.setDirectMemory(directMemory); usage2.setBandwidthIn(bandwidthIn); usage2.setBandwidthOut(bandwidthOut); - data.update(usage2, 5, 6, 7, 8, 9, conf); + data.update(usage2, 5, 6, 7, 8, 9, 10, conf); assertEquals(data.getCpu(), cpu); assertEquals(data.getMemory(), memory); @@ -99,6 +100,7 @@ public void testUpdateBySystemResourceUsage() { assertEquals(data.getMsgRateIn(), 7.0); assertEquals(data.getMsgRateOut(), 8.0); assertEquals(data.getBundleCount(), 9); + assertEquals(data.getTopics(), 10); assertEquals(data.getMaxResourceUsage(), 3.0); assertEquals(data.getWeightedMaxEMA(), 1.875); assertThat(data.getUpdatedAt(), greaterThanOrEqualTo(now)); @@ -137,7 +139,7 @@ public void testUpdateByBrokerLoadData() { usage1.setDirectMemory(directMemory); usage1.setBandwidthIn(bandwidthIn); usage1.setBandwidthOut(bandwidthOut); - other.update(usage1, 1, 2, 3, 4, 5, conf); + other.update(usage1, 1, 2, 3, 4, 5, 6, conf); data.update(other); assertEquals(data, other); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilterTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilterTestBase.java new file mode 100644 index 0000000000000..3de957c3b1a3a --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilterTestBase.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.filter; + +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.function.BiConsumer; + +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; +import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore; +import org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener; + +public class BrokerFilterTestBase { + + public LoadManagerContext getContext() { + LoadManagerContext mockContext = mock(LoadManagerContext.class); + ServiceConfiguration configuration = new ServiceConfiguration(); + var brokerLoadDataStore = new LoadDataStore() { + Map map = new HashMap<>(); + @Override + public void close() throws IOException { + + } + + @Override + public CompletableFuture pushAsync(String key, BrokerLoadData loadData) { + map.put(key, loadData); + return null; + } + + @Override + public CompletableFuture removeAsync(String key) { + return null; + } + + @Override + public Optional get(String key) { + var val = map.get(key); + if (val == null) { + return Optional.empty(); + } + return Optional.of(val); + } + + @Override + public void forEach(BiConsumer action) { + + } + + @Override + public Set> entrySet() { + return map.entrySet(); + } + + @Override + public int size() { + return map.size(); + } + }; + configuration.setPreferLaterVersions(true); + doReturn(configuration).when(mockContext).brokerConfiguration(); + doReturn(brokerLoadDataStore).when(mockContext).brokerLoadDataStore(); + return mockContext; + } + + public BrokerLookupData getLookupData() { + return getLookupData("3.0.0"); + } + + public BrokerLookupData getLookupData(String version) { + String webServiceUrl = "http://localhost:8080"; + String webServiceUrlTls = "https://localhoss:8081"; + String pulsarServiceUrl = "pulsar://localhost:6650"; + String pulsarServiceUrlTls = "pulsar+ssl://localhost:6651"; + Map advertisedListeners = new HashMap<>(); + Map protocols = new HashMap<>(){{ + put("kafka", "9092"); + }}; + return new BrokerLookupData( + webServiceUrl, webServiceUrlTls, pulsarServiceUrl, + pulsarServiceUrlTls, advertisedListeners, protocols, true, true, version); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java new file mode 100644 index 0000000000000..4c3255341b778 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.filter; + +import org.apache.commons.lang.reflect.FieldUtils; +import org.apache.pulsar.broker.loadbalance.BrokerFilterException; +import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; +import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore; +import org.testng.annotations.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.testng.Assert.assertEquals; + +/** + * Unit test for {@link BrokerMaxTopicCountFilter}. + */ +@Test(groups = "broker") +public class BrokerMaxTopicCountFilterTest extends BrokerFilterTestBase { + + @Test + public void test() throws IllegalAccessException, BrokerFilterException { + LoadManagerContext context = getContext(); + LoadDataStore store = context.brokerLoadDataStore(); + BrokerLoadData maxTopicLoadData = new BrokerLoadData(); + FieldUtils.writeDeclaredField(maxTopicLoadData, "topics", + context.brokerConfiguration().getLoadBalancerBrokerMaxTopics(), true); + BrokerLoadData exceedMaxTopicLoadData = new BrokerLoadData(); + FieldUtils.writeDeclaredField(exceedMaxTopicLoadData, "topics", + context.brokerConfiguration().getLoadBalancerBrokerMaxTopics() * 2, true); + store.pushAsync("broker1", maxTopicLoadData); + store.pushAsync("broker2", new BrokerLoadData()); + store.pushAsync("broker3", exceedMaxTopicLoadData); + + BrokerMaxTopicCountFilter filter = new BrokerMaxTopicCountFilter(); + Map originalBrokers = Map.of( + "broker1", getLookupData(), + "broker2", getLookupData(), + "broker3", getLookupData(), + "broker4", getLookupData() + ); + Map result = filter.filter(new HashMap<>(originalBrokers), context); + assertEquals(result, Map.of( + "broker2", getLookupData(), + "broker4", getLookupData() + )); + } + +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java index 1fcc3836a6fac..d36c79d60ed4e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java @@ -19,7 +19,6 @@ package org.apache.pulsar.broker.loadbalance.extensions.filter; import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; @@ -30,14 +29,13 @@ import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; -import org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener; import org.testng.annotations.Test; /** * Unit test for {@link BrokerVersionFilter}. */ @Test(groups = "broker") -public class BrokerVersionFilterTest { +public class BrokerVersionFilterTest extends BrokerFilterTestBase { @Test @@ -115,26 +113,4 @@ public void testInvalidVersionString() throws BrokerFilterException { BrokerVersionFilter brokerVersionFilter = new BrokerVersionFilter(); brokerVersionFilter.filter(new HashMap<>(originalBrokers), getContext()); } - - public LoadManagerContext getContext() { - LoadManagerContext mockContext = mock(LoadManagerContext.class); - ServiceConfiguration configuration = new ServiceConfiguration(); - configuration.setPreferLaterVersions(true); - doReturn(configuration).when(mockContext).brokerConfiguration(); - return mockContext; - } - - public BrokerLookupData getLookupData(String version) { - String webServiceUrl = "http://localhost:8080"; - String webServiceUrlTls = "https://localhoss:8081"; - String pulsarServiceUrl = "pulsar://localhost:6650"; - String pulsarServiceUrlTls = "pulsar+ssl://localhost:6651"; - Map advertisedListeners = new HashMap<>(); - Map protocols = new HashMap<>(){{ - put("kafka", "9092"); - }}; - return new BrokerLookupData( - webServiceUrl, webServiceUrlTls, pulsarServiceUrl, - pulsarServiceUrlTls, advertisedListeners, protocols, true, true, version); - } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java index 3e4dc4cb1c07b..ee7e708667a32 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java @@ -65,6 +65,7 @@ void setup() { doReturn(Executors.newSingleThreadScheduledExecutor()).when(pulsar).getLoadManagerExecutor(); doReturn(pulsarStats).when(brokerService).getPulsarStats(); brokerStats = new BrokerStats(0); + brokerStats.topics = 6; brokerStats.bundleCount = 5; brokerStats.msgRateIn = 3; brokerStats.msgRateOut = 4; @@ -88,7 +89,7 @@ public void testGenerate() throws IllegalAccessException { doReturn(0l).when(pulsarStats).getUpdatedAt(); var target = new BrokerLoadDataReporter(pulsar, "", store); var expected = new BrokerLoadData(); - expected.update(usage, 1, 2, 3, 4, 5, config); + expected.update(usage, 1, 2, 3, 4, 5, 6, config); FieldUtils.writeDeclaredField(expected, "updatedAt", 0l, true); var actual = target.generateLoadData(); FieldUtils.writeDeclaredField(actual, "updatedAt", 0l, true); @@ -103,7 +104,7 @@ public void testReport() throws IllegalAccessException { var localData = (BrokerLoadData) FieldUtils.readDeclaredField(target, "localData", true); localData.setReportedAt(System.currentTimeMillis()); var lastData = (BrokerLoadData) FieldUtils.readDeclaredField(target, "lastData", true); - lastData.update(usage, 1, 2, 3, 4, 5, config); + lastData.update(usage, 1, 2, 3, 4, 5, 6, config); target.reportAsync(false); verify(store, times(0)).pushAsync(any(), any()); @@ -117,7 +118,7 @@ public void testReport() throws IllegalAccessException { target.reportAsync(false); verify(store, times(2)).pushAsync(eq("broker-1"), any()); - lastData.update(usage, 10000, 2, 3, 4, 5, config); + lastData.update(usage, 10000, 2, 3, 4, 5, 6, config); target.reportAsync(false); verify(store, times(3)).pushAsync(eq("broker-1"), any()); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java index bdf5f846267e6..709a1113f35e4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java @@ -121,7 +121,7 @@ public BrokerLoadData getCpuLoad(LoadManagerContext ctx, int load) { usage1.setDirectMemory(directMemory); usage1.setBandwidthIn(bandwidthIn); usage1.setBandwidthOut(bandwidthOut); - loadData.update(usage1, 1,2,3,4,5, + loadData.update(usage1, 1,2,3,4,5,6, ctx.brokerConfiguration()); return loadData; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeightTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeightTest.java index da0866c689746..ebc2424cada19 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeightTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeightTest.java @@ -168,7 +168,7 @@ public void testNoLoadDataBrokers() { private BrokerLoadData createBrokerData(LoadManagerContext ctx, double usage, double limit) { var brokerLoadData = new BrokerLoadData(); SystemResourceUsage usages = createUsage(usage, limit); - brokerLoadData.update(usages, 1, 1, 1, 1, 1, + brokerLoadData.update(usages, 1, 1, 1, 1, 1, 1, ctx.brokerConfiguration()); return brokerLoadData; } @@ -185,7 +185,7 @@ private SystemResourceUsage createUsage(double usage, double limit) { private void updateLoad(LoadManagerContext ctx, String broker, double usage) { ctx.brokerLoadDataStore().get(broker).get().update(createUsage(usage, 100.0), - 1, 1, 1, 1, 1, ctx.brokerConfiguration()); + 1, 1, 1, 1, 1, 1, ctx.brokerConfiguration()); } public static LoadManagerContext getContext() { From 0e6ef109322058e7834ca23d968f66f5632cf2b1 Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Thu, 2 Mar 2023 19:11:43 -0800 Subject: [PATCH 078/174] [improve][broker] PIP-192 Switched Assigning and Releasing state order in the transfer protocol (#19683) --- .../extensions/channel/ServiceUnitState.java | 9 +- .../channel/ServiceUnitStateChannelImpl.java | 188 ++++++++++-------- .../ServiceUnitStateCompactionStrategy.java | 28 ++- .../channel/ServiceUnitStateData.java | 16 +- .../channel/ServiceUnitStateChannelTest.java | 12 +- ...erviceUnitStateCompactionStrategyTest.java | 43 ++-- .../channel/ServiceUnitStateDataTest.java | 13 +- .../channel/ServiceUnitStateTest.java | 19 +- .../ServiceUnitStateCompactionTest.java | 14 +- 9 files changed, 191 insertions(+), 151 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitState.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitState.java index 92fef8f65992a..42ef55593ae1a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitState.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitState.java @@ -47,9 +47,9 @@ public enum ServiceUnitState { // when the topic is compacted in the middle of assign, transfer or split. Init, Set.of(Free, Owned, Assigning, Releasing, Splitting, Deleted), Free, Set.of(Assigning, Init), - Owned, Set.of(Assigning, Splitting, Releasing), - Assigning, Set.of(Owned, Releasing), - Releasing, Set.of(Owned, Free), + Owned, Set.of(Splitting, Releasing), + Assigning, Set.of(Owned), + Releasing, Set.of(Assigning, Free), Splitting, Set.of(Deleted), Deleted, Set.of(Init) ); @@ -67,4 +67,7 @@ public static boolean isInFlightState(ServiceUnitState state) { return inFlightStates.contains(state); } + public static boolean isActiveState(ServiceUnitState state) { + return inFlightStates.contains(state) || state == Owned; + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index 4819a54fcad73..cddaf92d22771 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -28,6 +28,8 @@ import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Owned; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Releasing; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Splitting; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.isActiveState; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.isInFlightState; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.ChannelState.Closed; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.ChannelState.Constructed; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.ChannelState.LeaderElectionServiceStarted; @@ -430,8 +432,11 @@ public CompletableFuture> getOwnerAsync(String serviceUnit) { ServiceUnitState state = state(data); ownerLookUpCounters.get(state).incrementAndGet(); switch (state) { - case Owned, Splitting -> { - return CompletableFuture.completedFuture(Optional.of(data.broker())); + case Owned -> { + return CompletableFuture.completedFuture(Optional.of(data.dstBroker())); + } + case Splitting -> { + return CompletableFuture.completedFuture(Optional.of(data.sourceBroker())); } case Assigning, Releasing -> { return deferGetOwnerRequest(serviceUnit).thenApply( @@ -490,16 +495,15 @@ public CompletableFuture publishUnloadEventAsync(Unload unload) { EventType eventType = Unload; eventCounters.get(eventType).getTotal().incrementAndGet(); String serviceUnit = unload.serviceUnit(); - CompletableFuture future; + ServiceUnitStateData next; if (isTransferCommand(unload)) { - future = pubAsync(serviceUnit, new ServiceUnitStateData( - Assigning, unload.destBroker().get(), unload.sourceBroker(), getNextVersionId(serviceUnit))); + next = new ServiceUnitStateData( + Releasing, unload.destBroker().get(), unload.sourceBroker(), getNextVersionId(serviceUnit)); } else { - future = pubAsync(serviceUnit, new ServiceUnitStateData( - Releasing, unload.sourceBroker(), getNextVersionId(serviceUnit))); + next = new ServiceUnitStateData( + Releasing, null, unload.sourceBroker(), getNextVersionId(serviceUnit)); } - - return future.whenComplete((__, ex) -> { + return pubAsync(serviceUnit, next).whenComplete((__, ex) -> { if (ex != null) { eventCounters.get(eventType).getFailure().incrementAndGet(); } @@ -515,7 +519,7 @@ public CompletableFuture publishSplitEventAsync(Split split) { eventCounters.get(eventType).getTotal().incrementAndGet(); String serviceUnit = split.serviceUnit(); ServiceUnitStateData next = - new ServiceUnitStateData(Splitting, split.sourceBroker(), getNextVersionId(serviceUnit)); + new ServiceUnitStateData(Splitting, null, split.sourceBroker(), getNextVersionId(serviceUnit)); return pubAsync(serviceUnit, next).whenComplete((__, ex) -> { if (ex != null) { eventCounters.get(eventType).getFailure().incrementAndGet(); @@ -553,7 +557,8 @@ private static boolean isTransferCommand(ServiceUnitStateData data) { if (data == null) { return false; } - return StringUtils.isNotEmpty(data.sourceBroker()); + return StringUtils.isNotEmpty(data.dstBroker()) + && StringUtils.isNotEmpty(data.sourceBroker()); } private static boolean isTransferCommand(Unload data) { @@ -612,45 +617,41 @@ lookupServiceAddress, getLogEventTag(data), serviceUnit, private void handleOwnEvent(String serviceUnit, ServiceUnitStateData data) { var getOwnerRequest = getOwnerRequests.remove(serviceUnit); if (getOwnerRequest != null) { - getOwnerRequest.complete(data.broker()); + getOwnerRequest.complete(data.dstBroker()); } - if (isTargetBroker(data.broker())) { + if (isTargetBroker(data.dstBroker())) { log(null, serviceUnit, data, null); } } private void handleAssignEvent(String serviceUnit, ServiceUnitStateData data) { - if (isTargetBroker(data.broker())) { + if (isTargetBroker(data.dstBroker())) { ServiceUnitStateData next = new ServiceUnitStateData( - isTransferCommand(data) ? Releasing : Owned, data.broker(), data.sourceBroker(), - getNextVersionId(data)); + Owned, data.dstBroker(), data.sourceBroker(), getNextVersionId(data)); pubAsync(serviceUnit, next) .whenComplete((__, e) -> log(e, serviceUnit, data, next)); } } private void handleReleaseEvent(String serviceUnit, ServiceUnitStateData data) { - if (isTransferCommand(data)) { - if (isTargetBroker(data.sourceBroker())) { - ServiceUnitStateData next = - new ServiceUnitStateData(Owned, data.broker(), data.sourceBroker(), getNextVersionId(data)); + if (isTargetBroker(data.sourceBroker())) { + ServiceUnitStateData next; + if (isTransferCommand(data)) { + next = new ServiceUnitStateData( + Assigning, data.dstBroker(), data.sourceBroker(), getNextVersionId(data)); // TODO: when close, pass message to clients to connect to the new broker - closeServiceUnit(serviceUnit) - .thenCompose(__ -> pubAsync(serviceUnit, next)) - .whenComplete((__, e) -> log(e, serviceUnit, data, next)); - } - } else { - if (isTargetBroker(data.broker())) { - ServiceUnitStateData next = new ServiceUnitStateData(Free, data.broker(), getNextVersionId(data)); - closeServiceUnit(serviceUnit) - .thenCompose(__ -> pubAsync(serviceUnit, next)) - .whenComplete((__, e) -> log(e, serviceUnit, data, next)); + } else { + next = new ServiceUnitStateData( + Free, null, data.sourceBroker(), getNextVersionId(data)); } + closeServiceUnit(serviceUnit) + .thenCompose(__ -> pubAsync(serviceUnit, next)) + .whenComplete((__, e) -> log(e, serviceUnit, data, next)); } } private void handleSplitEvent(String serviceUnit, ServiceUnitStateData data) { - if (isTargetBroker(data.broker())) { + if (isTargetBroker(data.sourceBroker())) { splitServiceUnit(serviceUnit, data) .whenComplete((__, e) -> log(e, serviceUnit, data, null)); } @@ -661,7 +662,7 @@ private void handleFreeEvent(String serviceUnit, ServiceUnitStateData data) { if (getOwnerRequest != null) { getOwnerRequest.complete(null); } - if (isTargetBroker(data.broker())) { + if (isTargetBroker(data.sourceBroker())) { log(null, serviceUnit, data, null); } } @@ -671,7 +672,7 @@ private void handleDeleteEvent(String serviceUnit, ServiceUnitStateData data) { if (getOwnerRequest != null) { getOwnerRequest.completeExceptionally(new IllegalStateException(serviceUnit + "has been deleted.")); } - if (isTargetBroker(data.broker())) { + if (isTargetBroker(data.sourceBroker())) { log(null, serviceUnit, data, null); } } @@ -795,7 +796,7 @@ protected void splitServiceUnitOnceAndRetry(NamespaceService namespaceService, updateFuture.completeExceptionally(new BrokerServiceException.ServiceUnitNotReadyException(msg)); return; } - ServiceUnitStateData next = new ServiceUnitStateData(Owned, data.broker(), VERSION_ID_INIT); + ServiceUnitStateData next = new ServiceUnitStateData(Owned, data.sourceBroker(), VERSION_ID_INIT); NamespaceBundles targetNsBundle = splitBundlesPair.getLeft(); List splitBundles = Collections.unmodifiableList(splitBundlesPair.getRight()); List successPublishedBundles = @@ -833,7 +834,8 @@ protected void splitServiceUnitOnceAndRetry(NamespaceService namespaceService, updateFuture.thenAccept(r -> { // Delete the old bundle - pubAsync(serviceUnit, new ServiceUnitStateData(Deleted, data.broker(), getNextVersionId(data))) + pubAsync(serviceUnit, new ServiceUnitStateData( + Deleted, null, data.sourceBroker(), getNextVersionId(data))) .thenRun(() -> { // Update bundled_topic cache for load-report-generation pulsar.getBrokerService().refreshTopicToStatsMaps(bundle); @@ -983,18 +985,19 @@ private void doCleanup(String broker) throws ExecutionException, InterruptedExce long totalCleanupErrorCntStart = totalCleanupErrorCnt.get(); var availableBrokers = new HashSet<>(brokerRegistry.getAvailableBrokersAsync() .get(inFlightStateWaitingTimeInMillis, MILLISECONDS)); + for (var etr : tableview.entrySet()) { var stateData = etr.getValue(); var serviceUnit = etr.getKey(); var state = state(stateData); - if (StringUtils.equals(broker, stateData.broker())) { - if (ServiceUnitState.isInFlightState(state) || state == Owned) { + if (StringUtils.equals(broker, stateData.dstBroker())) { + if (isActiveState(state)) { overrideOwnership(serviceUnit, stateData, availableBrokers); orphanServiceUnitCleanupCnt++; } } else if (StringUtils.equals(broker, stateData.sourceBroker())) { - if (ServiceUnitState.isInFlightState(state)) { + if (isInFlightState(state)) { overrideOwnership(serviceUnit, stateData, availableBrokers); orphanServiceUnitCleanupCnt++; } @@ -1026,39 +1029,57 @@ private void doCleanup(String broker) throws ExecutionException, InterruptedExce cleanupJobs.remove(broker); } - private Optional getOverrideStateData(String serviceUnit, ServiceUnitStateData orphanData, - Set availableBrokers, - LoadManagerContext context) { + private Optional getRollForwardStateData( + Set availableBrokers, LoadManagerContext context, long nextVersionId) { + Optional selectedBroker = brokerSelector.select(availableBrokers, null, context); + if (selectedBroker.isEmpty()) { + return Optional.empty(); + } + return Optional.of(new ServiceUnitStateData(Owned, selectedBroker.get(), true, nextVersionId)); + } + + private Optional getOverrideInFlightStateData( + String serviceUnit, ServiceUnitStateData orphanData, + Set availableBrokers, + LoadManagerContext context) { long nextVersionId = getNextVersionId(orphanData); - if (isTransferCommand(orphanData)) { - // rollback to the src - return Optional.of(new ServiceUnitStateData(Owned, orphanData.sourceBroker(), true, nextVersionId)); - } else if (orphanData.state() == Assigning) { // assign - // roll-forward to another broker - Optional selectedBroker = brokerSelector.select(availableBrokers, null, context); - if (selectedBroker.isEmpty()) { - return Optional.empty(); + var state = orphanData.state(); + switch (state) { + case Assigning: { + return getRollForwardStateData(availableBrokers, context, nextVersionId); + } + case Splitting, Releasing: { + if (availableBrokers.contains(orphanData.sourceBroker())) { + // rollback to the src + return Optional.of(new ServiceUnitStateData(Owned, orphanData.sourceBroker(), true, nextVersionId)); + } else { + return getRollForwardStateData(availableBrokers, context, nextVersionId); + } + } + default: { + var msg = String.format("Failed to get the overrideStateData from serviceUnit=%s, orphanData=%s", + serviceUnit, orphanData); + log.error(msg); + throw new IllegalStateException(msg); } - return Optional.of(new ServiceUnitStateData(Owned, selectedBroker.get(), true, nextVersionId)); - } else if (orphanData.state() == Splitting || orphanData.state() == Releasing) { - // rollback to the target broker for split and unload - return Optional.of(new ServiceUnitStateData(Owned, orphanData.broker(), true, nextVersionId)); - } else { - var msg = String.format("Failed to get the overrideStateData from serviceUnit=%s, orphanData=%s", - serviceUnit, orphanData); - log.error(msg); - throw new IllegalStateException(msg); } } @VisibleForTesting protected void monitorOwnerships(List brokers) { if (!isChannelOwner()) { - log.warn("This broker is not the leader now. Skipping ownership monitor"); + log.warn("This broker is not the leader now. Skipping ownership monitor."); return; } + if (brokers == null || brokers.size() == 0) { - log.error("no active brokers found. Skipping the ownership monitor run."); + log.error("no active brokers found. Skipping ownership monitor."); + return; + } + + var metadataState = getMetadataState(); + if (metadataState != Stable) { + log.warn("metadata state:{} is not Stable. Skipping ownership monitor.", metadataState); return; } @@ -1071,34 +1092,35 @@ protected void monitorOwnerships(List brokers) { int orphanServiceUnitCleanupCnt = 0; long totalCleanupErrorCntStart = totalCleanupErrorCnt.get(); long now = System.currentTimeMillis(); - for (Map.Entry etr : tableview.entrySet()) { + for (var etr : tableview.entrySet()) { String serviceUnit = etr.getKey(); ServiceUnitStateData stateData = etr.getValue(); - String broker = stateData.broker(); + String dstBroker = stateData.dstBroker(); + String srcBroker = stateData.sourceBroker(); var state = stateData.state(); - if (!activeBrokers.contains(broker)) { - inactiveBrokers.add(stateData.broker()); - } else if (state != Owned - && now - stateData.timestamp() > inFlightStateWaitingTimeInMillis) { - if (state == Deleted || state == Free) { - if (now - stateData.timestamp() - > semiTerminalStateWaitingTimeInMillis) { - log.info("Found semi-terminal states to tombstone" - + " serviceUnit:{}, stateData:{}", serviceUnit, stateData); - tombstoneAsync(serviceUnit).whenComplete((__, e) -> { - if (e != null) { - log.error("Failed cleaning the ownership serviceUnit:{}, stateData:{}, " - + "cleanupErrorCnt:{}.", - serviceUnit, stateData, - totalCleanupErrorCnt.incrementAndGet() - totalCleanupErrorCntStart, e); - } - }); - serviceUnitTombstoneCleanupCnt++; - } - } else { + + if (isActiveState(state)) { + if (StringUtils.isNotBlank(srcBroker) && !activeBrokers.contains(srcBroker)) { + inactiveBrokers.add(srcBroker); + } else if (StringUtils.isNotBlank(dstBroker) && !activeBrokers.contains(dstBroker)) { + inactiveBrokers.add(dstBroker); + } else if (isInFlightState(state) + && now - stateData.timestamp() > inFlightStateWaitingTimeInMillis) { log.warn("Found orphan serviceUnit:{}, stateData:{}", serviceUnit, stateData); orphanServiceUnits.put(serviceUnit, stateData); } + } else if (now - stateData.timestamp() > semiTerminalStateWaitingTimeInMillis) { + log.info("Found semi-terminal states to tombstone" + + " serviceUnit:{}, stateData:{}", serviceUnit, stateData); + tombstoneAsync(serviceUnit).whenComplete((__, e) -> { + if (e != null) { + log.error("Failed cleaning the ownership serviceUnit:{}, stateData:{}, " + + "cleanupErrorCnt:{}.", + serviceUnit, stateData, + totalCleanupErrorCnt.incrementAndGet() - totalCleanupErrorCntStart, e); + } + }); + serviceUnitTombstoneCleanupCnt++; } } @@ -1112,7 +1134,7 @@ protected void monitorOwnerships(List brokers) { for (var etr : orphanServiceUnits.entrySet()) { var orphanServiceUnit = etr.getKey(); var orphanData = etr.getValue(); - var overrideData = getOverrideStateData( + var overrideData = getOverrideInFlightStateData( orphanServiceUnit, orphanData, activeBrokers, context); if (overrideData.isPresent()) { pubAsync(orphanServiceUnit, overrideData.get()).whenComplete((__, e) -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java index 8af0f0c027da4..ceb3ea3e9cb6c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java @@ -71,22 +71,21 @@ public boolean shouldKeepLeft(ServiceUnitStateData from, ServiceUnitStateData to switch (prevState) { case Owned: switch (state) { - case Assigning: - return invalidTransfer(from, to); case Splitting: + return isNotBlank(to.dstBroker()) + || !from.dstBroker().equals(to.sourceBroker()); case Releasing: - return isNotBlank(to.sourceBroker()) || targetNotEquals(from, to); + return invalidUnload(from, to); } case Assigning: switch (state) { - case Releasing: - return isBlank(to.sourceBroker()) || notEquals(from, to); case Owned: - return isNotBlank(to.sourceBroker()) || targetNotEquals(from, to); + return notEquals(from, to); } case Releasing: switch (state) { - case Owned: + case Assigning: + return isBlank(to.dstBroker()) || notEquals(from, to); case Free: return notEquals(from, to); } @@ -98,24 +97,21 @@ public boolean shouldKeepLeft(ServiceUnitStateData from, ServiceUnitStateData to case Free: switch (state) { case Assigning: - return isNotBlank(to.sourceBroker()); + return isNotBlank(to.sourceBroker()) || isBlank(to.dstBroker()); } } } return false; } - private boolean targetNotEquals(ServiceUnitStateData from, ServiceUnitStateData to) { - return !from.broker().equals(to.broker()); - } - private boolean notEquals(ServiceUnitStateData from, ServiceUnitStateData to) { - return !from.broker().equals(to.broker()) + return !StringUtils.equals(from.dstBroker(), to.dstBroker()) || !StringUtils.equals(from.sourceBroker(), to.sourceBroker()); } - private boolean invalidTransfer(ServiceUnitStateData from, ServiceUnitStateData to) { - return !from.broker().equals(to.sourceBroker()) - || from.broker().equals(to.broker()); + private boolean invalidUnload(ServiceUnitStateData from, ServiceUnitStateData to) { + return isBlank(to.sourceBroker()) + || !from.dstBroker().equals(to.sourceBroker()) + || from.dstBroker().equals(to.dstBroker()); } } \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java index ef25acff10a4b..c26dce83a4434 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java @@ -27,25 +27,25 @@ */ public record ServiceUnitStateData( - ServiceUnitState state, String broker, String sourceBroker, boolean force, long timestamp, long versionId) { + ServiceUnitState state, String dstBroker, String sourceBroker, boolean force, long timestamp, long versionId) { public ServiceUnitStateData { Objects.requireNonNull(state); - if (StringUtils.isBlank(broker)) { + if (StringUtils.isBlank(dstBroker) && StringUtils.isBlank(sourceBroker)) { throw new IllegalArgumentException("Empty broker"); } } - public ServiceUnitStateData(ServiceUnitState state, String broker, String sourceBroker, long versionId) { - this(state, broker, sourceBroker, false, System.currentTimeMillis(), versionId); + public ServiceUnitStateData(ServiceUnitState state, String dstBroker, String sourceBroker, long versionId) { + this(state, dstBroker, sourceBroker, false, System.currentTimeMillis(), versionId); } - public ServiceUnitStateData(ServiceUnitState state, String broker, long versionId) { - this(state, broker, null, false, System.currentTimeMillis(), versionId); + public ServiceUnitStateData(ServiceUnitState state, String dstBroker, long versionId) { + this(state, dstBroker, null, false, System.currentTimeMillis(), versionId); } - public ServiceUnitStateData(ServiceUnitState state, String broker, boolean force, long versionId) { - this(state, broker, null, force, System.currentTimeMillis(), versionId); + public ServiceUnitStateData(ServiceUnitState state, String dstBroker, boolean force, long versionId) { + this(state, dstBroker, null, force, System.currentTimeMillis(), versionId); } public static ServiceUnitState state(ServiceUnitStateData data) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java index 6aa8fe387605b..64a5f63196bea 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java @@ -152,13 +152,15 @@ protected void setup() throws Exception { } @BeforeMethod - protected void initTableViews() throws Exception { + protected void initChannels() throws Exception { cleanTableView(channel1, bundle); cleanTableView(channel2, bundle); cleanOwnershipMonitorCounters(channel1); cleanOwnershipMonitorCounters(channel2); cleanOpsCounters(channel1); cleanOpsCounters(channel2); + cleanMetadataState(channel1); + cleanMetadataState(channel2); } @@ -491,6 +493,7 @@ public void transferTestWhenDestBrokerFails() assertEquals(0, getOwnerRequests2.size()); // recovered, check the monitor update state : Assigned -> Owned + doReturn(Optional.of(lookupServiceAddress1)).when(brokerSelector).select(any(), any(), any()); FieldUtils.writeDeclaredField(channel2, "producer", producer, true); FieldUtils.writeDeclaredField(channel1, "inFlightStateWaitingTimeInMillis", 1 , true); @@ -937,7 +940,7 @@ public void ownerLookupCountTests() throws IllegalAccessException { channel1.getOwnerAsync(bundle); channel1.getOwnerAsync(bundle); - overrideTableView(channel1, bundle, new ServiceUnitStateData(Splitting, "b1", 1)); + overrideTableView(channel1, bundle, new ServiceUnitStateData(Splitting, null, "b1", 1)); channel1.getOwnerAsync(bundle); overrideTableView(channel1, bundle, new ServiceUnitStateData(Free, "b1", 1)); @@ -1363,6 +1366,11 @@ private void cleanOwnershipMonitorCounters(ServiceUnitStateChannel channel) thro FieldUtils.writeDeclaredField(channel, "totalInactiveBrokerCleanupCancelledCnt", 0, true); } + private void cleanMetadataState(ServiceUnitStateChannel channel) throws IllegalAccessException { + channel.handleMetadataSessionEvent(SessionReestablished); + FieldUtils.writeDeclaredField(channel, "lastMetadataSessionEventTimestamp", 0L, true); + } + private static long getCleanupMetric(ServiceUnitStateChannel channel, String metric) throws IllegalAccessException { Object var = FieldUtils.readDeclaredField(channel, metric, true); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java index 0cd05d8bd7559..64964826af652 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java @@ -71,19 +71,19 @@ public void testVersionId(){ assertFalse(strategy.shouldKeepLeft( new ServiceUnitStateData(Owned, dst, src, 10), - new ServiceUnitStateData(Assigning, "broker2", dst, 11))); + new ServiceUnitStateData(Releasing, "broker2", dst, 11))); assertFalse(strategy.shouldKeepLeft( new ServiceUnitStateData(Owned, dst, src, Long.MAX_VALUE), - new ServiceUnitStateData(Assigning, "broker2", dst, Long.MAX_VALUE + 1))); + new ServiceUnitStateData(Releasing, "broker2", dst, Long.MAX_VALUE + 1))); assertFalse(strategy.shouldKeepLeft( new ServiceUnitStateData(Owned, dst, src, Long.MAX_VALUE + 1), - new ServiceUnitStateData(Assigning, "broker2", dst, Long.MAX_VALUE + 2))); + new ServiceUnitStateData(Releasing, "broker2", dst, Long.MAX_VALUE + 2))); assertTrue(strategy.shouldKeepLeft( new ServiceUnitStateData(Owned, dst, src, 10), - new ServiceUnitStateData(Assigning, "broker2", dst, 5))); + new ServiceUnitStateData(Releasing, "broker2", dst, 5))); } @@ -133,41 +133,42 @@ public void testTransitionsAndBrokers() { assertTrue(strategy.shouldKeepLeft(data(Assigning, "dst1"), data2(Owned, "dst2"))); assertTrue(strategy.shouldKeepLeft(data(Assigning, dst), data2(Owned, src, dst))); assertFalse(strategy.shouldKeepLeft(data(Assigning, dst), data2(Owned, dst))); + assertFalse(strategy.shouldKeepLeft(data(Assigning, src, dst), data2(Owned, src, dst))); assertTrue(strategy.shouldKeepLeft(data(Assigning, src, dst), data2(Releasing, dst))); - assertTrue(strategy.shouldKeepLeft(data(Assigning, src, "dst1"), data2(Releasing, src, "dst2"))); - assertTrue(strategy.shouldKeepLeft(data(Assigning, "src1", dst), data2(Releasing, "src2", dst))); - assertFalse(strategy.shouldKeepLeft(data(Assigning, src, dst), data2(Releasing, src, dst))); + assertTrue(strategy.shouldKeepLeft(data(Assigning, src, dst), data2(Releasing, src, dst))); assertTrue(strategy.shouldKeepLeft(data(Assigning), data2(Splitting, dst))); assertTrue(strategy.shouldKeepLeft(data(Assigning), data2(Deleted, dst))); assertTrue(strategy.shouldKeepLeft(data(Owned), data2(Init))); assertTrue(strategy.shouldKeepLeft(data(Owned), data2(Free))); - assertTrue(strategy.shouldKeepLeft(data(Owned, src, "dst1"), data2(Assigning, src, "dst2"))); - assertTrue(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Assigning, dst))); - assertTrue(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Assigning, src, dst))); - assertTrue(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Assigning, dst, dst))); - assertFalse(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Assigning, dst, "dst1"))); + assertTrue(strategy.shouldKeepLeft(data(Owned), data2(Assigning))); assertTrue(strategy.shouldKeepLeft(data(Owned), data2(Owned))); assertTrue(strategy.shouldKeepLeft(data(Owned), data2(Releasing, dst))); assertTrue(strategy.shouldKeepLeft(data(Owned, src, "dst1"), data2(Releasing, src, "dst2"))); assertTrue(strategy.shouldKeepLeft(data(Owned, "dst1"), data2(Releasing, "dst2"))); - assertFalse(strategy.shouldKeepLeft(data(Owned, dst), data2(Releasing, dst))); - assertFalse(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Releasing, dst))); + assertTrue(strategy.shouldKeepLeft(data(Owned, dst), data2(Releasing, dst))); + assertTrue(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Releasing, null, dst))); + assertTrue(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Releasing, src, null))); + assertFalse(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Releasing, dst, null))); + assertTrue(strategy.shouldKeepLeft(data(Owned, src, "dst1"), data2(Releasing, src, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Owned, "src1", dst), data2(Releasing, "src2", dst))); + assertTrue(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Releasing, src, dst))); + assertFalse(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Releasing, dst, "dst2"))); assertTrue(strategy.shouldKeepLeft(data(Owned, src, "dst1"), data2(Splitting, src, "dst2"))); assertTrue(strategy.shouldKeepLeft(data(Owned, "dst1"), data2(Splitting, "dst2"))); - assertFalse(strategy.shouldKeepLeft(data(Owned, dst), data2(Splitting, dst))); - assertFalse(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Splitting, dst))); + assertFalse(strategy.shouldKeepLeft(data(Owned, dst), data2(Splitting, dst, null))); + assertFalse(strategy.shouldKeepLeft(data(Owned, src, dst), data2(Splitting, dst, null))); assertTrue(strategy.shouldKeepLeft(data(Owned), data2(Deleted, dst))); assertTrue(strategy.shouldKeepLeft(data(Releasing), data2(Init))); assertFalse(strategy.shouldKeepLeft(data(Releasing), data2(Free))); assertTrue(strategy.shouldKeepLeft(data(Releasing, "dst1"), data2(Free, "dst2"))); assertTrue(strategy.shouldKeepLeft(data(Releasing, "src1", dst), data2(Free, "src2", dst))); - assertTrue(strategy.shouldKeepLeft(data(Releasing), data2(Assigning))); - assertTrue(strategy.shouldKeepLeft(data(Releasing, "dst1"), data2(Owned, "dst2"))); - assertTrue(strategy.shouldKeepLeft(data(Releasing, src, "dst1"), data2(Owned, src, "dst2"))); - assertTrue(strategy.shouldKeepLeft(data(Releasing, "src1", dst), data2(Owned, "src2", dst))); - assertFalse(strategy.shouldKeepLeft(data(Releasing, src, dst), data2(Owned, src, dst))); + assertTrue(strategy.shouldKeepLeft(data(Releasing, src, "dst1"), data2(Assigning, src, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Releasing, src, "dst1"), data2(Assigning, src, "dst2"))); + assertTrue(strategy.shouldKeepLeft(data(Releasing, "src1", dst), data2(Assigning, "src2", dst))); + assertFalse(strategy.shouldKeepLeft(data(Releasing, src, dst), data2(Assigning, src, dst))); + assertTrue(strategy.shouldKeepLeft(data(Releasing), data2(Owned))); assertTrue(strategy.shouldKeepLeft(data(Releasing), data2(Releasing))); assertTrue(strategy.shouldKeepLeft(data(Releasing), data2(Splitting))); assertTrue(strategy.shouldKeepLeft(data(Releasing), data2(Deleted, dst))); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataTest.java index a48e2a4db8b37..9bfa8ad3ac5f7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataTest.java @@ -35,16 +35,15 @@ public class ServiceUnitStateDataTest { public void testConstructors() throws InterruptedException { ServiceUnitStateData data1 = new ServiceUnitStateData(Owned, "A", 1); assertEquals(data1.state(), Owned); - assertEquals(data1.broker(), "A"); + assertEquals(data1.dstBroker(), "A"); assertNull(data1.sourceBroker()); assertThat(data1.timestamp()).isGreaterThan(0); - ; Thread.sleep(10); ServiceUnitStateData data2 = new ServiceUnitStateData(Assigning, "A", "B", 1); assertEquals(data2.state(), Assigning); - assertEquals(data2.broker(), "A"); + assertEquals(data2.dstBroker(), "A"); assertEquals(data2.sourceBroker(), "B"); assertThat(data2.timestamp()).isGreaterThan(data1.timestamp()); } @@ -55,13 +54,13 @@ public void testNullState() { } @Test(expectedExceptions = IllegalArgumentException.class) - public void testNullBroker() { - new ServiceUnitStateData(Owned, null, 1); + public void testNullBrokers() { + new ServiceUnitStateData(Owned, null, null, 1); } @Test(expectedExceptions = IllegalArgumentException.class) - public void testEmptyBroker() { - new ServiceUnitStateData(Owned, "", 1); + public void testEmptyBrokers() { + new ServiceUnitStateData(Owned, "", "", 1); } @Test diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTest.java index f5f1fe7bc575f..620266aee46a1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTest.java @@ -42,6 +42,17 @@ public void testInFlights() { assertFalse(ServiceUnitState.isInFlightState(Deleted)); } + @Test + public void testActive() { + assertFalse(ServiceUnitState.isActiveState(Init)); + assertFalse(ServiceUnitState.isActiveState(Free)); + assertTrue(ServiceUnitState.isActiveState(Owned)); + assertTrue(ServiceUnitState.isActiveState(Assigning)); + assertTrue(ServiceUnitState.isActiveState(Releasing)); + assertTrue(ServiceUnitState.isActiveState(Splitting)); + assertFalse(ServiceUnitState.isActiveState(Deleted)); + } + @Test public void testTransitions() { @@ -65,13 +76,13 @@ public void testTransitions() { assertFalse(ServiceUnitState.isValidTransition(Assigning, Free)); assertFalse(ServiceUnitState.isValidTransition(Assigning, Assigning)); assertTrue(ServiceUnitState.isValidTransition(Assigning, Owned)); - assertTrue(ServiceUnitState.isValidTransition(Assigning, Releasing)); + assertFalse(ServiceUnitState.isValidTransition(Assigning, Releasing)); assertFalse(ServiceUnitState.isValidTransition(Assigning, Splitting)); assertFalse(ServiceUnitState.isValidTransition(Assigning, Deleted)); assertFalse(ServiceUnitState.isValidTransition(Owned, Init)); assertFalse(ServiceUnitState.isValidTransition(Owned, Free)); - assertTrue(ServiceUnitState.isValidTransition(Owned, Assigning)); + assertFalse(ServiceUnitState.isValidTransition(Owned, Assigning)); assertFalse(ServiceUnitState.isValidTransition(Owned, Owned)); assertTrue(ServiceUnitState.isValidTransition(Owned, Releasing)); assertTrue(ServiceUnitState.isValidTransition(Owned, Splitting)); @@ -79,8 +90,8 @@ public void testTransitions() { assertFalse(ServiceUnitState.isValidTransition(Releasing, Init)); assertTrue(ServiceUnitState.isValidTransition(Releasing, Free)); - assertFalse(ServiceUnitState.isValidTransition(Releasing, Assigning)); - assertTrue(ServiceUnitState.isValidTransition(Releasing, Owned)); + assertTrue(ServiceUnitState.isValidTransition(Releasing, Assigning)); + assertFalse(ServiceUnitState.isValidTransition(Releasing, Owned)); assertFalse(ServiceUnitState.isValidTransition(Releasing, Releasing)); assertFalse(ServiceUnitState.isValidTransition(Releasing, Splitting)); assertFalse(ServiceUnitState.isValidTransition(Releasing, Deleted)); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java index 543b7c629ac5f..02812898dc49a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java @@ -570,12 +570,12 @@ public void testSlowTableviewAfterCompaction() throws Exception { long versionId = 1; producer.newMessage().key(bundle).value(new ServiceUnitStateData(Owned, src, versionId++)).send(); for (int i = 0; i < 3; i++) { - var assignedStateData = new ServiceUnitStateData(Assigning, dst, src, versionId++); - producer.newMessage().key(bundle).value(assignedStateData).send(); - producer.newMessage().key(bundle).value(assignedStateData).send(); var releasedStateData = new ServiceUnitStateData(Releasing, dst, src, versionId++); producer.newMessage().key(bundle).value(releasedStateData).send(); producer.newMessage().key(bundle).value(releasedStateData).send(); + var assignedStateData = new ServiceUnitStateData(Assigning, dst, src, versionId++); + producer.newMessage().key(bundle).value(assignedStateData).send(); + producer.newMessage().key(bundle).value(assignedStateData).send(); var ownedStateData = new ServiceUnitStateData(Owned, dst, src, versionId++); producer.newMessage().key(bundle).value(ownedStateData).send(); producer.newMessage().key(bundle).value(ownedStateData).send(); @@ -726,7 +726,7 @@ public void testWholeBatchCompactedOut() throws Exception { .subscriptionName("sub1").readCompacted(true).subscribe()) { Message message = consumer.receive(); Assert.assertEquals(message.getKey(), "key1"); - Assert.assertEquals(new String(message.getValue().broker()), "my-message-4"); + Assert.assertEquals(new String(message.getValue().dstBroker()), "my-message-4"); } } @@ -860,7 +860,7 @@ public void testCompactMultipleTimesWithoutEmptyMessage() Message m1 = consumer.receive(); assertNotNull(m1); assertEquals(m1.getKey(), key); - assertEquals(m1.getValue().broker(), "19"); + assertEquals(m1.getValue().dstBroker(), "19"); Message none = consumer.receive(2, TimeUnit.SECONDS); assertNull(none); } @@ -908,7 +908,7 @@ public void testReadUnCompacted() Message received = consumer.receive(); assertNotNull(received); assertEquals(received.getKey(), key); - assertEquals(received.getValue().broker(), i + 9 + ""); + assertEquals(received.getValue().dstBroker(), i + 9 + ""); consumer.acknowledge(received); } Message none = consumer.receive(2, TimeUnit.SECONDS); @@ -946,7 +946,7 @@ public void testReadUnCompacted() Message received = consumer.receive(); assertNotNull(received); assertEquals(received.getKey(), key); - assertEquals(received.getValue().broker(), i + 20 + ""); + assertEquals(received.getValue().dstBroker(), i + 20 + ""); consumer.acknowledge(received); } Message none = consumer.receive(2, TimeUnit.SECONDS); From d17bdf85d184c37cfcc16cfcb8df6f2e198e6fec Mon Sep 17 00:00:00 2001 From: YANGLiiN Date: Fri, 3 Mar 2023 19:29:51 +0800 Subject: [PATCH 079/174] [fix][admin] fix create-subscription command --subscription description (#19591) --- .../java/org/apache/pulsar/admin/cli/CmdPersistentTopics.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdPersistentTopics.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdPersistentTopics.java index 5dbb3935ce98a..6e59be39c018b 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdPersistentTopics.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdPersistentTopics.java @@ -502,7 +502,7 @@ private class CreateSubscription extends CliCommand { private java.util.List params; @Parameter(names = { "-s", - "--subscription" }, description = "Subscription to reset position on", required = true) + "--subscription" }, description = "Subscription name", required = true) private String subscriptionName; @Parameter(names = { "--messageId", From bce80d9b525d1b21f8440185e7529d492db420eb Mon Sep 17 00:00:00 2001 From: Anonymitaet <50226895+Anonymitaet@users.noreply.github.com> Date: Sat, 4 Mar 2023 00:22:30 +0800 Subject: [PATCH 080/174] [fix][doc] correct description of REST API doc (namespace) (#19580) --- .../main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java index 12ff229c2f0ed..55766c173f71f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java @@ -2569,7 +2569,7 @@ public void getProperty( @DELETE @Path("/{tenant}/{namespace}/property/{key}") - @ApiOperation(value = "Get property value for a given key on a namespace.") + @ApiOperation(value = "Remove property value for a given key on a namespace.") @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Tenant or namespace doesn't exist"), }) public void removeProperty( From e75def80ff2f366919b55cfd42f1dfab3f719b88 Mon Sep 17 00:00:00 2001 From: Rajan Dhabalia Date: Sat, 4 Mar 2023 08:46:50 -0800 Subject: [PATCH 081/174] [fix][cli] Fix Pulsar admin tool is ignoring tls-trust-cert path arg (#19696) --- .../internal/PulsarAdminBuilderImpl.java | 2 ++ .../pulsar/admin/cli/PulsarAdminSupplier.java | 5 ++- .../pulsar/admin/cli/PulsarAdminTool.java | 12 +++++-- .../apache/pulsar/admin/cli/TestRunMain.java | 35 +++++++++++++++++++ 4 files changed, 50 insertions(+), 4 deletions(-) diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImpl.java index 3e7ee472e464b..009fa67fbaa29 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImpl.java @@ -21,6 +21,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; +import lombok.Getter; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminBuilder; @@ -33,6 +34,7 @@ public class PulsarAdminBuilderImpl implements PulsarAdminBuilder { + @Getter protected ClientConfigurationData conf; private ClassLoader clientBuilderClassLoader = null; diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminSupplier.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminSupplier.java index 6aa8d2b9c610a..764dc9de5dfdd 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminSupplier.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminSupplier.java @@ -53,7 +53,7 @@ static RootParamsKey fromRootParams(PulsarAdminTool.RootParams params) { } } - private final PulsarAdminBuilder adminBuilder; + protected final PulsarAdminBuilder adminBuilder; private RootParamsKey currentParamsKey; private PulsarAdmin admin; @@ -103,6 +103,9 @@ private static void applyRootParamsToAdminBuilder(PulsarAdminBuilder adminBuilde if (isNotBlank(rootParams.tlsProvider)) { adminBuilder.sslProvider(rootParams.tlsProvider); } + if (isNotBlank(rootParams.tlsTrustCertsFilePath)) { + adminBuilder.tlsTrustCertsFilePath(rootParams.tlsTrustCertsFilePath); + } } } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java index 7a7a4c4b44449..ca0a8a055cfc9 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java @@ -45,7 +45,7 @@ public class PulsarAdminTool { - private static boolean allowSystemExit = true; + protected static boolean allowSystemExit = true; private static int lastExitCode = Integer.MIN_VALUE; @@ -54,7 +54,7 @@ public class PulsarAdminTool { protected JCommander jcommander; protected RootParams rootParams; private final Properties properties; - private PulsarAdminSupplier pulsarAdminSupplier; + protected PulsarAdminSupplier pulsarAdminSupplier; @Getter public static class RootParams { @@ -277,11 +277,16 @@ protected boolean run(String[] args) { } public static void main(String[] args) throws Exception { + execute(args); + } + + @VisibleForTesting + public static PulsarAdminTool execute(String[] args) throws Exception { lastExitCode = 0; if (args.length == 0) { System.out.println("Usage: pulsar-admin CONF_FILE_PATH [options] [command] [command options]"); exit(0); - return; + return null; } String configFile = args[0]; Properties properties = new Properties(); @@ -299,6 +304,7 @@ public static void main(String[] args) throws Exception { } else { exit(1); } + return tool; } private static void exit(int code) { diff --git a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestRunMain.java b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestRunMain.java index 3ac7abf0efc41..c3dbd1cdc7c85 100644 --- a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestRunMain.java +++ b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestRunMain.java @@ -19,9 +19,15 @@ package org.apache.pulsar.admin.cli; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import java.io.File; +import java.io.FileOutputStream; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; import java.nio.file.Files; import java.nio.file.Path; import java.util.Properties; +import org.apache.pulsar.client.admin.internal.PulsarAdminBuilderImpl; import org.testng.annotations.Test; public class TestRunMain { @@ -75,4 +81,33 @@ public void testRunWithTlsProviderFlagWithConfigFile() throws Exception { "tenants"}); assertEquals(pulsarAdminTool.rootParams.tlsProvider, "OPENSSL"); } + + @Test + public void testMainArgs() throws Exception { + String tlsTrustCertsFilePathInFile = "ca-file.cert"; + String tlsTrustCertsFilePathInArg = "ca-arg.cert"; + File testConfigFile = new File("tmp." + System.currentTimeMillis() + ".properties"); + if (testConfigFile.exists()) { + testConfigFile.delete(); + } + PrintWriter printWriter = new PrintWriter(new OutputStreamWriter(new FileOutputStream(testConfigFile))); + printWriter.println("tlsTrustCertsFilePath=" + tlsTrustCertsFilePathInFile); + printWriter.println("tlsAllowInsecureConnection=" + false); + printWriter.println("tlsEnableHostnameVerification=" + false); + + printWriter.close(); + testConfigFile.deleteOnExit(); + + String argStrTemp = "%s %s --admin-url https://url:4443 " + "topics stats persistent://prop/cluster/ns/t1"; + boolean prevValue = PulsarAdminTool.allowSystemExit; + PulsarAdminTool.allowSystemExit = false; + + String argStr = argStr = argStrTemp.format(argStrTemp, testConfigFile.getAbsolutePath(), + "--tls-trust-cert-path " + tlsTrustCertsFilePathInArg); + PulsarAdminTool tool = PulsarAdminTool.execute(argStr.split(" ")); + assertNotNull(tool); + PulsarAdminBuilderImpl builder = (PulsarAdminBuilderImpl) tool.pulsarAdminSupplier.adminBuilder; + assertEquals(builder.getConf().getTlsTrustCertsFilePath(), tlsTrustCertsFilePathInArg); + PulsarAdminTool.allowSystemExit = prevValue; + } } From e0fe818633ab170d27e8a120ff8c9d5bc2f01a9d Mon Sep 17 00:00:00 2001 From: Baodi Shi Date: Sun, 5 Mar 2023 10:00:47 +0800 Subject: [PATCH 082/174] [fix][broker] Filter system topic when getting topic list by binary proto. (#19667) --- .../pulsar/broker/service/ServerCnx.java | 4 +- .../impl/PatternTopicsConsumerImplTest.java | 54 +++++++++++++++++++ .../pulsar/common/topics/TopicList.java | 4 +- 3 files changed, 58 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 5b9c7d39ff12f..071e8a46d5600 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -2182,8 +2182,8 @@ protected void handleGetTopicsOfNamespace(CommandGetTopicsOfNamespace commandGet getBrokerService().pulsar().getNamespaceService().getListOfTopics(namespaceName, mode) .thenAccept(topics -> { boolean filterTopics = false; - // filter transaction internal topic - List filteredTopics = TopicList.filterTransactionInternalName(topics); + // filter system topic + List filteredTopics = TopicList.filterSystemTopic(topics); if (enableSubscriptionPatternEvaluation && topicsPattern.isPresent()) { if (topicsPattern.get().length() <= maxSubscriptionPatternLength) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplTest.java index 6d19ee1dfe5a1..09a9a003f3ec5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplTest.java @@ -63,6 +63,8 @@ public class PatternTopicsConsumerImplTest extends ProducerConsumerBase { public void setup() throws Exception { // set isTcpLookup = true, to use BinaryProtoLookupService to get topics for a pattern. isTcpLookup = true; + // enabled transaction, to test pattern consumers not subscribe to transaction system topic. + conf.setTransactionCoordinatorEnabled(true); super.internalSetup(); super.producerBaseSetup(); } @@ -251,6 +253,58 @@ public void testBinaryProtoToGetTopicsOfNamespacePersistent() throws Exception { producer4.close(); } + @Test(timeOut = testTimeout) + public void testBinaryProtoSubscribeAllTopicOfNamespace() throws Exception { + String key = "testBinaryProtoSubscribeAllTopicOfNamespace"; + String subscriptionName = "my-ex-subscription-" + key; + String topicName1 = "persistent://my-property/my-ns/pattern-topic-1-" + key; + String topicName2 = "persistent://my-property/my-ns/pattern-topic-2-" + key; + String topicName3 = "persistent://my-property/my-ns/pattern-topic-3-" + key; + Pattern pattern = Pattern.compile("my-property/my-ns/.*"); + + // 1. create partition + TenantInfoImpl tenantInfo = createDefaultTenantInfo(); + admin.tenants().createTenant("prop", tenantInfo); + admin.topics().createPartitionedTopic(topicName1, 1); + admin.topics().createPartitionedTopic(topicName2, 2); + admin.topics().createPartitionedTopic(topicName3, 3); + + // 2. create producer to trigger create system topic. + Producer producer = pulsarClient.newProducer().topic(topicName1) + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); + + Consumer consumer = pulsarClient.newConsumer() + .topicsPattern(pattern) + .patternAutoDiscoveryPeriod(2) + .subscriptionName(subscriptionName) + .subscriptionType(SubscriptionType.Shared) + .ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS) + .receiverQueueSize(4) + .subscribe(); + assertTrue(consumer.getTopic().startsWith(PatternMultiTopicsConsumerImpl.DUMMY_TOPIC_NAME_PREFIX)); + + // 4. verify consumer get methods, to get right number of partitions and topics. + assertSame(pattern, ((PatternMultiTopicsConsumerImpl) consumer).getPattern()); + List topics = ((PatternMultiTopicsConsumerImpl) consumer).getPartitions(); + List> consumers = ((PatternMultiTopicsConsumerImpl) consumer).getConsumers(); + + assertEquals(topics.size(), 6); + assertEquals(consumers.size(), 6); + assertEquals(((PatternMultiTopicsConsumerImpl) consumer).getPartitionedTopics().size(), 3); + + topics.forEach(topic -> log.info("topic: {}", topic)); + consumers.forEach(c -> log.info("consumer: {}", c.getTopic())); + + IntStream.range(0, topics.size()).forEach(index -> + assertEquals(consumers.get(index).getTopic(), topics.get(index))); + + consumer.unsubscribe(); + producer.close(); + consumer.close(); + } + @Test(timeOut = testTimeout) public void testPubRateOnNonPersistent() throws Exception { cleanup(); diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/topics/TopicList.java b/pulsar-common/src/main/java/org/apache/pulsar/common/topics/TopicList.java index 7a01e09ed6461..250cea217ee5f 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/topics/TopicList.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/topics/TopicList.java @@ -57,9 +57,9 @@ public static List filterTopics(List original, Pattern topicsPat .collect(Collectors.toList()); } - public static List filterTransactionInternalName(List original) { + public static List filterSystemTopic(List original) { return original.stream() - .filter(topic -> !SystemTopicNames.isTransactionInternalName(TopicName.get(topic))) + .filter(topic -> !SystemTopicNames.isSystemTopic(TopicName.get(topic))) .collect(Collectors.toList()); } From e13865cec7f2bc5fb8cbae49b92cfef2e93c515a Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Sun, 5 Mar 2023 10:08:30 +0800 Subject: [PATCH 083/174] [fix] [client] fix memory leak if enabled pooled messages (#19585) --- .../impl/BrokerClientIntegrationTest.java | 7 +++- .../pulsar/client/impl/ConsumerBase.java | 3 +- .../pulsar/client/impl/ConsumerImpl.java | 12 +++++++ .../GrowableArrayBlockingQueue.java | 33 +++++++++++++++++++ 4 files changed, 52 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/BrokerClientIntegrationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/BrokerClientIntegrationTest.java index c29978c5f5283..716dd1019f4d8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/BrokerClientIntegrationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/BrokerClientIntegrationTest.java @@ -947,9 +947,14 @@ public void testPooledMessageWithAckTimeout(boolean isBatchingEnabled) throws Ex ByteBuf payload = ((MessageImpl) msg).getPayload(); assertNotEquals(payload.refCnt(), 0); consumer.redeliverUnacknowledgedMessages(); - assertEquals(payload.refCnt(), 0); + Awaitility.await().untilAsserted(() -> { + assertTrue(consumer.incomingMessages.size() >= 100); + }); consumer.close(); producer.close(); + admin.topics().delete(topic, false); + assertEquals(consumer.incomingMessages.size(), 0); + assertEquals(payload.refCnt(), 0); } /** diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java index 6988adcccf183..75d3b2edf6e28 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java @@ -29,7 +29,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutionException; @@ -82,7 +81,7 @@ public abstract class ConsumerBase extends HandlerState implements Consumer> incomingMessages; + final GrowableArrayBlockingQueue> incomingMessages; protected ConcurrentOpenHashMap unAckedChunkedMessageIdSequenceMap; protected final ConcurrentLinkedQueue>> pendingReceives; protected final int maxReceiverQueueSize; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 08a6bb15807c9..18abb5a52c45f 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -1075,6 +1075,18 @@ private void closeConsumerTasks() { } negativeAcksTracker.close(); stats.getStatTimeout().ifPresent(Timeout::cancel); + if (poolMessages) { + releasePooledMessagesAndStopAcceptNew(); + } + } + + /** + * If enabled pooled messages, we should release the messages after closing consumer and stop accept the new + * messages. + */ + private void releasePooledMessagesAndStopAcceptNew() { + incomingMessages.terminate(message -> message.release()); + clearIncomingMessages(); } void activeConsumerChanged(boolean isActive) { diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/GrowableArrayBlockingQueue.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/GrowableArrayBlockingQueue.java index 7825f9a0562be..467a455ed8b3d 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/GrowableArrayBlockingQueue.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/GrowableArrayBlockingQueue.java @@ -32,6 +32,7 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.StampedLock; import java.util.function.Consumer; +import javax.annotation.Nullable; /** * This implements a {@link BlockingQueue} backed by an array with no fixed capacity. @@ -53,6 +54,10 @@ public class GrowableArrayBlockingQueue extends AbstractQueue implements B .newUpdater(GrowableArrayBlockingQueue.class, "size"); private volatile int size = 0; + private volatile boolean terminated = false; + + private volatile Consumer itemAfterTerminatedHandler; + public GrowableArrayBlockingQueue() { this(64); } @@ -132,6 +137,13 @@ public void put(T e) { boolean wasEmpty = false; try { + if (terminated){ + if (itemAfterTerminatedHandler != null) { + itemAfterTerminatedHandler.accept(e); + } + return; + } + if (SIZE_UPDATER.get(this) == data.length) { expandArray(); } @@ -401,6 +413,27 @@ public String toString() { return sb.toString(); } + /** + * Make the queue not accept new items. if there are still new data trying to enter the queue, it will be handed + * by {@param itemAfterTerminatedHandler}. + */ + public void terminate(@Nullable Consumer itemAfterTerminatedHandler) { + // After wait for the in-flight item enqueue, it means the operation of terminate is finished. + long stamp = tailLock.writeLock(); + try { + terminated = true; + if (itemAfterTerminatedHandler != null) { + this.itemAfterTerminatedHandler = itemAfterTerminatedHandler; + } + } finally { + tailLock.unlockWrite(stamp); + } + } + + public boolean isTerminated() { + return terminated; + } + @SuppressWarnings("unchecked") private void expandArray() { // We already hold the tailLock From 4ed8a871bfa9a7a8e30d86d782e33a556646d7b1 Mon Sep 17 00:00:00 2001 From: houxiaoyu Date: Mon, 6 Mar 2023 12:48:41 +0800 Subject: [PATCH 084/174] [fix][test] testModularLoadManagerRemoveBundleAndLoad (#19710) --- .../pulsar/broker/namespace/NamespaceServiceTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java index f4c4799e576f9..ac5d92c880227 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java @@ -45,7 +45,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import lombok.Cleanup; @@ -823,10 +823,10 @@ private void waitResourceDataUpdateToZK(LoadManager loadManager) throws Exceptio // Wait until "ModularLoadManager" completes processing the ZK notification. ModularLoadManagerWrapper modularLoadManagerWrapper = (ModularLoadManagerWrapper) loadManager; ModularLoadManagerImpl modularLoadManager = (ModularLoadManagerImpl) modularLoadManagerWrapper.getLoadManager(); - ScheduledExecutorService scheduler = (ScheduledExecutorService) FieldUtils.readField( - modularLoadManager, "scheduler", true); + ExecutorService executors = (ExecutorService) FieldUtils.readField( + modularLoadManager, "executors", true); CompletableFuture waitForNoticeHandleFinishByLoadManager = new CompletableFuture<>(); - scheduler.execute(() -> waitForNoticeHandleFinishByLoadManager.complete(null)); + executors.execute(() -> waitForNoticeHandleFinishByLoadManager.complete(null)); waitForNoticeHandleFinishByLoadManager.join(); // Manually trigger "LoadResourceQuotaUpdaterTask" loadManager.writeResourceQuotasToZooKeeper(); From b2658af847511fef29455a22e96baa1cdc9d278f Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Tue, 7 Mar 2023 16:21:57 +0800 Subject: [PATCH 085/174] [improve][broker] PIP-192: Make unload and transfer admin API functional (#19538) PIP-192: https://github.com/apache/pulsar/issues/16691 ### Motivation Currently, the unload and transfer admin API still uses the old logic when enabling the `ExtensibleLoadManager`, we need to publish the message to `ServiceUnitStateChannel`, and wait for its response. ### Modifications This PR added a `UnloadManager ` to handle the duplicate unload request and return a `CompletableFuture` to the caller, so we can know when the unload operation is finished. The unload and transfer admin API also been fixed, now we can support do unload when using `ExtensibleLoadManager`. --- .../broker/admin/impl/NamespacesBase.java | 17 +- .../extensions/ExtensibleLoadManagerImpl.java | 53 +++++- .../channel/ServiceUnitStateChannel.java | 8 + .../channel/ServiceUnitStateChannelImpl.java | 24 ++- .../channel/StateChangeListeners.java | 67 +++++++ .../manager/StateChangeListener.java | 33 ++++ .../extensions/manager/UnloadManager.java | 103 ++++++++++ .../extensions/manager/package-info.java | 19 ++ .../extensions/scheduler/UnloadScheduler.java | 17 +- .../broker/namespace/NamespaceService.java | 8 + .../pulsar/broker/web/PulsarWebResource.java | 13 +- .../ExtensibleLoadManagerImplTest.java | 72 +++++++ .../extensions/manager/UnloadManagerTest.java | 178 ++++++++++++++++++ .../scheduler/UnloadSchedulerTest.java | 16 +- 14 files changed, 609 insertions(+), 19 deletions(-) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/StateChangeListeners.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/StateChangeListener.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManager.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/package-info.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManagerTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 30f01ece7de11..5be675f7b636c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -58,6 +58,7 @@ import org.apache.pulsar.broker.admin.AdminResource; import org.apache.pulsar.broker.authorization.AuthorizationService; import org.apache.pulsar.broker.loadbalance.LeaderBroker; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException; import org.apache.pulsar.broker.service.Subscription; @@ -983,8 +984,17 @@ public CompletableFuture setNamespaceBundleAffinityAsync(String bundleRang } return CompletableFuture.completedFuture(null); }) - .thenCompose(__ -> validateLeaderBrokerAsync()) + .thenCompose(__ -> { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config())) { + return CompletableFuture.completedFuture(null); + } + return validateLeaderBrokerAsync(); + }) .thenAccept(__ -> { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config())) { + return; + } + // For ExtensibleLoadManager, this operation will be ignored. pulsar().getLoadManager().get().setNamespaceBundleAffinity(bundleRange, destinationBroker); }); } @@ -1036,10 +1046,11 @@ public CompletableFuture internalUnloadNamespaceBundleAsync(String bundleR namespaceName, bundleRange); return CompletableFuture.completedFuture(null); } + Optional destinationBrokerOpt = Optional.ofNullable(destinationBroker); return validateNamespaceBundleOwnershipAsync(namespaceName, policies.bundles, bundleRange, authoritative, true) - .thenCompose(nsBundle -> - pulsar().getNamespaceService().unloadNamespaceBundle(nsBundle)); + .thenCompose(nsBundle -> pulsar().getNamespaceService() + .unloadNamespaceBundle(nsBundle, destinationBrokerOpt)); })); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index 650c12af57391..2bebe203d8750 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -35,6 +35,7 @@ import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.BrokerFilterException; +import org.apache.pulsar.broker.loadbalance.LoadManager; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; @@ -43,9 +44,11 @@ import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerFilter; import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerMaxTopicCountFilter; import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerVersionFilter; +import org.apache.pulsar.broker.loadbalance.extensions.manager.UnloadManager; import org.apache.pulsar.broker.loadbalance.extensions.models.AssignCounter; import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter; import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision; +import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter; import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision; import org.apache.pulsar.broker.loadbalance.extensions.reporter.BrokerLoadDataReporter; @@ -110,6 +113,8 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager { private ScheduledFuture brokerLoadDataReportTask; private ScheduledFuture topBundlesLoadDataReportTask; + private UnloadManager unloadManager; + private boolean started = false; private final AssignCounter assignCounter = new AssignCounter(); @@ -143,6 +148,13 @@ public static boolean isLoadManagerExtensionEnabled(ServiceConfiguration conf) { return ExtensibleLoadManagerImpl.class.getName().equals(conf.getLoadManagerClassName()); } + public static ExtensibleLoadManagerImpl get(LoadManager loadManager) { + if (!(loadManager instanceof ExtensibleLoadManagerWrapper loadManagerWrapper)) { + throw new IllegalArgumentException("The load manager should be 'ExtensibleLoadManagerWrapper'."); + } + return loadManagerWrapper.get(); + } + @Override public void start() throws PulsarServerException { if (this.started) { @@ -151,6 +163,8 @@ public void start() throws PulsarServerException { this.brokerRegistry = new BrokerRegistryImpl(pulsar); this.serviceUnitStateChannel = new ServiceUnitStateChannelImpl(pulsar); this.brokerRegistry.start(); + this.unloadManager = new UnloadManager(); + this.serviceUnitStateChannel.listen(unloadManager); this.serviceUnitStateChannel.start(); try { @@ -201,7 +215,8 @@ public void start() throws PulsarServerException { interval, TimeUnit.MILLISECONDS); // TODO: Start bundle split scheduler. - this.unloadScheduler = new UnloadScheduler(pulsar.getLoadManagerExecutor(), context, serviceUnitStateChannel); + this.unloadScheduler = new UnloadScheduler( + pulsar.getLoadManagerExecutor(), unloadManager, context, serviceUnitStateChannel); this.unloadScheduler.start(); this.started = true; } @@ -300,6 +315,12 @@ private CompletableFuture> selectAsync(ServiceUnitId bundle) { @Override public CompletableFuture checkOwnershipAsync(Optional topic, ServiceUnitId bundleUnit) { + return getOwnershipAsync(topic, bundleUnit) + .thenApply(broker -> brokerRegistry.getBrokerId().equals(broker.orElse(null))); + } + + private CompletableFuture> getOwnershipAsync(Optional topic, + ServiceUnitId bundleUnit) { final String bundle = bundleUnit.toString(); CompletableFuture> owner; if (topic.isPresent() && isInternalTopic(topic.get().toString())) { @@ -307,8 +328,35 @@ public CompletableFuture checkOwnershipAsync(Optional to } else { owner = serviceUnitStateChannel.getOwnerAsync(bundle); } + return owner; + } + + public CompletableFuture unloadNamespaceBundleAsync(ServiceUnitId bundle, + Optional destinationBroker) { + return getOwnershipAsync(Optional.empty(), bundle) + .thenCompose(brokerOpt -> { + if (brokerOpt.isEmpty()) { + String msg = String.format("Namespace bundle: %s is not owned by any broker.", bundle); + log.warn(msg); + throw new IllegalStateException(msg); + } + String sourceBroker = brokerOpt.get(); + if (destinationBroker.isPresent() && sourceBroker.endsWith(destinationBroker.get())) { + String msg = String.format("Namespace bundle: %s own by %s, cannot be transfer to same broker.", + bundle, sourceBroker); + log.warn(msg); + throw new IllegalArgumentException(msg); + } + return unloadAsync(new Unload(sourceBroker, bundle.toString(), destinationBroker), + conf.getNamespaceBundleUnloadingTimeoutMs(), TimeUnit.MILLISECONDS); + }); + } - return owner.thenApply(broker -> brokerRegistry.getBrokerId().equals(broker.orElse(null))); + private CompletableFuture unloadAsync(Unload unload, + long timeout, + TimeUnit timeoutUnit) { + CompletableFuture future = serviceUnitStateChannel.publishUnloadEventAsync(unload); + return unloadManager.waitAsync(future, unload.serviceUnit(), timeout, timeoutUnit); } @Override @@ -337,6 +385,7 @@ public void close() throws PulsarServerException { try { this.serviceUnitStateChannel.close(); } finally { + this.unloadManager.close(); this.started = false; } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java index 44950a21ffd20..dc4d582ddb081 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java @@ -23,6 +23,7 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import org.apache.pulsar.broker.PulsarServerException; +import org.apache.pulsar.broker.loadbalance.extensions.manager.StateChangeListener; import org.apache.pulsar.broker.loadbalance.extensions.models.Split; import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; import org.apache.pulsar.common.stats.Metrics; @@ -156,4 +157,11 @@ public interface ServiceUnitStateChannel extends Closeable { */ List getMetrics(); + /** + * Add a state change listener. + * + * @param listener State change listener. + */ + void listen(StateChangeListener listener); + } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index cddaf92d22771..5f24e41dda931 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -72,6 +72,7 @@ import org.apache.pulsar.broker.loadbalance.extensions.BrokerRegistry; import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerWrapper; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; +import org.apache.pulsar.broker.loadbalance.extensions.manager.StateChangeListener; import org.apache.pulsar.broker.loadbalance.extensions.models.Split; import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; import org.apache.pulsar.broker.loadbalance.extensions.strategy.BrokerSelectionStrategy; @@ -117,6 +118,7 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel { private final ConcurrentOpenHashMap> getOwnerRequests; private final String lookupServiceAddress; private final ConcurrentOpenHashMap> cleanupJobs; + private final StateChangeListeners stateChangeListeners; private final LeaderElectionService leaderElectionService; private BrokerSelectionStrategy brokerSelector; private BrokerRegistry brokerRegistry; @@ -191,6 +193,7 @@ public ServiceUnitStateChannelImpl(PulsarService pulsar) { this.getOwnerRequests = ConcurrentOpenHashMap.>newBuilder().build(); this.cleanupJobs = ConcurrentOpenHashMap.>newBuilder().build(); + this.stateChangeListeners = new StateChangeListeners(); this.semiTerminalStateWaitingTimeInMillis = config.getLoadBalancerServiceUnitStateCleanUpDelayTimeInSeconds() * 1000; this.inFlightStateWaitingTimeInMillis = MAX_IN_FLIGHT_STATE_WAITING_TIME_IN_MILLIS; @@ -357,6 +360,10 @@ public synchronized void close() throws PulsarServerException { log.info("Successfully cancelled the cleanup tasks"); } + if (stateChangeListeners != null) { + stateChangeListeners.close(); + } + log.info("Successfully closed the channel."); } catch (Exception e) { @@ -619,6 +626,7 @@ private void handleOwnEvent(String serviceUnit, ServiceUnitStateData data) { if (getOwnerRequest != null) { getOwnerRequest.complete(data.dstBroker()); } + stateChangeListeners.notify(serviceUnit, data, null); if (isTargetBroker(data.dstBroker())) { log(null, serviceUnit, data, null); } @@ -628,7 +636,7 @@ private void handleAssignEvent(String serviceUnit, ServiceUnitStateData data) { if (isTargetBroker(data.dstBroker())) { ServiceUnitStateData next = new ServiceUnitStateData( Owned, data.dstBroker(), data.sourceBroker(), getNextVersionId(data)); - pubAsync(serviceUnit, next) + stateChangeListeners.notifyOnCompletion(pubAsync(serviceUnit, next), serviceUnit, data) .whenComplete((__, e) -> log(e, serviceUnit, data, next)); } } @@ -644,15 +652,15 @@ private void handleReleaseEvent(String serviceUnit, ServiceUnitStateData data) { next = new ServiceUnitStateData( Free, null, data.sourceBroker(), getNextVersionId(data)); } - closeServiceUnit(serviceUnit) - .thenCompose(__ -> pubAsync(serviceUnit, next)) + stateChangeListeners.notifyOnCompletion(closeServiceUnit(serviceUnit) + .thenCompose(__ -> pubAsync(serviceUnit, next)), serviceUnit, data) .whenComplete((__, e) -> log(e, serviceUnit, data, next)); } } private void handleSplitEvent(String serviceUnit, ServiceUnitStateData data) { if (isTargetBroker(data.sourceBroker())) { - splitServiceUnit(serviceUnit, data) + stateChangeListeners.notifyOnCompletion(splitServiceUnit(serviceUnit, data), serviceUnit, data) .whenComplete((__, e) -> log(e, serviceUnit, data, null)); } } @@ -662,6 +670,7 @@ private void handleFreeEvent(String serviceUnit, ServiceUnitStateData data) { if (getOwnerRequest != null) { getOwnerRequest.complete(null); } + stateChangeListeners.notify(serviceUnit, data, null); if (isTargetBroker(data.sourceBroker())) { log(null, serviceUnit, data, null); } @@ -672,6 +681,7 @@ private void handleDeleteEvent(String serviceUnit, ServiceUnitStateData data) { if (getOwnerRequest != null) { getOwnerRequest.completeExceptionally(new IllegalStateException(serviceUnit + "has been deleted.")); } + stateChangeListeners.notify(serviceUnit, data, null); if (isTargetBroker(data.sourceBroker())) { log(null, serviceUnit, data, null); } @@ -682,6 +692,7 @@ private void handleInitEvent(String serviceUnit) { if (getOwnerRequest != null) { getOwnerRequest.complete(null); } + stateChangeListeners.notify(serviceUnit, null, null); log(null, serviceUnit, null, null); } @@ -1302,4 +1313,9 @@ public List getMetrics() { return metrics; } + + @Override + public void listen(StateChangeListener listener) { + this.stateChangeListeners.addListener(listener); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/StateChangeListeners.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/StateChangeListeners.java new file mode 100644 index 0000000000000..1d396f500b648 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/StateChangeListeners.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.channel; + +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CopyOnWriteArrayList; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.loadbalance.extensions.manager.StateChangeListener; + +@Slf4j +public class StateChangeListeners { + + private final List stateChangeListeners; + + public StateChangeListeners() { + stateChangeListeners = new CopyOnWriteArrayList<>(); + } + + public void addListener(StateChangeListener listener) { + Objects.requireNonNull(listener); + stateChangeListeners.add(listener); + } + + public void close() { + this.stateChangeListeners.clear(); + } + + /** + * Notify all currently added listeners on completion of the future. + * + * @return future of a new completion stage + */ + public CompletableFuture notifyOnCompletion(CompletableFuture future, + String serviceUnit, + ServiceUnitStateData data) { + return future.whenComplete((r, ex) -> notify(serviceUnit, data, ex)); + } + + public void notify(String serviceUnit, ServiceUnitStateData data, Throwable t) { + stateChangeListeners.forEach(listener -> { + try { + listener.handleEvent(serviceUnit, data, t); + } catch (Throwable ex) { + log.error("StateChangeListener: {} exception while handling {} for service unit {}", + listener, data, serviceUnit, ex); + } + }); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/StateChangeListener.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/StateChangeListener.java new file mode 100644 index 0000000000000..7ba8be8771b91 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/StateChangeListener.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.manager; + +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData; + +public interface StateChangeListener { + + /** + * Handle the service unit state change. + * + * @param serviceUnit - Service Unit(Namespace bundle). + * @param data - Service unit state data. + * @param t - Exception, if present. + */ + void handleEvent(String serviceUnit, ServiceUnitStateData data, Throwable t); +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManager.java new file mode 100644 index 0000000000000..ead6384daba8d --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManager.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.manager; + +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData; + +/** + * Unload manager. + */ +@Slf4j +public class UnloadManager implements StateChangeListener { + + private final Map> inFlightUnloadRequest; + + public UnloadManager() { + this.inFlightUnloadRequest = new ConcurrentHashMap<>(); + } + + private void complete(String serviceUnit, Throwable ex) { + inFlightUnloadRequest.computeIfPresent(serviceUnit, (__, future) -> { + if (!future.isDone()) { + if (ex != null) { + future.completeExceptionally(ex); + if (log.isDebugEnabled()) { + log.debug("Complete exceptionally unload bundle: {}", serviceUnit, ex); + } + } else { + future.complete(null); + if (log.isDebugEnabled()) { + log.debug("Complete unload bundle: {}", serviceUnit); + } + } + } + return null; + }); + } + + public CompletableFuture waitAsync(CompletableFuture eventPubFuture, + String bundle, + long timeout, + TimeUnit timeoutUnit) { + + return eventPubFuture.thenCompose(__ -> inFlightUnloadRequest.computeIfAbsent(bundle, ignore -> { + if (log.isDebugEnabled()) { + log.debug("Handle unload bundle: {}, timeout: {} {}", bundle, timeout, timeoutUnit); + } + CompletableFuture future = new CompletableFuture<>(); + future.orTimeout(timeout, timeoutUnit).whenComplete((v, ex) -> { + if (ex != null) { + inFlightUnloadRequest.remove(bundle); + log.warn("Failed to wait unload for serviceUnit: {}", bundle, ex); + } + }); + return future; + })); + } + + @Override + public void handleEvent(String serviceUnit, ServiceUnitStateData data, Throwable t) { + ServiceUnitState state = ServiceUnitStateData.state(data); + switch (state) { + case Free, Owned -> this.complete(serviceUnit, t); + default -> { + if (log.isDebugEnabled()) { + log.debug("Handling {} for service unit {}", data, serviceUnit); + } + } + } + } + + public void close() { + inFlightUnloadRequest.forEach((bundle, future) -> { + if (!future.isDone()) { + String msg = String.format("Unloading bundle: %s, but the unload manager already closed.", bundle); + log.warn(msg); + future.completeExceptionally(new IllegalStateException(msg)); + } + }); + inFlightUnloadRequest.clear(); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/package-info.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/package-info.java new file mode 100644 index 0000000000000..ac553c0690068 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/package-info.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.manager; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java index 5cdbd3027104d..e646026978754 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java @@ -31,6 +31,7 @@ import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel; +import org.apache.pulsar.broker.loadbalance.extensions.manager.UnloadManager; import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.Reflections; @@ -42,6 +43,8 @@ public class UnloadScheduler implements LoadManagerScheduler { private final ScheduledExecutorService loadManagerExecutor; + private final UnloadManager unloadManager; + private final LoadManagerContext context; private final ServiceUnitStateChannel channel; @@ -57,13 +60,16 @@ public class UnloadScheduler implements LoadManagerScheduler { private volatile CompletableFuture currentRunningFuture = null; public UnloadScheduler(ScheduledExecutorService loadManagerExecutor, + UnloadManager unloadManager, LoadManagerContext context, ServiceUnitStateChannel channel) { - this(loadManagerExecutor, context, channel, createNamespaceUnloadStrategy(context.brokerConfiguration())); + this(loadManagerExecutor, unloadManager, context, + channel, createNamespaceUnloadStrategy(context.brokerConfiguration())); } @VisibleForTesting protected UnloadScheduler(ScheduledExecutorService loadManagerExecutor, + UnloadManager unloadManager, LoadManagerContext context, ServiceUnitStateChannel channel, NamespaceUnloadStrategy strategy) { @@ -71,6 +77,7 @@ protected UnloadScheduler(ScheduledExecutorService loadManagerExecutor, this.recentlyUnloadedBundles = new HashMap<>(); this.recentlyUnloadedBrokers = new HashMap<>(); this.loadManagerExecutor = loadManagerExecutor; + this.unloadManager = unloadManager; this.context = context; this.conf = context.brokerConfiguration(); this.channel = channel; @@ -131,9 +138,11 @@ public synchronized void execute() { List> futures = new ArrayList<>(); unloadDecision.getUnloads().forEach((broker, unload) -> { log.info("[{}] Unloading bundle: {}", namespaceUnloadStrategy.getClass().getSimpleName(), unload); - futures.add(channel.publishUnloadEventAsync(unload).thenAccept(__ -> { - recentlyUnloadedBundles.put(unload.serviceUnit(), System.currentTimeMillis()); - recentlyUnloadedBrokers.put(unload.sourceBroker(), System.currentTimeMillis()); + futures.add(unloadManager.waitAsync(channel.publishUnloadEventAsync(unload), unload.serviceUnit(), + conf.getNamespaceBundleUnloadingTimeoutMs(), TimeUnit.MILLISECONDS) + .thenAccept(__ -> { + recentlyUnloadedBundles.put(unload.serviceUnit(), System.currentTimeMillis()); + recentlyUnloadedBrokers.put(unload.sourceBroker(), System.currentTimeMillis()); })); }); return FutureUtil.waitForAll(futures).exceptionally(ex -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index 33b15926c3c33..899539e1db6a7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -717,6 +717,14 @@ private Optional> getLeastLoadedFromLoadManager(ServiceUnit } public CompletableFuture unloadNamespaceBundle(NamespaceBundle bundle) { + return unloadNamespaceBundle(bundle, Optional.empty()); + } + + public CompletableFuture unloadNamespaceBundle(NamespaceBundle bundle, Optional destinationBroker) { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) { + return ExtensibleLoadManagerImpl.get(loadManager.get()) + .unloadNamespaceBundleAsync(bundle, destinationBroker); + } // unload namespace bundle return unloadNamespaceBundle(bundle, config.getNamespaceBundleUnloadingTimeoutMs(), TimeUnit.MILLISECONDS); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java index 5484a70e1aad0..321a127ad97bd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java @@ -55,6 +55,7 @@ import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; import org.apache.pulsar.broker.authorization.AuthorizationService; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; import org.apache.pulsar.broker.namespace.LookupOptions; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.resources.BookieResources; @@ -626,14 +627,18 @@ protected NamespaceBundle validateNamespaceBundleOwnership(NamespaceName fqnn, B } } - protected CompletableFuture validateNamespaceBundleOwnershipAsync(NamespaceName fqnn, - BundlesData bundles, String bundleRange, boolean authoritative, boolean readOnly) { + protected CompletableFuture validateNamespaceBundleOwnershipAsync( + NamespaceName fqnn, BundlesData bundles, String bundleRange, + boolean authoritative, boolean readOnly) { NamespaceBundle nsBundle; try { nsBundle = validateNamespaceBundleRange(fqnn, bundles, bundleRange); } catch (WebApplicationException wae) { return CompletableFuture.failedFuture(wae); } + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config())) { + return CompletableFuture.completedFuture(nsBundle); + } return validateBundleOwnershipAsync(nsBundle, authoritative, readOnly) .thenApply(__ -> nsBundle); } @@ -992,6 +997,10 @@ protected boolean isLeaderBroker() { } protected static boolean isLeaderBroker(PulsarService pulsar) { + // For extensible load manager, it doesn't have leader election service on pulsar broker. + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(pulsar.getConfig())) { + return true; + } return pulsar.getLeaderElectionService().isLeader(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index d1bf29725d004..ec82f5c383e2e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -43,10 +43,12 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; import com.google.common.collect.Sets; import java.util.LinkedHashMap; import java.util.Set; +import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -77,18 +79,22 @@ import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter; import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision; import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter; +import org.apache.pulsar.broker.loadbalance.extensions.scheduler.TransferShedder; import org.apache.pulsar.broker.lookup.LookupResult; import org.apache.pulsar.broker.namespace.LookupOptions; import org.apache.pulsar.broker.resources.NamespaceResources; import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.broker.resources.TenantResources; import org.apache.pulsar.broker.testcontext.PulsarTestContext; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.impl.TableViewImpl; import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.TenantInfo; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.stats.Metrics; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; @@ -124,10 +130,15 @@ public class ExtensibleLoadManagerImplTest extends MockedPulsarServiceBaseTest { public void setup() throws Exception { conf.setAllowAutoTopicCreation(true); conf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); + conf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); + conf.setLoadBalancerSheddingEnabled(false); super.internalSetup(conf); pulsar1 = pulsar; ServiceConfiguration defaultConf = getDefaultConf(); + defaultConf.setAllowAutoTopicCreation(true); defaultConf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); + defaultConf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); + defaultConf.setLoadBalancerSheddingEnabled(false); additionalPulsarTestContext = createAdditionalPulsarTestContext(defaultConf); pulsar2 = additionalPulsarTestContext.getPulsarService(); @@ -148,6 +159,14 @@ public void setup() throws Exception { channel2 = (ServiceUnitStateChannelImpl) FieldUtils.readField(secondaryLoadManager, "serviceUnitStateChannel", true); + admin.clusters().createCluster(this.conf.getClusterName(), + ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build()); + admin.tenants().createTenant("public", + new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), + Sets.newHashSet(this.conf.getClusterName()))); + admin.namespaces().createNamespace("public/default"); + admin.namespaces().setNamespaceReplicationClusters("public/default", + Sets.newHashSet(this.conf.getClusterName())); } protected void beforePulsarStart(PulsarService pulsar) throws Exception { @@ -307,6 +326,59 @@ public Map filter(Map broker assertTrue(brokerLookupData.isPresent()); } + @Test(timeOut = 30 * 1000) + public void testUnloadAdminAPI() throws Exception { + TopicName topicName = TopicName.get("test-unload"); + NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get(); + + String broker = admin.lookups().lookupTopic(topicName.toString()); + log.info("Assign the bundle {} to {}", bundle, broker); + + checkOwnershipState(broker, bundle); + admin.namespaces().unloadNamespaceBundle(topicName.getNamespace(), bundle.getBundleRange()); + assertFalse(primaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()); + assertFalse(secondaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()); + + broker = admin.lookups().lookupTopic(topicName.toString()); + log.info("Assign the bundle {} to {}", bundle, broker); + + String dstBrokerUrl = pulsar1.getLookupServiceAddress(); + String dstBrokerServiceUrl; + if (broker.equals(pulsar1.getBrokerServiceUrl())) { + dstBrokerUrl = pulsar2.getLookupServiceAddress(); + dstBrokerServiceUrl = pulsar2.getBrokerServiceUrl(); + } else { + dstBrokerServiceUrl = pulsar1.getBrokerServiceUrl(); + } + checkOwnershipState(broker, bundle); + + admin.namespaces().unloadNamespaceBundle(topicName.getNamespace(), bundle.getBundleRange(), dstBrokerUrl); + + assertEquals(admin.lookups().lookupTopic(topicName.toString()), dstBrokerServiceUrl); + + // Test transfer to current broker. + try { + admin.namespaces() + .unloadNamespaceBundle(topicName.getNamespace(), bundle.getBundleRange(), dstBrokerUrl); + fail(); + } catch (PulsarAdminException ex) { + assertTrue(ex.getMessage().contains("cannot be transfer to same broker")); + } + } + + private void checkOwnershipState(String broker, NamespaceBundle bundle) + throws ExecutionException, InterruptedException { + var targetLoadManager = secondaryLoadManager; + var otherLoadManager = primaryLoadManager; + if (broker.equals(pulsar1.getBrokerServiceUrl())) { + targetLoadManager = primaryLoadManager; + otherLoadManager = secondaryLoadManager; + } + assertTrue(targetLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()); + assertFalse(otherLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()); + } + + @Test public void testGetMetrics() throws Exception { { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManagerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManagerTest.java new file mode 100644 index 0000000000000..75ef913b8a851 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManagerTest.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.manager; + +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.VERSION_ID_INIT; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData; +import org.apache.pulsar.common.util.FutureUtil; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker") +public class UnloadManagerTest { + + @Test + public void testEventPubFutureHasException() { + UnloadManager manager = new UnloadManager(); + CompletableFuture future = + manager.waitAsync(FutureUtil.failedFuture(new Exception("test")), + "bundle-1", 10, TimeUnit.SECONDS); + + assertTrue(future.isCompletedExceptionally()); + try { + future.get(); + fail(); + } catch (Exception ex) { + assertEquals(ex.getCause().getMessage(), "test"); + } + } + + @Test + public void testTimeout() throws IllegalAccessException { + UnloadManager manager = new UnloadManager(); + CompletableFuture future = + manager.waitAsync(CompletableFuture.completedFuture(null), + "bundle-1", 3, TimeUnit.SECONDS); + Map> inFlightUnloadRequestMap = getInFlightUnloadRequestMap(manager); + + assertEquals(inFlightUnloadRequestMap.size(), 1); + + try { + future.get(); + fail(); + } catch (Exception ex) { + assertTrue(ex.getCause() instanceof TimeoutException); + } + + assertEquals(inFlightUnloadRequestMap.size(), 0); + } + + @Test + public void testSuccess() throws IllegalAccessException, ExecutionException, InterruptedException { + UnloadManager manager = new UnloadManager(); + CompletableFuture future = + manager.waitAsync(CompletableFuture.completedFuture(null), + "bundle-1", 5, TimeUnit.SECONDS); + Map> inFlightUnloadRequestMap = getInFlightUnloadRequestMap(manager); + + assertEquals(inFlightUnloadRequestMap.size(), 1); + + manager.handleEvent("bundle-1", + new ServiceUnitStateData(ServiceUnitState.Assigning, "broker-1", VERSION_ID_INIT), null); + assertEquals(inFlightUnloadRequestMap.size(), 1); + + manager.handleEvent("bundle-1", + new ServiceUnitStateData(ServiceUnitState.Deleted, "broker-1", VERSION_ID_INIT), null); + assertEquals(inFlightUnloadRequestMap.size(), 1); + + manager.handleEvent("bundle-1", + new ServiceUnitStateData(ServiceUnitState.Splitting, "broker-1", VERSION_ID_INIT), null); + assertEquals(inFlightUnloadRequestMap.size(), 1); + + manager.handleEvent("bundle-1", + new ServiceUnitStateData(ServiceUnitState.Releasing, "broker-1", VERSION_ID_INIT), null); + assertEquals(inFlightUnloadRequestMap.size(), 1); + + manager.handleEvent("bundle-1", + new ServiceUnitStateData(ServiceUnitState.Init, "broker-1", VERSION_ID_INIT), null); + assertEquals(inFlightUnloadRequestMap.size(), 1); + + manager.handleEvent("bundle-1", + new ServiceUnitStateData(ServiceUnitState.Free, "broker-1", VERSION_ID_INIT), null); + assertEquals(inFlightUnloadRequestMap.size(), 0); + future.get(); + + // Success with Owned state. + future = manager.waitAsync(CompletableFuture.completedFuture(null), + "bundle-1", 5, TimeUnit.SECONDS); + inFlightUnloadRequestMap = getInFlightUnloadRequestMap(manager); + + assertEquals(inFlightUnloadRequestMap.size(), 1); + + manager.handleEvent("bundle-1", + new ServiceUnitStateData(ServiceUnitState.Owned, "broker-1", VERSION_ID_INIT), null); + assertEquals(inFlightUnloadRequestMap.size(), 0); + future.get(); + } + + @Test + public void testFailedStage() throws IllegalAccessException { + UnloadManager manager = new UnloadManager(); + CompletableFuture future = + manager.waitAsync(CompletableFuture.completedFuture(null), + "bundle-1", 5, TimeUnit.SECONDS); + Map> inFlightUnloadRequestMap = getInFlightUnloadRequestMap(manager); + + assertEquals(inFlightUnloadRequestMap.size(), 1); + + manager.handleEvent("bundle-1", + new ServiceUnitStateData(ServiceUnitState.Owned, "broker-1", VERSION_ID_INIT), + new IllegalStateException("Failed stage.")); + + try { + future.get(); + fail(); + } catch (Exception ex) { + assertTrue(ex.getCause() instanceof IllegalStateException); + assertEquals(ex.getCause().getMessage(), "Failed stage."); + } + + assertEquals(inFlightUnloadRequestMap.size(), 0); + } + + @Test + public void testClose() throws IllegalAccessException { + UnloadManager manager = new UnloadManager(); + CompletableFuture future = + manager.waitAsync(CompletableFuture.completedFuture(null), + "bundle-1", 5, TimeUnit.SECONDS); + Map> inFlightUnloadRequestMap = getInFlightUnloadRequestMap(manager); + assertEquals(inFlightUnloadRequestMap.size(), 1); + manager.close(); + assertEquals(inFlightUnloadRequestMap.size(), 0); + + try { + future.get(); + fail(); + } catch (Exception ex) { + assertTrue(ex.getCause() instanceof IllegalStateException); + } + } + + private Map> getInFlightUnloadRequestMap(UnloadManager manager) + throws IllegalAccessException { + Map> inFlightUnloadRequest = + (Map>) FieldUtils.readField(manager, "inFlightUnloadRequest", true); + + return inFlightUnloadRequest; + } + +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java index cda5f81d81b93..73d4eb1f18bfb 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.loadbalance.extensions.scheduler; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; @@ -31,6 +32,7 @@ import org.apache.pulsar.broker.loadbalance.extensions.BrokerRegistry; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel; +import org.apache.pulsar.broker.loadbalance.extensions.manager.UnloadManager; import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision; import org.apache.pulsar.client.util.ExecutorProvider; @@ -71,17 +73,20 @@ public void testExecuteSuccess() { LoadManagerContext context = setupContext(); BrokerRegistry registry = context.brokerRegistry(); ServiceUnitStateChannel channel = mock(ServiceUnitStateChannel.class); + UnloadManager unloadManager = mock(UnloadManager.class); NamespaceUnloadStrategy unloadStrategy = mock(NamespaceUnloadStrategy.class); doReturn(CompletableFuture.completedFuture(true)).when(channel).isChannelOwnerAsync(); doReturn(CompletableFuture.completedFuture(Lists.newArrayList("broker-1", "broker-2"))) .when(registry).getAvailableBrokersAsync(); doReturn(CompletableFuture.completedFuture(null)).when(channel).publishUnloadEventAsync(any()); + doReturn(CompletableFuture.completedFuture(null)).when(unloadManager) + .waitAsync(any(), any(), anyLong(), any()); UnloadDecision decision = new UnloadDecision(); Unload unload = new Unload("broker-1", "bundle-1"); decision.getUnloads().put("broker-1", unload); doReturn(decision).when(unloadStrategy).findBundlesForUnloading(any(), any(), any()); - UnloadScheduler scheduler = new UnloadScheduler(loadManagerExecutor, context, channel, unloadStrategy); + UnloadScheduler scheduler = new UnloadScheduler(loadManagerExecutor, unloadManager, context, channel, unloadStrategy); scheduler.execute(); @@ -101,6 +106,7 @@ public void testExecuteMoreThenOnceWhenFirstNotDone() throws InterruptedExceptio LoadManagerContext context = setupContext(); BrokerRegistry registry = context.brokerRegistry(); ServiceUnitStateChannel channel = mock(ServiceUnitStateChannel.class); + UnloadManager unloadManager = mock(UnloadManager.class); NamespaceUnloadStrategy unloadStrategy = mock(NamespaceUnloadStrategy.class); doReturn(CompletableFuture.completedFuture(true)).when(channel).isChannelOwnerAsync(); doAnswer(__ -> CompletableFuture.supplyAsync(() -> { @@ -112,7 +118,7 @@ public void testExecuteMoreThenOnceWhenFirstNotDone() throws InterruptedExceptio } return Lists.newArrayList("broker-1", "broker-2"); }, Executors.newFixedThreadPool(1))).when(registry).getAvailableBrokersAsync(); - UnloadScheduler scheduler = new UnloadScheduler(loadManagerExecutor, context, channel, unloadStrategy); + UnloadScheduler scheduler = new UnloadScheduler(loadManagerExecutor, unloadManager, context, channel, unloadStrategy); ExecutorService executorService = Executors.newFixedThreadPool(10); CountDownLatch latch = new CountDownLatch(10); @@ -133,7 +139,8 @@ public void testDisableLoadBalancer() { context.brokerConfiguration().setLoadBalancerEnabled(false); ServiceUnitStateChannel channel = mock(ServiceUnitStateChannel.class); NamespaceUnloadStrategy unloadStrategy = mock(NamespaceUnloadStrategy.class); - UnloadScheduler scheduler = new UnloadScheduler(loadManagerExecutor, context, channel, unloadStrategy); + UnloadManager unloadManager = mock(UnloadManager.class); + UnloadScheduler scheduler = new UnloadScheduler(loadManagerExecutor, unloadManager, context, channel, unloadStrategy); scheduler.execute(); @@ -152,7 +159,8 @@ public void testNotChannelOwner() { context.brokerConfiguration().setLoadBalancerEnabled(false); ServiceUnitStateChannel channel = mock(ServiceUnitStateChannel.class); NamespaceUnloadStrategy unloadStrategy = mock(NamespaceUnloadStrategy.class); - UnloadScheduler scheduler = new UnloadScheduler(loadManagerExecutor, context, channel, unloadStrategy); + UnloadManager unloadManager = mock(UnloadManager.class); + UnloadScheduler scheduler = new UnloadScheduler(loadManagerExecutor, unloadManager, context, channel, unloadStrategy); doReturn(CompletableFuture.completedFuture(false)).when(channel).isChannelOwnerAsync(); scheduler.execute(); From af1360fb167c1f9484fda5771df3ea9b21d1440b Mon Sep 17 00:00:00 2001 From: sinan liu <54846009+Denovo1998@users.noreply.github.com> Date: Tue, 7 Mar 2023 16:34:32 +0800 Subject: [PATCH 086/174] [PIP-236][fix][broker]Fix using schema to create consumer fails after using AUTO_CONSUME consumer to subscribe topic (#17449) Fixes #17354 PIP #19113 ### Motivation *Fixed the failure to use schema to create consumer after using AUTO-CONSUME consumer to subscribe an empty topic, and Broker returned the error message as `IncompatibleSchemaException("Topic does not have schema to check")`.* ### Modifications *In PersistentTopic::addSchemaIfIdleOrCheckCompatible, when there is an active consumer, but the consumer is using the AUTO_CONSUME schema to subscribe to the topic. Continuing to create a schema consumer to subscribe to the topic will fail.* - When `numActiveConsumers != 0`, and check the schema of the currently existing consumers is AUTO_CONSUME schema. --- .../pulsar/broker/service/Consumer.java | 16 ++++ .../pulsar/broker/service/ServerCnx.java | 3 +- .../broker/service/SubscriptionOption.java | 2 + .../nonpersistent/NonPersistentTopic.java | 21 ++++-- .../service/persistent/PersistentTopic.java | 42 ++++++----- .../pulsar/client/api/SimpleSchemaTest.java | 73 +++++++++++++++++++ .../pulsar/client/impl/ConsumerImpl.java | 6 ++ .../client/impl/schema/AutoConsumeSchema.java | 6 ++ .../pulsar/common/protocol/Commands.java | 12 ++- pulsar-common/src/main/proto/PulsarApi.proto | 2 + 10 files changed, 153 insertions(+), 30 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index 47da95b34ac3f..1ee3f513ef288 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -58,6 +58,7 @@ import org.apache.pulsar.common.policies.data.ClusterData.ClusterUrl; import org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl; import org.apache.pulsar.common.protocol.Commands; +import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.stats.Rate; import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.FutureUtil; @@ -142,12 +143,24 @@ public class Consumer { private long negtiveUnackedMsgsTimestamp; + @Getter + private final SchemaType schemaType; + public Consumer(Subscription subscription, SubType subType, String topicName, long consumerId, int priorityLevel, String consumerName, boolean isDurable, TransportCnx cnx, String appId, Map metadata, boolean readCompacted, KeySharedMeta keySharedMeta, MessageId startMessageId, long consumerEpoch) { + this(subscription, subType, topicName, consumerId, priorityLevel, consumerName, isDurable, cnx, appId, + metadata, readCompacted, keySharedMeta, startMessageId, consumerEpoch, null); + } + public Consumer(Subscription subscription, SubType subType, String topicName, long consumerId, + int priorityLevel, String consumerName, + boolean isDurable, TransportCnx cnx, String appId, + Map metadata, boolean readCompacted, + KeySharedMeta keySharedMeta, MessageId startMessageId, + long consumerEpoch, SchemaType schemaType) { this.subscription = subscription; this.subType = subType; this.topicName = topicName; @@ -204,6 +217,8 @@ public Consumer(Subscription subscription, SubType subType, String topicName, lo this.consumerEpoch = consumerEpoch; this.isAcknowledgmentAtBatchIndexLevelEnabled = subscription.getTopic().getBrokerService() .getPulsar().getConfiguration().isAcknowledgmentAtBatchIndexLevelEnabled(); + + this.schemaType = schemaType; } @VisibleForTesting @@ -231,6 +246,7 @@ public Consumer(Subscription subscription, SubType subType, String topicName, lo this.clientAddress = null; this.startMessageId = null; this.isAcknowledgmentAtBatchIndexLevelEnabled = false; + this.schemaType = null; MESSAGE_PERMITS_UPDATER.set(this, availablePermits); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 071e8a46d5600..4fc79a124acd8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -1195,8 +1195,9 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { .replicatedSubscriptionStateArg(isReplicated).keySharedMeta(keySharedMeta) .subscriptionProperties(subscriptionProperties) .consumerEpoch(consumerEpoch) + .schemaType(schema == null ? null : schema.getType()) .build(); - if (schema != null) { + if (schema != null && schema.getType() != SchemaType.AUTO_CONSUME) { return topic.addSchemaIfIdleOrCheckCompatible(schema) .thenCompose(v -> topic.subscribe(option)); } else { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SubscriptionOption.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SubscriptionOption.java index d375c539e550e..af56d023616b4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SubscriptionOption.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SubscriptionOption.java @@ -29,6 +29,7 @@ import org.apache.pulsar.common.api.proto.CommandSubscribe; import org.apache.pulsar.common.api.proto.KeySharedMeta; import org.apache.pulsar.common.api.proto.KeyValue; +import org.apache.pulsar.common.schema.SchemaType; @Getter @Builder @@ -49,6 +50,7 @@ public class SubscriptionOption { private KeySharedMeta keySharedMeta; private Optional> subscriptionProperties; private long consumerEpoch; + private SchemaType schemaType; public static Optional> getPropertiesMap(List list) { if (list == null) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 3b046570d732e..a0a8462a22753 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -87,6 +87,7 @@ import org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl; import org.apache.pulsar.common.policies.data.stats.SubscriptionStatsImpl; import org.apache.pulsar.common.protocol.schema.SchemaData; +import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.metadata.api.MetadataStoreException; @@ -255,7 +256,8 @@ public CompletableFuture subscribe(SubscriptionOption option) { option.isDurable(), option.getStartMessageId(), option.getMetadata(), option.isReadCompacted(), option.getStartMessageRollbackDurationSec(), option.isReplicatedSubscriptionStateArg(), - option.getKeySharedMeta(), option.getSubscriptionProperties().orElse(null)); + option.getKeySharedMeta(), option.getSubscriptionProperties().orElse(null), + option.getSchemaType()); } @Override @@ -268,7 +270,7 @@ public CompletableFuture subscribe(final TransportCnx cnx, String subs KeySharedMeta keySharedMeta) { return internalSubscribe(cnx, subscriptionName, consumerId, subType, priorityLevel, consumerName, isDurable, startMessageId, metadata, readCompacted, resetStartMessageBackInSec, - replicateSubscriptionState, keySharedMeta, null); + replicateSubscriptionState, keySharedMeta, null, null); } private CompletableFuture internalSubscribe(final TransportCnx cnx, String subscriptionName, @@ -279,7 +281,8 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St long resetStartMessageBackInSec, boolean replicateSubscriptionState, KeySharedMeta keySharedMeta, - Map subscriptionProperties) { + Map subscriptionProperties, + SchemaType schemaType) { return brokerService.checkTopicNsOwnership(getName()).thenCompose(__ -> { final CompletableFuture future = new CompletableFuture<>(); @@ -321,8 +324,8 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St name -> new NonPersistentSubscription(this, subscriptionName, isDurable, subscriptionProperties)); Consumer consumer = new Consumer(subscription, subType, topic, consumerId, priorityLevel, consumerName, - false, cnx, cnx.getAuthRole(), metadata, readCompacted, keySharedMeta, - MessageId.latest, DEFAULT_CONSUMER_EPOCH); + false, cnx, cnx.getAuthRole(), metadata, readCompacted, keySharedMeta, MessageId.latest, + DEFAULT_CONSUMER_EPOCH, schemaType); if (isMigrated()) { consumer.topicMigrated(getClusterMigrationUrl()); } @@ -1162,12 +1165,14 @@ public CompletableFuture getLastMessageId() { @Override public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schema) { return hasSchema().thenCompose((hasSchema) -> { - int numActiveConsumers = subscriptions.values().stream() - .mapToInt(subscription -> subscription.getConsumers().size()) + int numActiveConsumersWithoutAutoSchema = subscriptions.values().stream() + .mapToInt(subscription -> subscription.getConsumers().stream() + .filter(consumer -> consumer.getSchemaType() != SchemaType.AUTO_CONSUME) + .toList().size()) .sum(); if (hasSchema || (!producers.isEmpty()) - || (numActiveConsumers != 0) + || (numActiveConsumersWithoutAutoSchema != 0) || ENTRIES_ADDED_COUNTER_UPDATER.get(this) != 0) { return checkSchemaCompatibleForConsumer(schema); } else { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 4277edb074c5e..fd0b069421275 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -155,6 +155,7 @@ import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.schema.SchemaData; +import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.topics.TopicCompactionStrategy; import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.common.util.DateFormatter; @@ -727,7 +728,8 @@ public CompletableFuture subscribe(SubscriptionOption option) { option.getStartMessageId(), option.getMetadata(), option.isReadCompacted(), option.getInitialPosition(), option.getStartMessageRollbackDurationSec(), option.isReplicatedSubscriptionStateArg(), option.getKeySharedMeta(), - option.getSubscriptionProperties().orElse(Collections.emptyMap()), option.getConsumerEpoch()); + option.getSubscriptionProperties().orElse(Collections.emptyMap()), + option.getConsumerEpoch(), option.getSchemaType()); } private CompletableFuture internalSubscribe(final TransportCnx cnx, String subscriptionName, @@ -740,7 +742,8 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St boolean replicatedSubscriptionStateArg, KeySharedMeta keySharedMeta, Map subscriptionProperties, - long consumerEpoch) { + long consumerEpoch, + SchemaType schemaType) { if (readCompacted && !(subType == SubType.Failover || subType == SubType.Exclusive)) { return FutureUtil.failedFuture(new NotAllowedException( "readCompacted only allowed on failover or exclusive subscriptions")); @@ -828,7 +831,7 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St CompletableFuture future = subscriptionFuture.thenCompose(subscription -> { Consumer consumer = new Consumer(subscription, subType, topic, consumerId, priorityLevel, consumerName, isDurable, cnx, cnx.getAuthRole(), metadata, - readCompacted, keySharedMeta, startMessageId, consumerEpoch); + readCompacted, keySharedMeta, startMessageId, consumerEpoch, schemaType); return addConsumerToSubscription(subscription, consumer).thenCompose(v -> { if (subscription instanceof PersistentSubscription persistentSubscription) { @@ -907,7 +910,7 @@ public CompletableFuture subscribe(final TransportCnx cnx, String subs KeySharedMeta keySharedMeta) { return internalSubscribe(cnx, subscriptionName, consumerId, subType, priorityLevel, consumerName, isDurable, startMessageId, metadata, readCompacted, initialPosition, startMessageRollbackDurationSec, - replicatedSubscriptionStateArg, keySharedMeta, null, DEFAULT_CONSUMER_EPOCH); + replicatedSubscriptionStateArg, keySharedMeta, null, DEFAULT_CONSUMER_EPOCH, null); } private CompletableFuture getDurableSubscription(String subscriptionName, @@ -3107,21 +3110,22 @@ public synchronized OffloadProcessStatus offloadStatus() { @Override public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schema) { - return hasSchema() - .thenCompose((hasSchema) -> { - int numActiveConsumers = subscriptions.values().stream() - .mapToInt(subscription -> subscription.getConsumers().size()) - .sum(); - if (hasSchema - || (!producers.isEmpty()) - || (numActiveConsumers != 0) - || (ledger.getTotalSize() != 0)) { - return checkSchemaCompatibleForConsumer(schema); - } else { - return addSchema(schema).thenCompose(schemaVersion -> - CompletableFuture.completedFuture(null)); - } - }); + return hasSchema().thenCompose((hasSchema) -> { + int numActiveConsumersWithoutAutoSchema = subscriptions.values().stream() + .mapToInt(subscription -> subscription.getConsumers().stream() + .filter(consumer -> consumer.getSchemaType() != SchemaType.AUTO_CONSUME) + .toList().size()) + .sum(); + if (hasSchema + || (!producers.isEmpty()) + || (numActiveConsumersWithoutAutoSchema != 0) + || (ledger.getTotalSize() != 0)) { + return checkSchemaCompatibleForConsumer(schema); + } else { + return addSchema(schema).thenCompose(schemaVersion -> + CompletableFuture.completedFuture(null)); + } + }); } public synchronized void checkReplicatedSubscriptionControllerState() { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java index 21d6a7ed89a56..c8c7c3b2ccc38 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java @@ -1239,6 +1239,79 @@ public void testAutoCreatedSchema(String domain) throws Exception { Assert.assertEquals(admin.schemas().getSchemaInfo(topic2).getType(), SchemaType.STRING); } + @Test(dataProvider = "topicDomain") + public void testSubscribeWithSchemaAfterAutoConsumeNewTopic(String domain) throws Exception { + final String topic = domain + "my-property/my-ns/testSubscribeWithSchemaAfterAutoConsume-1"; + + @Cleanup + Consumer autoConsumer1 = pulsarClient.newConsumer(Schema.AUTO_CONSUME()) + .topic(topic) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName("sub0") + .consumerName("autoConsumer1") + .subscribe(); + @Cleanup + Consumer autoConsumer2 = pulsarClient.newConsumer(Schema.AUTO_CONSUME()) + .topic(topic) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName("sub0") + .consumerName("autoConsumer2") + .subscribe(); + @Cleanup + Consumer autoConsumer3 = pulsarClient.newConsumer(Schema.AUTO_CONSUME()) + .topic(topic) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName("sub1") + .consumerName("autoConsumer3") + .subscribe(); + @Cleanup + Consumer autoConsumer4 = pulsarClient.newConsumer(Schema.AUTO_CONSUME()) + .topic(topic) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName("sub1") + .consumerName("autoConsumer4") + .subscribe(); + try { + log.info("The autoConsumer1 isConnected: " + autoConsumer1.isConnected()); + log.info("The autoConsumer2 isConnected: " + autoConsumer2.isConnected()); + log.info("The autoConsumer3 isConnected: " + autoConsumer3.isConnected()); + log.info("The autoConsumer4 isConnected: " + autoConsumer4.isConnected()); + admin.schemas().getSchemaInfo(topic); + fail("The schema of topic should not exist"); + } catch (PulsarAdminException e) { + assertEquals(e.getStatusCode(), 404); + } + + @Cleanup + Consumer consumerWithSchema1 = pulsarClient.newConsumer(Schema.AVRO(V1Data.class)) + .topic(topic) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName("sub0") + .consumerName("consumerWithSchema-1") + .subscribe(); + @Cleanup + Consumer consumerWithSchema2 = pulsarClient.newConsumer(Schema.AVRO(V1Data.class)) + .topic(topic) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName("sub0") + .consumerName("consumerWithSchema-2") + .subscribe(); + @Cleanup + Consumer consumerWithSchema3 = pulsarClient.newConsumer(Schema.AVRO(V1Data.class)) + .topic(topic) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName("sub1") + .consumerName("consumerWithSchema-3") + .subscribe(); + @Cleanup + Consumer consumerWithSchema4 = pulsarClient.newConsumer(Schema.AVRO(V1Data.class)) + .topic(topic) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName("sub1") + .consumerName("consumerWithSchema-4") + .subscribe(); + } + @DataProvider(name = "keyEncodingType") public static Object[] keyEncodingType() { return new Object[] { KeyValueEncodingType.SEPARATED, KeyValueEncodingType.INLINE }; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 18abb5a52c45f..beaa34bf20520 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -84,6 +84,7 @@ import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; import org.apache.pulsar.client.impl.crypto.MessageCryptoBc; +import org.apache.pulsar.client.impl.schema.AutoConsumeSchema; import org.apache.pulsar.client.impl.transaction.TransactionImpl; import org.apache.pulsar.client.util.ExecutorProvider; import org.apache.pulsar.client.util.RetryMessageUtil; @@ -811,6 +812,11 @@ public void connectionOpened(final ClientCnx cnx) { if (si != null && (SchemaType.BYTES == si.getType() || SchemaType.NONE == si.getType())) { // don't set schema for Schema.BYTES si = null; + } else { + if (schema instanceof AutoConsumeSchema + && Commands.peerSupportsCarryAutoConsumeSchemaToBroker(cnx.getRemoteEndpointProtocolVersion())) { + si = AutoConsumeSchema.SCHEMA_INFO; + } } // startMessageRollbackDurationInSec should be consider only once when consumer connects to first time long startMessageRollbackDuration = (startMessageRollbackDurationInSec > 0 diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AutoConsumeSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AutoConsumeSchema.java index 33fcd18876be6..82a3b69da20b6 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AutoConsumeSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AutoConsumeSchema.java @@ -57,6 +57,12 @@ public class AutoConsumeSchema implements Schema { private SchemaInfoProvider schemaInfoProvider; + public static final SchemaInfo SCHEMA_INFO = SchemaInfoImpl.builder() + .name("AutoConsume") + .type(SchemaType.AUTO_CONSUME) + .schema(new byte[0]) + .build(); + private ConcurrentMap> initSchemaMap() { ConcurrentMap> schemaMap = new ConcurrentHashMap<>(); // The Schema.BYTES will not be uploaded to the broker and store in the schema storage, diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java index 081dfe4275b24..8a5684cf676b0 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java @@ -771,7 +771,9 @@ public static ByteBuf newProducer(String topic, long producerId, long requestId, } private static Schema.Type getSchemaType(SchemaType type) { - if (type.getValue() < 0) { + if (type == SchemaType.AUTO_CONSUME) { + return Schema.Type.AutoConsume; + } else if (type.getValue() < 0) { return Schema.Type.None; } else { return Schema.Type.valueOf(type.getValue()); @@ -779,7 +781,9 @@ private static Schema.Type getSchemaType(SchemaType type) { } public static SchemaType getSchemaType(Schema.Type type) { - if (type.getValue() < 0) { + if (type == Schema.Type.AutoConsume) { + return SchemaType.AUTO_CONSUME; + } else if (type.getValue() < 0) { // this is unexpected return SchemaType.NONE; } else { @@ -1965,6 +1969,10 @@ public static boolean peerSupportsAckReceipt(int peerVersion) { return peerVersion >= ProtocolVersion.v17.getValue(); } + public static boolean peerSupportsCarryAutoConsumeSchemaToBroker(int peerVersion) { + return peerVersion >= ProtocolVersion.v21.getValue(); + } + private static org.apache.pulsar.common.api.proto.ProducerAccessMode convertProducerAccessMode( ProducerAccessMode accessMode) { switch (accessMode) { diff --git a/pulsar-common/src/main/proto/PulsarApi.proto b/pulsar-common/src/main/proto/PulsarApi.proto index acf75eab85826..d9c41eeec9740 100644 --- a/pulsar-common/src/main/proto/PulsarApi.proto +++ b/pulsar-common/src/main/proto/PulsarApi.proto @@ -45,6 +45,7 @@ message Schema { LocalTime = 18; LocalDateTime = 19; ProtobufNative = 20; + AutoConsume = 21; } required string name = 1; @@ -263,6 +264,7 @@ enum ProtocolVersion { v18 = 18; // Add client support for broker entry metadata v19 = 19; // Add CommandTcClientConnectRequest and CommandTcClientConnectResponse v20 = 20; // Add client support for topic migration redirection CommandTopicMigrated + v21 = 21; // Carry the AUTO_CONSUME schema to the Broker after this version } message CommandConnect { From e973388eb77f226e0546f88c87821f8c7ee26281 Mon Sep 17 00:00:00 2001 From: feynmanlin <315157973@qq.com> Date: Wed, 8 Mar 2023 16:48:11 +0800 Subject: [PATCH 087/174] [fix][broker] Fixed history load not releasing (#19726) --- .../loadbalance/ModularLoadManagerStrategy.java | 7 +++++++ .../impl/LeastResourceUsageWithWeight.java | 9 +++++++-- .../impl/ModularLoadManagerImpl.java | 1 + .../ModularLoadManagerStrategyTest.java | 17 +++++++++++++++++ 4 files changed, 32 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerStrategy.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerStrategy.java index 2be8200aef5c1..91a619eafc226 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerStrategy.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerStrategy.java @@ -46,6 +46,13 @@ public interface ModularLoadManagerStrategy { Optional selectBroker(Set candidates, BundleData bundleToAssign, LoadData loadData, ServiceConfiguration conf); + /** + * Triggered when active brokers change. + */ + default void onActiveBrokersChange(Set activeBrokers) { + + } + /** * Create a placement strategy using the configuration. * diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LeastResourceUsageWithWeight.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LeastResourceUsageWithWeight.java index f50f9ed36538a..ab3e63e9d133f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LeastResourceUsageWithWeight.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LeastResourceUsageWithWeight.java @@ -128,8 +128,9 @@ private double updateAndGetMaxResourceUsageWithWeight(String broker, BrokerData * @return The name of the selected broker as it appears on ZooKeeper. */ @Override - public Optional selectBroker(Set candidates, BundleData bundleToAssign, LoadData loadData, - ServiceConfiguration conf) { + public synchronized Optional selectBroker(Set candidates, BundleData bundleToAssign, + LoadData loadData, + ServiceConfiguration conf) { if (candidates.isEmpty()) { log.info("There are no available brokers as candidates at this point for bundle: {}", bundleToAssign); return Optional.empty(); @@ -167,4 +168,8 @@ public Optional selectBroker(Set candidates, BundleData bundleTo } return Optional.of(bestBrokers.get(ThreadLocalRandom.current().nextInt(bestBrokers.size()))); } + @Override + public synchronized void onActiveBrokersChange(Set activeBrokers) { + brokerAvgResourceUsageWithWeight.keySet().removeIf((key) -> !activeBrokers.contains(key)); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java index ea2472eb199d4..59f9836b30975 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java @@ -494,6 +494,7 @@ private void cleanupDeadBrokersData() { for (LoadSheddingStrategy loadSheddingStrategy : loadSheddingPipeline) { loadSheddingStrategy.onActiveBrokersChange(activeBrokers); } + placementStrategy.onActiveBrokersChange(activeBrokers); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerStrategyTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerStrategyTest.java index b967deaa72686..c64c9950a95a9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerStrategyTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerStrategyTest.java @@ -228,6 +228,23 @@ public void testRoundRobinBrokerSelector() throws IllegalAccessException { assertEquals(((AtomicInteger) FieldUtils.readDeclaredField(strategy, "count", true)).get(), 0); } + public void testActiveBrokersChange() throws Exception { + LoadData loadData = new LoadData(); + Map brokerDataMap = loadData.getBrokerData(); + brokerDataMap.put("1", initBrokerData()); + brokerDataMap.put("2", initBrokerData()); + brokerDataMap.put("3", initBrokerData()); + ServiceConfiguration conf = new ServiceConfiguration(); + LeastResourceUsageWithWeight strategy = new LeastResourceUsageWithWeight(); + strategy.selectBroker(brokerDataMap.keySet(), new BundleData(), loadData, conf); + Field field = LeastResourceUsageWithWeight.class.getDeclaredField("brokerAvgResourceUsageWithWeight"); + field.setAccessible(true); + Map map = (Map) field.get(strategy); + assertEquals(map.size(), 3); + strategy.onActiveBrokersChange(new HashSet<>()); + assertEquals(map.size(), 0); + } + private BrokerData initBrokerData(double usage, double limit) { LocalBrokerData localBrokerData = new LocalBrokerData(); localBrokerData.setCpu(new ResourceUsage(usage, limit)); From cbd799f05eb26aeac5ffd5bbb9751ad9e5928dd3 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Thu, 9 Mar 2023 10:17:39 +0800 Subject: [PATCH 088/174] [fix][meta] Fix deadlock causes session notification not to work (#19754) ### Motivation This is a namespace bundle double-owners problem. We found it in the memory dumps. The memory dumps show that the notification thread has been blocked for a long time by the leader election deadlock, And many notifications are blocked in the executor queue. This causes we can't to revalidate the locks, and they are still thinking them working well. For private reasons, I can't share the namespace bundle snapshot, but the blocked thread easily explains it. image ^^ blocked thread image ^^ executor queue ### Modifications - Avoid putting the new task to single thread executor causes deadlock. --- .../coordination/impl/LeaderElectionImpl.java | 10 ++- .../impl/LeaderElectionImplTest.java | 65 +++++++++++++++++++ 2 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java index 409d49dcd26ab..ad2a5bef70610 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java @@ -19,6 +19,7 @@ package org.apache.pulsar.metadata.coordination.impl; import com.fasterxml.jackson.databind.type.TypeFactory; +import com.google.common.annotations.VisibleForTesting; import java.util.EnumSet; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -111,13 +112,13 @@ private synchronized CompletableFuture elect() { } else { return tryToBecomeLeader(); } - }).thenComposeAsync(leaderElectionState -> { + }).thenCompose(leaderElectionState -> { // make sure that the cache contains the current leader // so that getLeaderValueIfPresent works on all brokers cache.refresh(path); return cache.get(path) .thenApply(__ -> leaderElectionState); - }, executor); + }); } private synchronized CompletableFuture handleExistingLeaderValue(GetResult res) { @@ -336,4 +337,9 @@ private void handlePathNotification(Notification notification) { } } } + + @VisibleForTesting + protected ScheduledExecutorService getSchedulerExecutor() { + return executor; + } } diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java new file mode 100644 index 0000000000000..027521d2ffc17 --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.metadata.coordination.impl; + +import java.util.concurrent.CompletableFuture; +import java.util.function.Supplier; +import lombok.Cleanup; +import org.apache.pulsar.metadata.BaseMetadataStoreTest; +import org.apache.pulsar.metadata.api.MetadataStoreConfig; +import org.apache.pulsar.metadata.api.coordination.CoordinationService; +import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; +import org.testng.annotations.Test; + +public class LeaderElectionImplTest extends BaseMetadataStoreTest { + + @Test(dataProvider = "impl", timeOut = 10000) + public void validateDeadLock(String provider, Supplier urlSupplier) + throws Exception { + if (provider.equals("Memory") || provider.equals("RocksDB")) { + // There are no multiple sessions for the local memory provider + return; + } + + @Cleanup + MetadataStoreExtended store = MetadataStoreExtended.create(urlSupplier.get(), + MetadataStoreConfig.builder().build()); + + String path = newKey(); + + @Cleanup + CoordinationService cs = new CoordinationServiceImpl(store); + + @Cleanup + LeaderElectionImpl le = (LeaderElectionImpl) cs.getLeaderElection(String.class, + path, __ -> { + }); + final CompletableFuture blockFuture = new CompletableFuture<>(); + // simulate handleSessionNotification method logic + le.getSchedulerExecutor().execute(() -> { + try { + le.elect("test-2").join(); + blockFuture.complete(null); + } catch (Throwable ex) { + blockFuture.completeExceptionally(ex); + } + }); + blockFuture.join(); + } +} From cdeef00c5f6a5bd3197b4ca6de0a0505b18835d8 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Thu, 9 Mar 2023 10:38:21 +0800 Subject: [PATCH 089/174] [fix] [broker] Topic close failure leaves subscription in a permanent fence state (#19692) Motivation : After a Topic close failure or a delete failure, the fence state will be reset to get the topic back to work,but it will not reset the fence state of the subscription, which will result in the consumer never being able to connect to the broker. Modifications: Reset the fence state of subscriptions if the operation of topic close is failed. --- .../persistent/PersistentSubscription.java | 39 ++++++++++--- .../service/persistent/PersistentTopic.java | 1 + .../persistent/PersistentTopicTest.java | 55 +++++++++++++++++++ 3 files changed, 87 insertions(+), 8 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index dfeca26750342..e07d7bee500a7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -126,6 +126,7 @@ public class PersistentSubscription extends AbstractSubscription implements Subs private volatile ReplicatedSubscriptionSnapshotCache replicatedSubscriptionSnapshotCache; private final PendingAckHandle pendingAckHandle; private volatile Map subscriptionProperties; + private volatile CompletableFuture fenceFuture; static { REPLICATED_SUBSCRIPTION_CURSOR_PROPERTIES.put(REPLICATED_SUBSCRIPTION_PROPERTY, 1L); @@ -897,7 +898,10 @@ public CompletableFuture close() { */ @Override public synchronized CompletableFuture disconnect() { - CompletableFuture disconnectFuture = new CompletableFuture<>(); + if (fenceFuture != null){ + return fenceFuture; + } + fenceFuture = new CompletableFuture<>(); // block any further consumers on this subscription IS_FENCED_UPDATER.set(this, TRUE); @@ -905,19 +909,38 @@ public synchronized CompletableFuture disconnect() { (dispatcher != null ? dispatcher.close() : CompletableFuture.completedFuture(null)) .thenCompose(v -> close()).thenRun(() -> { log.info("[{}][{}] Successfully disconnected and closed subscription", topicName, subName); - disconnectFuture.complete(null); + fenceFuture.complete(null); }).exceptionally(exception -> { - IS_FENCED_UPDATER.set(this, FALSE); - if (dispatcher != null) { - dispatcher.reset(); - } log.error("[{}][{}] Error disconnecting consumers from subscription", topicName, subName, exception); - disconnectFuture.completeExceptionally(exception); + fenceFuture.completeExceptionally(exception); + resumeAfterFence(); return null; }); + return fenceFuture; + } - return disconnectFuture; + /** + * Resume subscription after topic deletion or close failure. + */ + public synchronized void resumeAfterFence() { + // If "fenceFuture" is null, it means that "disconnect" has never been called. + if (fenceFuture != null) { + fenceFuture.whenComplete((ignore, ignoreEx) -> { + synchronized (PersistentSubscription.this) { + try { + if (IS_FENCED_UPDATER.compareAndSet(this, TRUE, FALSE)) { + if (dispatcher != null) { + dispatcher.reset(); + } + } + fenceFuture = null; + } catch (Exception ex) { + log.error("[{}] Resume subscription [{}] failure", topicName, subName, ex); + } + } + }); + } } /** diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index fd0b069421275..3b9fbbceb2422 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -3234,6 +3234,7 @@ private void fenceTopicToCloseOrDelete() { } private void unfenceTopicToResume() { + subscriptions.values().forEach(sub -> sub.resumeAfterFence()); isFenced = false; isClosingOrDeleting = false; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java index 565157508683f..80a79e0234de4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java @@ -19,7 +19,10 @@ package org.apache.pulsar.broker.service.persistent; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; @@ -39,11 +42,15 @@ import java.util.Collection; import java.util.List; import java.util.UUID; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import lombok.Cleanup; import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.BrokerTestBase; import org.apache.pulsar.broker.stats.PrometheusMetricsTest; import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsGenerator; @@ -470,4 +477,52 @@ public void testCompatibilityWithPartitionKeyword() throws PulsarAdminException, Assert.assertThrows(PulsarAdminException.NotFoundException.class, () -> admin.topics().getPartitionedTopicMetadata(topicName)); } + + @Test + public void testDeleteTopicFail() throws Exception { + final String fullyTopicName = "persistent://prop/ns-abc/" + "tp_" + + UUID.randomUUID().toString().replaceAll("-", ""); + // Mock topic. + BrokerService brokerService = spy(pulsar.getBrokerService()); + doReturn(brokerService).when(pulsar).getBrokerService(); + + // Create a sub, and send one message. + Consumer consumer1 = pulsarClient.newConsumer(Schema.STRING).topic(fullyTopicName).subscriptionName("sub1") + .subscribe(); + consumer1.close(); + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(fullyTopicName).create(); + producer.send("1"); + producer.close(); + + // Make a failed delete operation. + AtomicBoolean makeDeletedFailed = new AtomicBoolean(true); + PersistentTopic persistentTopic = (PersistentTopic) brokerService.getTopic(fullyTopicName, false).get().get(); + doAnswer(invocation -> { + CompletableFuture future = (CompletableFuture) invocation.getArguments()[1]; + if (makeDeletedFailed.get()) { + future.completeExceptionally(new RuntimeException("mock ex for test")); + } else { + future.complete(null); + } + return null; + }).when(brokerService) + .deleteTopicAuthenticationWithRetry(any(String.class), any(CompletableFuture.class), anyInt()); + try { + persistentTopic.delete().get(); + } catch (Exception e) { + org.testng.Assert.assertTrue(e instanceof ExecutionException); + org.testng.Assert.assertTrue(e.getCause() instanceof java.lang.RuntimeException); + org.testng.Assert.assertEquals(e.getCause().getMessage(), "mock ex for test"); + } + + // Assert topic works after deleting failure. + Consumer consumer2 = pulsarClient.newConsumer(Schema.STRING).topic(fullyTopicName).subscriptionName("sub1") + .subscribe(); + org.testng.Assert.assertEquals("1", consumer2.receive(2, TimeUnit.SECONDS).getValue()); + consumer2.close(); + + // Make delete success. + makeDeletedFailed.set(false); + persistentTopic.delete().get(); + } } From 401fb055ed428b7f3e33a49cea78b25b04f7448e Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Thu, 9 Mar 2023 15:13:50 +0800 Subject: [PATCH 090/174] [fix] [broker] delete topic failed if disabled system topic (#19735) Motivation: After PR #18823, The cmd delete topic will fail if disabled the feature system topic. Modifications: do not delete the system policy if disabled the feature system topic --- .../service/persistent/PersistentTopic.java | 3 +- .../client/impl/DisabledSystemTopicTest.java | 61 +++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/client/impl/DisabledSystemTopicTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 3b9fbbceb2422..79717ed57303f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -1246,7 +1246,8 @@ private CompletableFuture delete(boolean failIfHasSubscriptions, deleteTopicAuthenticationFuture.thenCompose(ignore -> deleteSchema()) .thenCompose(ignore -> { - if (!SystemTopicNames.isTopicPoliciesSystemTopic(topic)) { + if (!SystemTopicNames.isTopicPoliciesSystemTopic(topic) + && brokerService.getPulsar().getConfiguration().isSystemTopicEnabled()) { return deleteTopicPolicies(); } else { return CompletableFuture.completedFuture(null); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/DisabledSystemTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/DisabledSystemTopicTest.java new file mode 100644 index 0000000000000..7747d3a576c90 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/DisabledSystemTopicTest.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import java.util.UUID; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker") +public class DisabledSystemTopicTest extends ProducerConsumerBase { + + @Override + @BeforeMethod + public void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + } + + @Override + @AfterMethod(alwaysRun = true) + public void cleanup() throws Exception { + super.internalCleanup(); + } + + protected void doInitConf() throws Exception { + super.doInitConf(); + conf.setTransactionCoordinatorEnabled(false); + conf.setSystemTopicEnabled(false); + } + + @Test + public void testDeleteTopic() throws Exception { + String topicName = "persistent://my-property/my-ns/tp_" + UUID.randomUUID().toString(); + + admin.topics().createNonPartitionedTopic(topicName); + admin.topics().delete(topicName, false); + + admin.topics().createPartitionedTopic(topicName, 3); + admin.topics().deletePartitionedTopic(topicName); + } +} From 4a450aa1091d15db14e3d69454c3bb89b3ae6d7d Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Thu, 9 Mar 2023 16:18:00 +0800 Subject: [PATCH 091/174] [fix][broker] Fix potential exception cause the policy service init fail. (#19746) --- .../SystemTopicBasedTopicPoliciesService.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index 9ec374264e91c..9b10055f36fac 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -341,20 +341,19 @@ private void initPolicesCache(SystemTopicClient.Reader reader, Comp return; } if (hasMore) { - reader.readNextAsync().whenComplete((msg, e) -> { - if (e != null) { - log.error("[{}] Failed to read event from the system topic.", - reader.getSystemTopic().getTopicName(), e); - future.completeExceptionally(e); - cleanCacheAndCloseReader(reader.getSystemTopic().getTopicName().getNamespaceObject(), false); - return; - } + reader.readNextAsync().thenAccept(msg -> { refreshTopicPoliciesCache(msg); if (log.isDebugEnabled()) { log.debug("[{}] Loop next event reading for system topic.", reader.getSystemTopic().getTopicName().getNamespaceObject()); } initPolicesCache(reader, future); + }).exceptionally(e -> { + log.error("[{}] Failed to read event from the system topic.", + reader.getSystemTopic().getTopicName(), e); + future.completeExceptionally(e); + cleanCacheAndCloseReader(reader.getSystemTopic().getTopicName().getNamespaceObject(), false); + return null; }); } else { if (log.isDebugEnabled()) { From d4930a31c052dd8fcd5982b649898967a24f8961 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Thu, 9 Mar 2023 01:09:34 -0800 Subject: [PATCH 092/174] [fix][io] KCA: 'desanitize' topic name for the pulsar's ctx calls (#19756) --- .../io/kafka/connect/KafkaConnectSink.java | 22 +++++++++- .../connect/PulsarKafkaSinkTaskContext.java | 14 ++++-- .../kafka/connect/KafkaConnectSinkTest.java | 44 +++++++++++++++++++ 3 files changed, 75 insertions(+), 5 deletions(-) diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java index 37d0987e61023..efbad2ef47ae0 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java @@ -95,6 +95,12 @@ public class KafkaConnectSink implements Sink { CacheBuilder.newBuilder().maximumSize(1000) .expireAfterAccess(30, TimeUnit.MINUTES).build(); + // Can't really safely expire these entries. If we do, we could end up with + // a sanitized topic name that used in e.g. resume() after a long pause but can't be + // // re-resolved into a form usable for Pulsar. + private final Cache desanitizedTopicCache = + CacheBuilder.newBuilder().build(); + private int maxBatchBitsForOffset = 12; private boolean useIndexAsOffset = true; @@ -184,7 +190,18 @@ public void open(Map config, SinkContext ctx) throws Exception { }); task = (SinkTask) taskClass.getConstructor().newInstance(); taskContext = - new PulsarKafkaSinkTaskContext(configs.get(0), ctx, task::open); + new PulsarKafkaSinkTaskContext(configs.get(0), ctx, task::open, kafkaName -> { + if (sanitizeTopicName) { + String pulsarTopicName = desanitizedTopicCache.getIfPresent(kafkaName); + if (log.isDebugEnabled()) { + log.debug("desanitizedTopicCache got: kafkaName: {}, pulsarTopicName: {}", + kafkaName, pulsarTopicName); + } + return pulsarTopicName != null ? pulsarTopicName : kafkaName; + } else { + return kafkaName; + } + }); task.initialize(taskContext); task.start(configs.get(0)); @@ -486,6 +503,9 @@ protected String sanitizeNameIfNeeded(String name, boolean sanitize) { if (sanitizedName.matches("^[^a-zA-Z_].*")) { sanitizedName = "_" + sanitizedName; } + // do this once, sanitize() can be called on already sanitized name + // so avoid replacing with (sanitizedName -> sanitizedName). + desanitizedTopicCache.get(sanitizedName, () -> name); return sanitizedName; }); } catch (ExecutionException e) { diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaSinkTaskContext.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaSinkTaskContext.java index 99a8bf2908237..7a908b553a89a 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaSinkTaskContext.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaSinkTaskContext.java @@ -33,6 +33,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; +import java.util.function.Function; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.TopicPartition; @@ -49,6 +50,7 @@ public class PulsarKafkaSinkTaskContext implements SinkTaskContext { private final SinkContext ctx; private final OffsetBackingStore offsetStore; + private Function desanitizeTopicName; private final String topicNamespace; private final Consumer> onPartitionChange; private final AtomicBoolean runRepartition = new AtomicBoolean(false); @@ -57,11 +59,13 @@ public class PulsarKafkaSinkTaskContext implements SinkTaskContext { public PulsarKafkaSinkTaskContext(Map config, SinkContext ctx, - Consumer> onPartitionChange) { + Consumer> onPartitionChange, + Function desanitizeTopicName) { this.config = config; this.ctx = ctx; offsetStore = new PulsarOffsetBackingStore(ctx.getPulsarClient()); + this.desanitizeTopicName = desanitizeTopicName; PulsarKafkaWorkerConfig pulsarKafkaWorkerConfig = new PulsarKafkaWorkerConfig(config); offsetStore.configure(pulsarKafkaWorkerConfig); offsetStore.start(); @@ -144,7 +148,9 @@ private void fillOffsetMap(Map offsetMap, TopicPartition private void seekAndUpdateOffset(TopicPartition topicPartition, long offset) { try { - ctx.seek(topicPartition.topic(), topicPartition.partition(), MessageIdUtils.getMessageId(offset)); + ctx.seek(desanitizeTopicName.apply(topicPartition.topic()), + topicPartition.partition(), + MessageIdUtils.getMessageId(offset)); } catch (PulsarClientException e) { log.error("Failed to seek topic {} partition {} offset {}", topicPartition.topic(), topicPartition.partition(), offset, e); @@ -202,7 +208,7 @@ public Set assignment() { public void pause(TopicPartition... topicPartitions) { for (TopicPartition tp: topicPartitions) { try { - ctx.pause(tp.topic(), tp.partition()); + ctx.pause(desanitizeTopicName.apply(tp.topic()), tp.partition()); } catch (PulsarClientException e) { log.error("Failed to pause topic {} partition {}", tp.topic(), tp.partition(), e); throw new RuntimeException("Failed to pause topic " + tp.topic() + " partition " + tp.partition(), e); @@ -214,7 +220,7 @@ public void pause(TopicPartition... topicPartitions) { public void resume(TopicPartition... topicPartitions) { for (TopicPartition tp: topicPartitions) { try { - ctx.resume(tp.topic(), tp.partition()); + ctx.resume(desanitizeTopicName.apply(tp.topic()), tp.partition()); } catch (PulsarClientException e) { log.error("Failed to resume topic {} partition {}", tp.topic(), tp.partition(), e); throw new RuntimeException("Failed to resume topic " + tp.topic() + " partition " + tp.partition(), e); diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java index fe2def250233c..567562d338b98 100644 --- a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java +++ b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java @@ -314,6 +314,50 @@ public void seekPauseResumeTest() throws Exception { sink.close(); } + @Test + public void seekPauseResumeWithSanitizeTest() throws Exception { + KafkaConnectSink sink = new KafkaConnectSink(); + props.put("sanitizeTopicName", "true"); + sink.open(props, context); + + String pulsarTopicName = "persistent://a-b/c-d/fake-topic.a"; + + final GenericRecord rec = getGenericRecord("value", Schema.STRING); + Message msg = mock(MessageImpl.class); + when(msg.getValue()).thenReturn(rec); + final MessageId msgId = new MessageIdImpl(10, 10, 0); + when(msg.getMessageId()).thenReturn(msgId); + + final AtomicInteger status = new AtomicInteger(0); + Record record = PulsarRecord.builder() + .topicName(pulsarTopicName) + .message(msg) + .ackFunction(status::incrementAndGet) + .failFunction(status::decrementAndGet) + .schema(Schema.STRING) + .build(); + + sink.write(record); + sink.flush(); + + assertEquals(status.get(), 1); + + final TopicPartition tp = new TopicPartition(sink.sanitizeNameIfNeeded(pulsarTopicName, true), 0); + assertNotEquals(MessageIdUtils.getOffset(msgId), 0); + assertEquals(sink.currentOffset(tp.topic(), tp.partition()), MessageIdUtils.getOffset(msgId)); + + sink.taskContext.offset(tp, 0); + verify(context, times(1)).seek(pulsarTopicName, + tp.partition(), MessageIdUtils.getMessageId(0)); + assertEquals(sink.currentOffset(tp.topic(), tp.partition()), 0); + + sink.taskContext.pause(tp); + verify(context, times(1)).pause(pulsarTopicName, tp.partition()); + sink.taskContext.resume(tp); + verify(context, times(1)).resume(pulsarTopicName, tp.partition()); + + sink.close(); + } @Test public void subscriptionTypeTest() throws Exception { From 7d5d2df0f0caf7630e89e4aab79a87e632475ab8 Mon Sep 17 00:00:00 2001 From: HuangZeGui Date: Thu, 9 Mar 2023 19:35:58 +0800 Subject: [PATCH 093/174] [fix][doc] Update the comment of the method of OwnershipCache (#19553) Co-authored-by: huangzegui --- .../org/apache/pulsar/broker/namespace/OwnershipCache.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java index a9dd44d4589d9..7d0b5a4147721 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java @@ -158,8 +158,7 @@ public CompletableFuture checkOwnershipAsync(NamespaceBundle bundle) { * @param suName * name of the ServiceUnit * @return The ephemeral node data showing the current ownership info in ZooKeeper - * @throws Exception - * throws exception if no ownership info is found + * or empty if no ownership info is found */ public CompletableFuture> getOwnerAsync(NamespaceBundle suName) { CompletableFuture ownedBundleFuture = ownedBundlesCache.getIfPresent(suName); From 42a65aaa55e03c147adabac55ad5227fc8c8f596 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Thu, 9 Mar 2023 20:27:06 +0800 Subject: [PATCH 094/174] [fix][test]testRedirectOfGetCoordinatorInternalStats (#19715) Fix https://github.com/apache/pulsar/issues/18600 ### Motivation Reason for test failure: 1. Create 16 brokers and 3 transaction coordinator topics. 2. The 3 transaction coordinator topics are assigned to broker-0. 3. Because the admin is connected to broker-0, we do not hope the topics are assigned to broker-0. 4. Recreate the 3 transaction coordinator topics. 5. The topics are assigned to broker-0 again. 6. Repeat execute steps 3,4,5 7. Timeout ### Modifications Do not recreate the transaction coordinator topics. Recreate an admin connect the broker which is not the owner of the transaction coordinator topics. --- .../AdminApiTransactionMultiBrokerTest.java | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionMultiBrokerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionMultiBrokerTest.java index 463a65fc8ae89..52aadde7b2621 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionMultiBrokerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionMultiBrokerTest.java @@ -18,13 +18,14 @@ */ package org.apache.pulsar.broker.admin.v3; +import static org.mockito.Mockito.spy; import java.util.Map; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.transaction.TransactionTestBase; +import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.common.naming.SystemTopicNames; -import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -46,21 +47,26 @@ protected void cleanup() throws Exception { super.internalCleanup(); } + /** + * This test is used to verify the redirect request of `getCoordinatorInternalStats`. + *

+ * 1. Set up 16 broker and create 3 transaction coordinator topic. + * 2. The 3 transaction coordinator topic will be assigned to these brokers through some kind of + * load-balancing strategy. (In current implementations, they tend to be assigned to a broker.) + * 3. Find a broker x which is not the owner of the transaction coordinator topic. + * 4. Create a admin connected to broker x, and use the admin to call ` getCoordinatorInternalStats`. + *

+ */ @Test public void testRedirectOfGetCoordinatorInternalStats() throws Exception { - Map map = admin.lookups() + PulsarAdmin localAdmin = this.admin; + Map map = localAdmin.lookups() .lookupPartitionedTopic(SystemTopicNames.TRANSACTION_COORDINATOR_ASSIGN.toString()); - while (map.containsValue(getPulsarServiceList().get(0).getBrokerServiceUrl())) { - pulsarServiceList.get(0).getPulsarResources() - .getNamespaceResources() - .getPartitionedTopicResources() - .deletePartitionedTopicAsync(SystemTopicNames.TRANSACTION_COORDINATOR_ASSIGN); - pulsarServiceList.get(0).getPulsarResources() - .getNamespaceResources() - .getPartitionedTopicResources() - .createPartitionedTopic(SystemTopicNames.TRANSACTION_COORDINATOR_ASSIGN, - new PartitionedTopicMetadata(NUM_PARTITIONS)); - map = admin.lookups().lookupPartitionedTopic(SystemTopicNames.TRANSACTION_COORDINATOR_ASSIGN.toString()); + + for (int i = 0; map.containsValue(getPulsarServiceList().get(i).getBrokerServiceUrl()); i++) { + if (!map.containsValue(getPulsarServiceList().get(i + 1).getBrokerServiceUrl())) + localAdmin = spy(createNewPulsarAdmin(PulsarAdmin.builder() + .serviceHttpUrl(pulsarServiceList.get(i + 1).getWebServiceAddress()))); } if (pulsarClient != null) { pulsarClient.shutdown(); @@ -72,7 +78,7 @@ public void testRedirectOfGetCoordinatorInternalStats() throws Exception { .enableTransaction(true) .build(); for (int i = 0; i < NUM_PARTITIONS; i++) { - admin.transactions().getCoordinatorInternalStats(i, false); + localAdmin.transactions().getCoordinatorInternalStats(i, false); } } } From bae28f90c39bd85a25a35e6195440c7558251ddb Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Thu, 9 Mar 2023 23:07:48 +0800 Subject: [PATCH 095/174] [fix] [doc] fix multiple apis in the automatically generated documentation use the same anchor point (#19193) Generate operationId using the specified rule `{className}_{methodName}` of swagger. --- pulsar-broker/pom.xml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index 8fc1527a6e3e8..7442d95e467e6 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -697,11 +697,13 @@ com.github.kongchen swagger-maven-plugin - 3.1.7 + 3.1.8 false + {{className}}_{{methodName}} + json org.apache.pulsar.broker.admin.v2.Bookies org.apache.pulsar.broker.admin.v2.BrokerStats @@ -737,6 +739,8 @@ false + {{className}}_{{methodName}} + json org.apache.pulsar.broker.lookup.v2 http,https /lookup @@ -754,6 +758,8 @@ false + {{className}}_{{methodName}} + json org.apache.pulsar.broker.admin.v3.Functions http,https /admin/v3 @@ -771,6 +777,8 @@ false + {{className}}_{{methodName}} + json org.apache.pulsar.broker.admin.v3.Transactions http,https /admin/v3 @@ -788,6 +796,8 @@ false + {{className}}_{{methodName}} + json org.apache.pulsar.broker.admin.v3.Sources http,https /admin/v3 @@ -805,6 +815,8 @@ false + {{className}}_{{methodName}} + json org.apache.pulsar.broker.admin.v3.Sinks http,https /admin/v3 @@ -822,6 +834,8 @@ false + {{className}}_{{methodName}} + json org.apache.pulsar.broker.admin.v3.Packages http,https /admin/v3 From da30b2e211d0a097abfa8d0392f1fd4b13a46328 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Thu, 9 Mar 2023 07:36:09 -0800 Subject: [PATCH 096/174] [StructuredEventLog] Added support for Log4j2 structured logging if available (#16353) --- structured-event-log/pom.xml | 7 + .../structuredeventlog/Initializer.java | 44 ++++ .../StructuredEventLog.java | 6 +- .../log4j2/Log4j2Event.java | 208 ++++++++++++++++++ .../log4j2/Log4j2StructuredEventLog.java | 46 ++++ .../log4j2/package-info.java | 19 ++ .../slf4j/StructuredEventLogTest.java | 32 +-- 7 files changed, 342 insertions(+), 20 deletions(-) create mode 100644 structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/Initializer.java create mode 100644 structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/log4j2/Log4j2Event.java create mode 100644 structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/log4j2/Log4j2StructuredEventLog.java create mode 100644 structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/log4j2/package-info.java diff --git a/structured-event-log/pom.xml b/structured-event-log/pom.xml index 6daaa21991515..8c058306fc278 100644 --- a/structured-event-log/pom.xml +++ b/structured-event-log/pom.xml @@ -37,6 +37,13 @@ org.slf4j slf4j-api + + + + org.apache.logging.log4j + log4j-core + provided + org.hamcrest hamcrest diff --git a/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/Initializer.java b/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/Initializer.java new file mode 100644 index 0000000000000..bb78c21e75631 --- /dev/null +++ b/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/Initializer.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.structuredeventlog; + +import org.apache.pulsar.structuredeventlog.log4j2.Log4j2StructuredEventLog; +import org.apache.pulsar.structuredeventlog.slf4j.Slf4jStructuredEventLog; + +class Initializer { + static StructuredEventLog get() { + return INSTANCE; + } + + private static final StructuredEventLog INSTANCE; + + static { + StructuredEventLog log = null; + try { + // Use Log4j2 if available in the classpath + Class.forName("org.apache.logging.log4j.LogManager"); + log = Log4j2StructuredEventLog.INSTANCE; + } catch (Throwable t) { + // Fallback to Slf4j otherwise + log = Slf4jStructuredEventLog.INSTANCE; + } + + INSTANCE = log; + } +} diff --git a/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/StructuredEventLog.java b/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/StructuredEventLog.java index bc64545416b7f..30e6e417f0448 100644 --- a/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/StructuredEventLog.java +++ b/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/StructuredEventLog.java @@ -18,8 +18,6 @@ */ package org.apache.pulsar.structuredeventlog; -import org.apache.pulsar.structuredeventlog.slf4j.Slf4jStructuredEventLog; - /** * Structured event logging interface * @@ -85,7 +83,7 @@ public interface StructuredEventLog { /** * Create a new logger object, from which root events can be created. */ - static StructuredEventLog newLogger() { - return Slf4jStructuredEventLog.INSTANCE; + static StructuredEventLog get() { + return Initializer.get(); } } diff --git a/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/log4j2/Log4j2Event.java b/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/log4j2/Log4j2Event.java new file mode 100644 index 0000000000000..187d2064b3888 --- /dev/null +++ b/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/log4j2/Log4j2Event.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.structuredeventlog.log4j2; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.StringMapMessage; +import org.apache.pulsar.structuredeventlog.Event; +import org.apache.pulsar.structuredeventlog.EventResources; +import org.apache.pulsar.structuredeventlog.EventResourcesImpl; + +class Log4j2Event implements Event { + private static final Logger stringLogger = LogManager.getLogger(); + + private final Clock clock; + private String traceId = null; + private String parentId = null; + private List attributes = null; + private Level level = Level.INFO; + private Throwable throwable = null; + private Instant startTime = null; + private final EventResourcesImpl resources; + + Log4j2Event(Clock clock, EventResourcesImpl parentResources) { + this.clock = clock; + this.resources = new EventResourcesImpl(parentResources); + } + + @Override + public Event newChildEvent() { + return new Log4j2Event(clock, resources).traceId(traceId); + } + + @Override + public Event traceId(String traceId) { + this.traceId = traceId; + return this; + } + + @Override + public Event parentId(String parentId) { + this.parentId = parentId; + return this; + } + + @Override + public Event timed() { + startTime = clock.instant(); + return this; + } + + @Override + public Event sampled(Object samplingKey, int duration, TimeUnit unit) { + throw new UnsupportedOperationException("TODO"); + } + + @Override + public Event resources(EventResources other) { + if (other instanceof EventResourcesImpl) { + this.resources.copyFrom((EventResourcesImpl) other); + } + return this; + } + + @Override + public Event resource(String key, Object value) { + resources.resource(key, value); + return this; + } + + @Override + public Event resource(String key, Supplier value) { + resources.resource(key, value); + return this; + } + + @Override + public Event attr(String key, Object value) { + getAttributes().add(key); + getAttributes().add(value); + return this; + } + + @Override + public Event attr(String key, Supplier value) { + this.attr(key, (Object) value); + return this; + } + + @Override + public Event exception(Throwable t) { + this.throwable = t; + return this; + } + + @Override + public Event atError() { + this.level = Level.ERROR; + return this; + } + + @Override + public Event atInfo() { + this.level = Level.INFO; + return this; + } + + @Override + public Event atWarn() { + this.level = Level.WARN; + return this; + } + + @Override + public void log(Enum event) { + throw new UnsupportedOperationException(); + } + + @Override + public void log(String event) { + logInternal(stringLogger, event); + } + + private void logInternal(Logger logger, String msg) { + StringMapMessage event = new StringMapMessage(); + event.with("msg", msg); + if (traceId != null) { + event.with("traceId", traceId); + } + if (parentId != null) { + event.with("parentId", parentId); + } + resources.forEach(event::with); + if (attributes != null) { + EventResourcesImpl.forEach(attributes, event::with); + } + if (startTime != null) { + event.with("startTimestamp", startTime.toString()); + event.with("durationMs", String.valueOf(Duration.between(startTime, clock.instant()).toMillis())); + } + switch (level) { + case ERROR: + if (throwable != null) { + + logger.error(event, throwable); + } else { + logger.error(event); + } + break; + case WARN: + if (throwable != null) { + logger.warn(event, throwable); + } else { + logger.warn(event); + } + break; + case INFO: + default: + if (throwable != null) { + logger.info(event, throwable); + } else { + logger.info(event); + } + break; + } + } + + @Override + public void stash() { + throw new UnsupportedOperationException("TODO"); + } + + private List getAttributes() { + if (attributes == null) { + attributes = new ArrayList<>(); + } + return attributes; + } + + enum Level { + INFO, + WARN, + ERROR + } +} diff --git a/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/log4j2/Log4j2StructuredEventLog.java b/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/log4j2/Log4j2StructuredEventLog.java new file mode 100644 index 0000000000000..e854a814e14f1 --- /dev/null +++ b/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/log4j2/Log4j2StructuredEventLog.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.structuredeventlog.log4j2; + +import java.time.Clock; +import org.apache.pulsar.structuredeventlog.Event; +import org.apache.pulsar.structuredeventlog.EventResources; +import org.apache.pulsar.structuredeventlog.EventResourcesImpl; +import org.apache.pulsar.structuredeventlog.StructuredEventLog; + +public class Log4j2StructuredEventLog implements StructuredEventLog { + public static final Log4j2StructuredEventLog INSTANCE = new Log4j2StructuredEventLog(); + // Visible for testing + Clock clock = Clock.systemUTC(); + + @Override + public Event newRootEvent() { + return new Log4j2Event(clock, null); + } + + @Override + public EventResources newEventResources() { + return new EventResourcesImpl(null); + } + + @Override + public Event unstash() { + throw new UnsupportedOperationException("TODO"); + } +} diff --git a/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/log4j2/package-info.java b/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/log4j2/package-info.java new file mode 100644 index 0000000000000..c784880755abb --- /dev/null +++ b/structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/log4j2/package-info.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.structuredeventlog.log4j2; diff --git a/structured-event-log/src/test/java/org/apache/pulsar/structuredeventlog/slf4j/StructuredEventLogTest.java b/structured-event-log/src/test/java/org/apache/pulsar/structuredeventlog/slf4j/StructuredEventLogTest.java index 2da8294784432..48662b59ba923 100644 --- a/structured-event-log/src/test/java/org/apache/pulsar/structuredeventlog/slf4j/StructuredEventLogTest.java +++ b/structured-event-log/src/test/java/org/apache/pulsar/structuredeventlog/slf4j/StructuredEventLogTest.java @@ -76,7 +76,7 @@ public void setupLog4j() throws Exception { @Test public void testTraceId() throws Exception { - StructuredEventLog log = StructuredEventLog.newLogger(); + StructuredEventLog log = Slf4jStructuredEventLog.INSTANCE; Event e = log.newRootEvent(); e.newChildEvent().log("child"); @@ -97,7 +97,7 @@ public void testTraceId() throws Exception { @Test public void testParentId() throws Exception { - StructuredEventLog log = StructuredEventLog.newLogger(); + StructuredEventLog log = Slf4jStructuredEventLog.INSTANCE; Event e1 = log.newRootEvent(); Event e2 = e1.newChildEvent(); @@ -124,7 +124,7 @@ public void testParentId() throws Exception { @Test public void testResources() throws Exception { - StructuredEventLog log = StructuredEventLog.newLogger(); + StructuredEventLog log = Slf4jStructuredEventLog.INSTANCE; EventResources res = log.newEventResources() .resource("r1", "v1") @@ -167,7 +167,7 @@ public void testResources() throws Exception { @Test public void testResourcesNullTest() throws Exception { - StructuredEventLog log = StructuredEventLog.newLogger(); + StructuredEventLog log = Slf4jStructuredEventLog.INSTANCE; EventResources res = log.newEventResources() .resource(null, "v1") @@ -205,7 +205,7 @@ public void testResourcesNullTest() throws Exception { @Test public void testAttributes() throws Exception { - StructuredEventLog log = StructuredEventLog.newLogger(); + StructuredEventLog log = Slf4jStructuredEventLog.INSTANCE; Event e1 = log.newRootEvent() .attr("a1", "v1") @@ -238,7 +238,7 @@ public void testAttributes() throws Exception { @Test public void testAttributedNullTest() throws Exception { - StructuredEventLog log = StructuredEventLog.newLogger(); + StructuredEventLog log = Slf4jStructuredEventLog.INSTANCE; log.newRootEvent() .attr(null, "v1") @@ -262,7 +262,7 @@ public void testAttributedNullTest() throws Exception { @Test public void testInfoLevel() throws Exception { - StructuredEventLog log = StructuredEventLog.newLogger(); + StructuredEventLog log = Slf4jStructuredEventLog.INSTANCE; log.newRootEvent().log("info1"); log.newRootEvent().atInfo().log("info2"); @@ -281,7 +281,7 @@ public void testInfoLevel() throws Exception { @SuppressWarnings("unchecked") @Test public void testInfoLevelException() throws Exception { - StructuredEventLog log = StructuredEventLog.newLogger(); + StructuredEventLog log = Slf4jStructuredEventLog.INSTANCE; log.newRootEvent().exception(new Throwable("cause1")).log("info1"); log.newRootEvent().atInfo().exception(new Throwable("cause2")).log("info2"); @@ -296,7 +296,7 @@ public void testInfoLevelException() throws Exception { @Test public void testWarnLevel() throws Exception { - StructuredEventLog log = StructuredEventLog.newLogger(); + StructuredEventLog log = Slf4jStructuredEventLog.INSTANCE; log.newRootEvent().atWarn().log("warn1"); @@ -308,7 +308,7 @@ public void testWarnLevel() throws Exception { @SuppressWarnings("unchecked") @Test public void testWarnLevelException() throws Exception { - StructuredEventLog log = StructuredEventLog.newLogger(); + StructuredEventLog log = Slf4jStructuredEventLog.INSTANCE; log.newRootEvent().atWarn().exception(new Throwable("cause1")).log("warn1"); @@ -319,7 +319,7 @@ public void testWarnLevelException() throws Exception { @Test public void testErrorLevel() throws Exception { - StructuredEventLog log = StructuredEventLog.newLogger(); + StructuredEventLog log = Slf4jStructuredEventLog.INSTANCE; log.newRootEvent().atError().log("error1"); @@ -331,7 +331,7 @@ public void testErrorLevel() throws Exception { @SuppressWarnings("unchecked") @Test public void testErrorLevelException() throws Exception { - StructuredEventLog log = StructuredEventLog.newLogger(); + StructuredEventLog log = Slf4jStructuredEventLog.INSTANCE; log.newRootEvent().atError().exception(new Throwable("cause1")).log("error1"); @@ -344,8 +344,8 @@ public void testErrorLevelException() throws Exception { @Test public void testTimedEvent() throws Exception { MockClock clock = new MockClock(); - StructuredEventLog log = StructuredEventLog.newLogger(); - ((Slf4jStructuredEventLog)log).clock = clock; + Slf4jStructuredEventLog log = Slf4jStructuredEventLog.INSTANCE; + log.clock = clock; Event e = log.newRootEvent().timed(); clock.advanceTime(1234, TimeUnit.MILLISECONDS); e.log("timed"); @@ -363,7 +363,7 @@ public enum Events { @Test public void testEventGroups() throws Exception { - StructuredEventLog log = StructuredEventLog.newLogger(); + StructuredEventLog log = Slf4jStructuredEventLog.INSTANCE; log.newRootEvent().log(Events.TEST_EVENT); List> logged = getLogged(); @@ -379,7 +379,7 @@ public enum BareEvents { @Test public void testBareEnum() throws Exception { - StructuredEventLog log = StructuredEventLog.newLogger(); + StructuredEventLog log = Slf4jStructuredEventLog.INSTANCE; log.newRootEvent().log(BareEvents.BARE_EVENT); List> logged = getLogged(); From 9feb85b19ca160b4be3acbfe15f39edde07608f5 Mon Sep 17 00:00:00 2001 From: houxiaoyu Date: Fri, 10 Mar 2023 16:21:25 +0800 Subject: [PATCH 097/174] [fix][client] Fix topic list watcher fail log (#19733) --- .../java/org/apache/pulsar/client/impl/TopicListWatcher.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicListWatcher.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicListWatcher.java index 88ada6a344b8d..384d1b688b8d5 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicListWatcher.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicListWatcher.java @@ -148,8 +148,8 @@ public void connectionOpened(ClientCnx cnx) { cnx.channel().close(); return null; } - log.warn("[{}][{}] Failed to subscribe to topic on {}", topic, - getHandlerName(), cnx.channel().remoteAddress()); + log.warn("[{}][{}] Failed to create topic list watcher on {}", + topic, getHandlerName(), cnx.channel().remoteAddress()); if (e.getCause() instanceof PulsarClientException && PulsarClientException.isRetriableError(e.getCause()) From 90b0f0a17579d22d413853ed4941d81debbe0cbe Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Fri, 10 Mar 2023 01:58:40 -0800 Subject: [PATCH 098/174] [fix][io] KCA: Option to use kafka connector's SourceConnector class to create task and task config (#19772) --- .../connect/AbstractKafkaConnectSource.java | 48 ++++++++++++++--- .../kafka/connect/KafkaConnectSourceTest.java | 54 ++++++++++++------- 2 files changed, 77 insertions(+), 25 deletions(-) diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/AbstractKafkaConnectSource.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/AbstractKafkaConnectSource.java index 6b4ae9d080257..36d2b4bcdbf0d 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/AbstractKafkaConnectSource.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/AbstractKafkaConnectSource.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.io.kafka.connect; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; import io.confluent.connect.avro.AvroConverter; import io.confluent.kafka.schemaregistry.client.MockSchemaRegistryClient; import io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig; @@ -33,7 +35,9 @@ import java.util.concurrent.atomic.AtomicInteger; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.connect.connector.Task; import org.apache.kafka.connect.runtime.TaskConfig; +import org.apache.kafka.connect.source.SourceConnector; import org.apache.kafka.connect.source.SourceRecord; import org.apache.kafka.connect.source.SourceTask; import org.apache.kafka.connect.source.SourceTaskContext; @@ -55,6 +59,7 @@ public abstract class AbstractKafkaConnectSource implements Source { // kafka connect related variables private SourceTaskContext sourceTaskContext; + private SourceConnector connector; @Getter private SourceTask sourceTask; public Converter keyConverter; @@ -71,6 +76,8 @@ public abstract class AbstractKafkaConnectSource implements Source { // number of outstandingRecords that have been polled but not been acked private final AtomicInteger outstandingRecords = new AtomicInteger(0); + public static final String CONNECTOR_CLASS = "kafkaConnectorSourceClass"; + @Override public void open(Map config, SourceContext sourceContext) throws Exception { Map stringConfig = new HashMap<>(); @@ -80,12 +87,6 @@ public void open(Map config, SourceContext sourceContext) throws } }); - // get the source class name from config and create source task from reflection - sourceTask = Class.forName(stringConfig.get(TaskConfig.TASK_CLASS_CONFIG)) - .asSubclass(SourceTask.class) - .getDeclaredConstructor() - .newInstance(); - topicNamespace = stringConfig.get(PulsarKafkaWorkerConfig.TOPIC_NAMESPACE_CONFIG); // initialize the key and value converter @@ -129,8 +130,36 @@ public void open(Map config, SourceContext sourceContext) throws sourceTaskContext = new PulsarIOSourceTaskContext(offsetReader, pulsarKafkaWorkerConfig); + final Map taskConfig; + if (config.get(CONNECTOR_CLASS) != null) { + String kafkaConnectorFQClassName = config.get(CONNECTOR_CLASS).toString(); + Class clazz = Class.forName(kafkaConnectorFQClassName); + connector = (SourceConnector) clazz.getConstructor().newInstance(); + + Class taskClass = connector.taskClass(); + sourceTask = (SourceTask) taskClass.getConstructor().newInstance(); + + connector.initialize(new PulsarKafkaSinkContext()); + connector.start(stringConfig); + + List> configs = connector.taskConfigs(1); + checkNotNull(configs); + checkArgument(configs.size() == 1); + taskConfig = configs.get(0); + } else { + // for backward compatibility with old configuration + // that use the task directly + + // get the source class name from config and create source task from reflection + sourceTask = Class.forName(stringConfig.get(TaskConfig.TASK_CLASS_CONFIG)) + .asSubclass(SourceTask.class) + .getDeclaredConstructor() + .newInstance(); + taskConfig = stringConfig; + } + sourceTask.initialize(sourceTaskContext); - sourceTask.start(stringConfig); + sourceTask.start(taskConfig); } @Override @@ -178,6 +207,11 @@ public void close() { sourceTask = null; } + if (connector != null) { + connector.stop(); + connector = null; + } + if (offsetStore != null) { offsetStore.stop(); offsetStore = null; diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSourceTest.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSourceTest.java index 7dcf3ce8393df..8852ba02b040f 100644 --- a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSourceTest.java +++ b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSourceTest.java @@ -23,7 +23,6 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; - import java.io.File; import java.io.OutputStream; import java.nio.file.Files; @@ -47,7 +46,6 @@ @Slf4j public class KafkaConnectSourceTest extends ProducerConsumerBase { - private Map config = new HashMap<>(); private String offsetTopicName; // The topic to publish data to, for kafkaSource private String topicName; @@ -62,18 +60,10 @@ protected void setup() throws Exception { super.internalSetup(); super.producerBaseSetup(); - config.put(TaskConfig.TASK_CLASS_CONFIG, "org.apache.kafka.connect.file.FileStreamSourceTask"); - config.put(PulsarKafkaWorkerConfig.KEY_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.storage.StringConverter"); - config.put(PulsarKafkaWorkerConfig.VALUE_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.storage.StringConverter"); - this.offsetTopicName = "persistent://my-property/my-ns/kafka-connect-source-offset"; - config.put(PulsarKafkaWorkerConfig.OFFSET_STORAGE_TOPIC_CONFIG, offsetTopicName); - this.topicName = "persistent://my-property/my-ns/kafka-connect-source"; - config.put(FileStreamSourceConnector.TOPIC_CONFIG, topicName); tempFile = File.createTempFile("some-file-name", null); - config.put(FileStreamSourceConnector.FILE_CONFIG, tempFile.getAbsoluteFile().toString()); - config.put(FileStreamSourceConnector.TASK_BATCH_SIZE_CONFIG, String.valueOf(FileStreamSourceConnector.DEFAULT_TASK_BATCH_SIZE)); + tempFile.deleteOnExit(); this.context = mock(SourceContext.class); this.client = PulsarClient.builder() @@ -91,16 +81,44 @@ protected void cleanup() throws Exception { tempFile.delete(); super.internalCleanup(); } - protected void completedFlush(Throwable error, Void result) { - if (error != null) { - log.error("Failed to flush {} offsets to storage: ", this, error); - } else { - log.info("Finished flushing {} offsets to storage", this); - } + + @Test + public void testOpenAndReadConnectorConfig() throws Exception { + Map config = getConfig(); + config.put(AbstractKafkaConnectSource.CONNECTOR_CLASS, + "org.apache.kafka.connect.file.FileStreamSourceConnector"); + + testOpenAndReadTask(config); } @Test - public void testOpenAndRead() throws Exception { + public void testOpenAndReadTaskDirect() throws Exception { + Map config = getConfig(); + + config.put(TaskConfig.TASK_CLASS_CONFIG, + "org.apache.kafka.connect.file.FileStreamSourceTask"); + + testOpenAndReadTask(config); + } + + private Map getConfig() { + Map config = new HashMap<>(); + + config.put(PulsarKafkaWorkerConfig.KEY_CONVERTER_CLASS_CONFIG, + "org.apache.kafka.connect.storage.StringConverter"); + config.put(PulsarKafkaWorkerConfig.VALUE_CONVERTER_CLASS_CONFIG, + "org.apache.kafka.connect.storage.StringConverter"); + + config.put(PulsarKafkaWorkerConfig.OFFSET_STORAGE_TOPIC_CONFIG, offsetTopicName); + + config.put(FileStreamSourceConnector.TOPIC_CONFIG, topicName); + config.put(FileStreamSourceConnector.FILE_CONFIG, tempFile.getAbsoluteFile().toString()); + config.put(FileStreamSourceConnector.TASK_BATCH_SIZE_CONFIG, + String.valueOf(FileStreamSourceConnector.DEFAULT_TASK_BATCH_SIZE)); + return config; + } + + private void testOpenAndReadTask(Map config) throws Exception { kafkaConnectSource = new KafkaConnectSource(); kafkaConnectSource.open(config, context); From bfc620bf6f2586b099fe423d7b07cc88754ef312 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Sat, 11 Mar 2023 13:02:53 +0800 Subject: [PATCH 099/174] [fix] [test] fix flaky test BucketDelayedDeliveryTrackerTest. testWithBkException (#19751) This test sets the maximum number of buckets to 10, then creates multiple buckets by writing, then uses the merge mechanism to make the final number of buckets less than or equal to 10. But `BucketDelayedDeliveryTracker` doesn't guarantee that every merger operation will work, these cases will make the operation fail: - the last persistention of the bucket has not finished. - all buckets are full. So remove this validation --- .../delayed/bucket/BucketDelayedDeliveryTrackerTest.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java index 08e1f78725bf4..0ba9e5f4ca2e7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java @@ -308,10 +308,6 @@ public void testWithBkException(BucketDelayedDeliveryTracker tracker) { assertEquals(110, tracker.getNumberOfDelayedMessages()); - int size = tracker.getImmutableBuckets().asMapOfRanges().size(); - - assertEquals(10, size); - tracker.addMessage(111, 1011, 111 * 10); MutableLong delayedMessagesInSnapshot = new MutableLong(); From b6a73823ce2ed47baa606b2ecc5f1569e4c101a5 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Sat, 11 Mar 2023 22:59:29 +0800 Subject: [PATCH 100/174] [improve][broker] PIP-192: Support broker isolation policy (#19592) --- .../extensions/ExtensibleLoadManagerImpl.java | 10 +- .../extensions/filter/BrokerFilter.java | 12 +- .../filter/BrokerIsolationPoliciesFilter.java | 60 +++++ .../filter/BrokerMaxTopicCountFilter.java | 8 + .../filter/BrokerVersionFilter.java | 11 +- .../policies/IsolationPoliciesHelper.java | 68 ++++++ .../extensions/policies/package-info.java | 19 ++ .../extensions/scheduler/TransferShedder.java | 79 +++++- .../ExtensibleLoadManagerImplTest.java | 61 ++++- .../BrokerIsolationPoliciesFilterTest.java | 222 +++++++++++++++++ .../filter/BrokerMaxTopicCountFilterTest.java | 2 +- .../filter/BrokerVersionFilterTest.java | 13 +- .../scheduler/TransferShedderTest.java | 227 ++++++++++++++++-- 13 files changed, 750 insertions(+), 42 deletions(-) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/package-info.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index 2bebe203d8750..82790f44fcb28 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -42,6 +42,7 @@ import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData; import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerFilter; +import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerIsolationPoliciesFilter; import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerMaxTopicCountFilter; import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerVersionFilter; import org.apache.pulsar.broker.loadbalance.extensions.manager.UnloadManager; @@ -139,6 +140,7 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager { public ExtensibleLoadManagerImpl() { this.brokerFilterPipeline = new ArrayList<>(); this.brokerFilterPipeline.add(new BrokerMaxTopicCountFilter()); + this.brokerFilterPipeline.add(new BrokerIsolationPoliciesFilter()); this.brokerFilterPipeline.add(new BrokerVersionFilter()); // TODO: Make brokerSelectionStrategy configurable. this.brokerSelectionStrategy = new LeastResourceUsageWithWeight(); @@ -225,6 +227,7 @@ public void start() throws PulsarServerException { public void initialize(PulsarService pulsar) { this.pulsar = pulsar; this.conf = pulsar.getConfiguration(); + this.brokerFilterPipeline.forEach(brokerFilter -> brokerFilter.initialize(pulsar)); } @Override @@ -287,7 +290,6 @@ private CompletableFuture> selectAsync(ServiceUnitId bundle) { BrokerRegistry brokerRegistry = getBrokerRegistry(); return brokerRegistry.getAvailableBrokerLookupDataAsync() .thenCompose(availableBrokers -> { - // TODO: Support isolation policies LoadManagerContext context = this.getContext(); Map availableBrokerCandidates = new HashMap<>(availableBrokers); @@ -296,11 +298,13 @@ private CompletableFuture> selectAsync(ServiceUnitId bundle) { List filterPipeline = getBrokerFilterPipeline(); for (final BrokerFilter filter : filterPipeline) { try { - filter.filter(availableBrokerCandidates, context); + filter.filter(availableBrokerCandidates, bundle, context); + // Preserve the filter successes result. + availableBrokers.keySet().retainAll(availableBrokerCandidates.keySet()); } catch (BrokerFilterException e) { // TODO: We may need to revisit this error case. log.error("Failed to filter out brokers.", e); - availableBrokerCandidates = availableBrokers; + availableBrokerCandidates = new HashMap<>(availableBrokers); } } if (availableBrokerCandidates.isEmpty()) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java index 35f4b6817f131..30d25f559b11e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java @@ -19,9 +19,11 @@ package org.apache.pulsar.broker.loadbalance.extensions.filter; import java.util.Map; +import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; +import org.apache.pulsar.common.naming.ServiceUnitId; /** * Filter out unqualified Brokers, which are not entered into LoadBalancer for decision-making. @@ -33,14 +35,22 @@ public interface BrokerFilter { */ String name(); + /** + * Initialize this broker filter using the given pulsar service. + */ + void initialize(PulsarService pulsar); + /** * Filter out unqualified brokers based on implementation. * * @param brokers The full broker and lookup data. + * @param serviceUnit The current serviceUnit. * @param context The load manager context. * @return Filtered broker list. */ - Map filter(Map brokers, LoadManagerContext context) + Map filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) throws BrokerFilterException; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java new file mode 100644 index 0000000000000..b28c77f76f3eb --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.filter; + +import java.util.Map; +import java.util.Set; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.loadbalance.BrokerFilterException; +import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; +import org.apache.pulsar.broker.loadbalance.extensions.policies.IsolationPoliciesHelper; +import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies; +import org.apache.pulsar.common.naming.ServiceUnitId; + + +@Slf4j +public class BrokerIsolationPoliciesFilter implements BrokerFilter { + + public static final String FILTER_NAME = "broker_isolation_policies_filter"; + + private IsolationPoliciesHelper isolationPoliciesHelper; + + @Override + public String name() { + return FILTER_NAME; + } + + @Override + public void initialize(PulsarService pulsar) { + this.isolationPoliciesHelper = new IsolationPoliciesHelper(new SimpleResourceAllocationPolicies(pulsar)); + } + + @Override + public Map filter(Map availableBrokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) + throws BrokerFilterException { + Set brokerCandidateCache = + isolationPoliciesHelper.applyIsolationPolicies(availableBrokers, serviceUnit); + availableBrokers.keySet().retainAll(brokerCandidateCache); + return availableBrokers; + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java index e3f8faca32468..b98edd3d425e5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java @@ -20,10 +20,12 @@ import java.util.Map; import java.util.Optional; +import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; +import org.apache.pulsar.common.naming.ServiceUnitId; public class BrokerMaxTopicCountFilter implements BrokerFilter { @@ -34,8 +36,14 @@ public String name() { return FILTER_NAME; } + @Override + public void initialize(PulsarService pulsar) { + // No-op + } + @Override public Map filter(Map brokers, + ServiceUnitId serviceUnit, LoadManagerContext context) throws BrokerFilterException { int loadBalancerBrokerMaxTopics = context.brokerConfiguration().getLoadBalancerBrokerMaxTopics(); brokers.keySet().removeIf(broker -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java index 869fb049a3cd8..b7332a5ff10a0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java @@ -22,11 +22,13 @@ import java.util.Iterator; import java.util.Map; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.BrokerFilterBadVersionException; import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; +import org.apache.pulsar.common.naming.ServiceUnitId; /** * Filter by broker version. @@ -45,7 +47,9 @@ public class BrokerVersionFilter implements BrokerFilter { * */ @Override - public Map filter(Map brokers, LoadManagerContext context) + public Map filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) throws BrokerFilterException { ServiceConfiguration conf = context.brokerConfiguration(); if (!conf.isPreferLaterVersions() || brokers.isEmpty()) { @@ -144,4 +148,9 @@ public Version getLatestVersionNumber(Map brokerMap) public String name() { return FILTER_NAME; } + + @Override + public void initialize(PulsarService pulsar) { + // No-op + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java new file mode 100644 index 0000000000000..4d7a5bf22d661 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.policies; + +import io.netty.util.concurrent.FastThreadLocal; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; +import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; +import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies; +import org.apache.pulsar.common.naming.ServiceUnitId; + +@Slf4j +public class IsolationPoliciesHelper { + + private final SimpleResourceAllocationPolicies policies; + + public IsolationPoliciesHelper(SimpleResourceAllocationPolicies policies) { + this.policies = policies; + } + + private static final FastThreadLocal> localBrokerCandidateCache = new FastThreadLocal<>() { + @Override + protected Set initialValue() { + return new HashSet<>(); + } + }; + + public Set applyIsolationPolicies(Map availableBrokers, + ServiceUnitId serviceUnit) { + Set brokerCandidateCache = localBrokerCandidateCache.get(); + brokerCandidateCache.clear(); + LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, + availableBrokers.keySet(), new LoadManagerShared.BrokerTopicLoadingPredicate() { + @Override + public boolean isEnablePersistentTopics(String brokerUrl) { + BrokerLookupData lookupData = availableBrokers.get(brokerUrl.replace("http://", "")); + return lookupData != null && lookupData.persistentTopicsEnabled(); + } + + @Override + public boolean isEnableNonPersistentTopics(String brokerUrl) { + BrokerLookupData lookupData = availableBrokers.get(brokerUrl.replace("http://", "")); + return lookupData != null && lookupData.nonPersistentTopicsEnabled(); + } + }); + return brokerCandidateCache; + } + +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/package-info.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/package-info.java new file mode 100644 index 0000000000000..76e0c4a894290 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/package-info.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.policies; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java index 7f9128de81754..810fda320af68 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java @@ -21,10 +21,15 @@ import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.CoolDown; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBrokers; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.OutDatedData; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Unknown; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.MinMaxPriorityQueue; import java.util.Map; import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import lombok.Getter; import lombok.experimental.Accessors; import org.apache.commons.lang3.StringUtils; @@ -32,12 +37,15 @@ import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData; import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision; +import org.apache.pulsar.broker.loadbalance.extensions.policies.IsolationPoliciesHelper; import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore; import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies; +import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.slf4j.Logger; @@ -71,6 +79,7 @@ public class TransferShedder implements NamespaceUnloadStrategy { private final LoadStats stats = new LoadStats(); private final PulsarService pulsar; private final SimpleResourceAllocationPolicies allocationPolicies; + private final IsolationPoliciesHelper isolationPoliciesHelper; private final UnloadDecision decision = new UnloadDecision(); @@ -78,11 +87,13 @@ public class TransferShedder implements NamespaceUnloadStrategy { public TransferShedder(){ this.pulsar = null; this.allocationPolicies = null; + this.isolationPoliciesHelper = null; } public TransferShedder(PulsarService pulsar){ this.pulsar = pulsar; this.allocationPolicies = new SimpleResourceAllocationPolicies(pulsar); + this.isolationPoliciesHelper = new IsolationPoliciesHelper(allocationPolicies); } @@ -265,6 +276,17 @@ public UnloadDecision findBundlesForUnloading(LoadManagerContext context, final double targetStd = conf.getLoadBalancerBrokerLoadTargetStd(); boolean transfer = conf.isLoadBalancerTransferEnabled(); + + Map availableBrokers; + try { + availableBrokers = context.brokerRegistry().getAvailableBrokerLookupDataAsync() + .get(context.brokerConfiguration().getMetadataStoreOperationTimeoutSeconds(), TimeUnit.SECONDS); + } catch (ExecutionException | InterruptedException | TimeoutException e) { + decision.skip(Unknown); + log.warn("Failed to fetch available brokers. Reason:{}. Stop unloading.", decision.getReason(), e); + return decision; + } + while (true) { if (!stats.hasTransferableBrokers()) { if (debugMode) { @@ -334,7 +356,9 @@ public UnloadDecision findBundlesForUnloading(LoadManagerContext context, int remainingTopBundles = topBundlesLoadData.size(); for (var e : topBundlesLoadData) { String bundle = e.bundleName(); - if (!recentlyUnloadedBundles.containsKey(bundle) && isTransferable(bundle)) { + if (!recentlyUnloadedBundles.containsKey(bundle) + && isTransferable(context, availableBrokers, + bundle, maxBroker, Optional.of(minBroker))) { var bundleData = e.stats(); double throughput = bundleData.msgThroughputIn + bundleData.msgThroughputOut; if (remainingTopBundles > 1 @@ -342,8 +366,7 @@ public UnloadDecision findBundlesForUnloading(LoadManagerContext context, || !atLeastOneBundleSelected)) { if (transfer) { selectedBundlesCache.put(maxBroker, - new Unload(maxBroker, bundle, - Optional.of(minBroker))); + new Unload(maxBroker, bundle, Optional.of(minBroker))); } else { selectedBundlesCache.put(maxBroker, new Unload(maxBroker, bundle)); @@ -412,18 +435,27 @@ private boolean hasMsgThroughput(LoadManagerContext context, String broker) { } - private boolean isTransferable(String bundle) { + private boolean isTransferable(LoadManagerContext context, + Map availableBrokers, + String bundle, + String maxBroker, + Optional broker) { if (pulsar == null || allocationPolicies == null) { return true; } - NamespaceName namespace = NamespaceName.get(LoadManagerShared.getNamespaceNameFromBundleName(bundle)); - if (allocationPolicies.areIsolationPoliciesPresent(namespace)) { + String namespace = LoadManagerShared.getNamespaceNameFromBundleName(bundle); + NamespaceName namespaceName = NamespaceName.get(namespace); + final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle); + NamespaceBundle namespaceBundle = + pulsar.getNamespaceService().getNamespaceBundleFactory().getBundle(namespace, bundleRange); + + if (!canTransferWithIsolationPoliciesToBroker(context, availableBrokers, namespaceBundle, maxBroker, broker)) { return false; } try { var localPoliciesOptional = pulsar - .getPulsarResources().getLocalPolicies().getLocalPolicies(namespace); + .getPulsarResources().getLocalPolicies().getLocalPolicies(namespaceName); if (localPoliciesOptional.isPresent() && StringUtils.isNotBlank( localPoliciesOptional.get().namespaceAntiAffinityGroup)) { return false; @@ -434,4 +466,37 @@ private boolean isTransferable(String bundle) { } return true; } + + /** + * Check the gave bundle and broker can be transfer or unload with isolation policies applied. + * + * @param context The load manager context. + * @param availableBrokers The available brokers. + * @param namespaceBundle The bundle try to unload or transfer. + * @param currentBroker The current broker. + * @param targetBroker The broker will be transfer to. + * @return Can be transfer/unload or not. + */ + private boolean canTransferWithIsolationPoliciesToBroker(LoadManagerContext context, + Map availableBrokers, + NamespaceBundle namespaceBundle, + String currentBroker, + Optional targetBroker) { + if (isolationPoliciesHelper == null + || !allocationPolicies.areIsolationPoliciesPresent(namespaceBundle.getNamespaceObject())) { + return true; + } + boolean transfer = context.brokerConfiguration().isLoadBalancerTransferEnabled(); + Set candidates = isolationPoliciesHelper.applyIsolationPolicies(availableBrokers, namespaceBundle); + + // Remove the current bundle owner broker. + candidates.remove(currentBroker); + + // Unload: Check if there are any more candidates available for selection. + if (targetBroker.isEmpty() || !transfer) { + return !candidates.isEmpty(); + } + // Transfer: Check if this broker is among the candidates. + return candidates.contains(targetBroker.get()); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index ec82f5c383e2e..441415a9d35e5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -90,6 +90,7 @@ import org.apache.pulsar.client.impl.TableViewImpl; import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.ServiceUnitId; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.Policies; @@ -290,12 +291,19 @@ public String name() { return "Mock broker filter"; } + @Override + public void initialize(PulsarService pulsar) { + // No-op + } + @Override public Map filter(Map brokers, - LoadManagerContext context) { + ServiceUnitId serviceUnit, + LoadManagerContext context) throws BrokerFilterException { brokers.remove(pulsar1.getLookupServiceAddress()); return brokers; } + })).when(primaryLoadManager).getBrokerFilterPipeline(); Optional brokerLookupData = primaryLoadManager.assign(Optional.empty(), bundle).get(); @@ -308,14 +316,10 @@ public void testFilterHasException() throws Exception { TopicName topicName = TopicName.get("test-filter-has-exception"); NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get(); - doReturn(List.of(new BrokerFilter() { - @Override - public String name() { - return "Mock broker filter"; - } - + doReturn(List.of(new MockBrokerFilter() { @Override public Map filter(Map brokers, + ServiceUnitId serviceUnit, LoadManagerContext context) throws BrokerFilterException { brokers.clear(); throw new BrokerFilterException("Test"); @@ -379,6 +383,35 @@ private void checkOwnershipState(String broker, NamespaceBundle bundle) } + @Test + public void testMoreThenOneFilter() throws Exception { + TopicName topicName = TopicName.get("test-filter-has-exception"); + NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get(); + + String lookupServiceAddress1 = pulsar1.getLookupServiceAddress(); + doReturn(List.of(new MockBrokerFilter() { + @Override + public Map filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) throws BrokerFilterException { + brokers.remove(lookupServiceAddress1); + return brokers; + } + },new MockBrokerFilter() { + @Override + public Map filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) throws BrokerFilterException { + brokers.clear(); + throw new BrokerFilterException("Test"); + } + })).when(primaryLoadManager).getBrokerFilterPipeline(); + + Optional brokerLookupData = primaryLoadManager.assign(Optional.empty(), bundle).get(); + assertTrue(brokerLookupData.isPresent()); + assertEquals(brokerLookupData.get().getWebServiceUrl(), pulsar2.getWebServiceAddress()); + } + @Test public void testGetMetrics() throws Exception { { @@ -565,6 +598,20 @@ SplitDecision.Reason.Balanced, new MutableLong(6) assertEquals(actual, expected); } + private static abstract class MockBrokerFilter implements BrokerFilter { + + @Override + public String name() { + return "Mock-broker-filter"; + } + + @Override + public void initialize(PulsarService pulsar) { + // No-op + } + + } + private static void cleanTableView(ServiceUnitStateChannel channel) throws IllegalAccessException { var tv = (TableViewImpl) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java new file mode 100644 index 0000000000000..a079a23bcea04 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.filter; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import org.apache.commons.lang.reflect.FieldUtils; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.BrokerFilterException; +import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; +import org.apache.pulsar.broker.loadbalance.extensions.policies.IsolationPoliciesHelper; +import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies; +import org.apache.pulsar.common.naming.NamespaceBundle; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener; +import org.testng.annotations.Test; + +/** + * Unit test for {@link BrokerIsolationPoliciesFilter}. + */ +@Test(groups = "broker") +public class BrokerIsolationPoliciesFilterTest { + + /** + * It verifies namespace-isolation policies with primary and secondary brokers. + * + * usecase: + * + * 
+     *  1. Namespace: primary=broker1, secondary=broker2, shared=broker3, min_limit = 1
+     *     a. available-brokers: broker1, broker2, broker3 => result: broker1
+     *     b. available-brokers: broker2, broker3          => result: broker2
+     *     c. available-brokers: broker3                   => result: NULL
+     *  2. Namespace: primary=broker1, secondary=broker2, shared=broker3, min_limit = 2
+     *     a. available-brokers: broker1, broker2, broker3 => result: broker1, broker2
+     *     b. available-brokers: broker2, broker3          => result: broker2
+     *     c. available-brokers: broker3                   => result: NULL
+     *
+ */ + @Test + public void testFilterWithNamespaceIsolationPoliciesForPrimaryAndSecondaryBrokers() + throws IllegalAccessException, BrokerFilterException { + var namespace = "my-tenant/my-ns"; + NamespaceName namespaceName = NamespaceName.get(namespace); + + BrokerIsolationPoliciesFilter filter = new BrokerIsolationPoliciesFilter(); + var policies = mock(SimpleResourceAllocationPolicies.class); + + // 1. Namespace: primary=broker1, secondary=broker2, shared=broker3, min_limit = 1 + setIsolationPolicies(policies, namespaceName, Set.of("broker1"), Set.of("broker2"), Set.of("broker3"), 1); + IsolationPoliciesHelper isolationPoliciesHelper = new IsolationPoliciesHelper(policies); + FieldUtils.writeDeclaredField(filter, "isolationPoliciesHelper", isolationPoliciesHelper, true); + + // a. available-brokers: broker1, broker2, broker3 => result: broker1 + Map result = filter.filter(new HashMap<>(Map.of( + "broker1", getLookupData(), + "broker2", getLookupData(), + "broker3", getLookupData())), namespaceName, getContext()); + assertEquals(result.keySet(), Set.of("broker1")); + + // b. available-brokers: broker2, broker3 => result: broker2 + result = filter.filter(new HashMap<>(Map.of( + "broker2", getLookupData(), + "broker3", getLookupData())), namespaceName, getContext()); + assertEquals(result.keySet(), Set.of("broker2")); + + // c. available-brokers: broker3 => result: NULL + result = filter.filter(new HashMap<>(Map.of( + "broker3", getLookupData())), namespaceName, getContext()); + assertTrue(result.isEmpty()); + + // 2. Namespace: primary=broker1, secondary=broker2, shared=broker3, min_limit = 2 + setIsolationPolicies(policies, namespaceName, Set.of("broker1"), Set.of("broker2"), Set.of("broker3"), 2); + + // a. available-brokers: broker1, broker2, broker3 => result: broker1, broker2 + result = filter.filter(new HashMap<>(Map.of( + "broker1", getLookupData(), + "broker2", getLookupData(), + "broker3", getLookupData())), namespaceName, getContext()); + assertEquals(result.keySet(), Set.of("broker1", "broker2")); + + // b. available-brokers: broker2, broker3 => result: broker2 + result = filter.filter(new HashMap<>(Map.of( + "broker2", getLookupData(), + "broker3", getLookupData())), namespaceName, getContext()); + assertEquals(result.keySet(), Set.of("broker2")); + + // c. available-brokers: broker3 => result: NULL + result = filter.filter(new HashMap<>(Map.of( + "broker3", getLookupData())), namespaceName, getContext()); + assertTrue(result.isEmpty()); + } + + @Test + public void testFilterWithPersistentOrNonPersistentDisabled() + throws IllegalAccessException, BrokerFilterException { + var namespace = "my-tenant/my-ns"; + NamespaceName namespaceName = NamespaceName.get(namespace); + NamespaceBundle namespaceBundle = mock(NamespaceBundle.class); + doReturn(true).when(namespaceBundle).hasNonPersistentTopic(); + doReturn(namespaceName).when(namespaceBundle).getNamespaceObject(); + + BrokerIsolationPoliciesFilter filter = new BrokerIsolationPoliciesFilter(); + + var policies = mock(SimpleResourceAllocationPolicies.class); + doReturn(false).when(policies).areIsolationPoliciesPresent(eq(namespaceName)); + doReturn(true).when(policies).isSharedBroker(any()); + IsolationPoliciesHelper isolationPoliciesHelper = new IsolationPoliciesHelper(policies); + FieldUtils.writeDeclaredField(filter, "isolationPoliciesHelper", isolationPoliciesHelper, true); + + Map result = filter.filter(new HashMap<>(Map.of( + "broker1", getLookupData(), + "broker2", getLookupData(), + "broker3", getLookupData())), namespaceBundle, getContext()); + assertEquals(result.keySet(), Set.of("broker1", "broker2", "broker3")); + + + result = filter.filter(new HashMap<>(Map.of( + "broker1", getLookupData(true, false), + "broker2", getLookupData(true, false), + "broker3", getLookupData())), namespaceBundle, getContext()); + assertEquals(result.keySet(), Set.of("broker3")); + + doReturn(false).when(namespaceBundle).hasNonPersistentTopic(); + + result = filter.filter(new HashMap<>(Map.of( + "broker1", getLookupData(), + "broker2", getLookupData(), + "broker3", getLookupData())), namespaceBundle, getContext()); + assertEquals(result.keySet(), Set.of("broker1", "broker2", "broker3")); + + result = filter.filter(new HashMap<>(Map.of( + "broker1", getLookupData(false, true), + "broker2", getLookupData(), + "broker3", getLookupData())), namespaceBundle, getContext()); + assertEquals(result.keySet(), Set.of("broker2", "broker3")); + } + + private void setIsolationPolicies(SimpleResourceAllocationPolicies policies, + NamespaceName namespaceName, + Set primary, + Set secondary, + Set shared, + int min_limit) { + reset(policies); + doReturn(true).when(policies).areIsolationPoliciesPresent(eq(namespaceName)); + doReturn(false).when(policies).isPrimaryBroker(eq(namespaceName), any()); + doReturn(false).when(policies).isSecondaryBroker(eq(namespaceName), any()); + doReturn(false).when(policies).isSharedBroker(any()); + + primary.forEach(broker -> { + doReturn(true).when(policies).isPrimaryBroker(eq(namespaceName), eq(broker)); + }); + + secondary.forEach(broker -> { + doReturn(true).when(policies).isSecondaryBroker(eq(namespaceName), eq(broker)); + }); + + shared.forEach(broker -> { + doReturn(true).when(policies).isSharedBroker(eq(broker)); + }); + + doAnswer(invocationOnMock -> { + Integer totalPrimaryCandidates = invocationOnMock.getArgument(1, Integer.class); + return totalPrimaryCandidates < min_limit; + }).when(policies).shouldFailoverToSecondaries(eq(namespaceName), anyInt()); + } + + public BrokerLookupData getLookupData() { + return getLookupData(true, true); + } + + public BrokerLookupData getLookupData(boolean persistentTopicsEnabled, + boolean nonPersistentTopicsEnabled) { + String webServiceUrl = "http://localhost:8080"; + String webServiceUrlTls = "https://localhoss:8081"; + String pulsarServiceUrl = "pulsar://localhost:6650"; + String pulsarServiceUrlTls = "pulsar+ssl://localhost:6651"; + Map advertisedListeners = new HashMap<>(); + Map protocols = new HashMap<>(){{ + put("kafka", "9092"); + }}; + return new BrokerLookupData( + webServiceUrl, webServiceUrlTls, pulsarServiceUrl, + pulsarServiceUrlTls, advertisedListeners, protocols, + persistentTopicsEnabled, nonPersistentTopicsEnabled, "3.0.0"); + } + + public LoadManagerContext getContext() { + LoadManagerContext mockContext = mock(LoadManagerContext.class); + doReturn(new ServiceConfiguration()).when(mockContext).brokerConfiguration(); + return mockContext; + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java index 4c3255341b778..da13a9526a881 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java @@ -58,7 +58,7 @@ public void test() throws IllegalAccessException, BrokerFilterException { "broker3", getLookupData(), "broker4", getLookupData() ); - Map result = filter.filter(new HashMap<>(originalBrokers), context); + Map result = filter.filter(new HashMap<>(originalBrokers), null, context); assertEquals(result, Map.of( "broker2", getLookupData(), "broker4", getLookupData() diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java index d36c79d60ed4e..cafd8f0ea7a4c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java @@ -41,7 +41,7 @@ public class BrokerVersionFilterTest extends BrokerFilterTestBase { @Test public void testFilterEmptyBrokerList() throws BrokerFilterException { BrokerVersionFilter brokerVersionFilter = new BrokerVersionFilter(); - Map result = brokerVersionFilter.filter(new HashMap<>(), getContext()); + Map result = brokerVersionFilter.filter(new HashMap<>(), null, getContext()); assertTrue(result.isEmpty()); } @@ -58,7 +58,7 @@ public void testDisabledFilter() throws BrokerFilterException { ); Map brokers = new HashMap<>(originalBrokers); BrokerVersionFilter brokerVersionFilter = new BrokerVersionFilter(); - Map result = brokerVersionFilter.filter(brokers, context); + Map result = brokerVersionFilter.filter(brokers, null, context); assertEquals(result, originalBrokers); } @@ -71,7 +71,8 @@ public void testFilter() throws BrokerFilterException { "localhost:6653", getLookupData("2.10.1") ); BrokerVersionFilter brokerVersionFilter = new BrokerVersionFilter(); - Map result = brokerVersionFilter.filter(new HashMap<>(originalBrokers), getContext()); + Map result = brokerVersionFilter.filter( + new HashMap<>(originalBrokers), null, getContext()); assertEquals(result, Map.of( "localhost:6651", getLookupData("2.10.1"), "localhost:6652", getLookupData("2.10.1"), @@ -84,7 +85,7 @@ public void testFilter() throws BrokerFilterException { "localhost:6652", getLookupData("2.10.1"), "localhost:6653", getLookupData("2.10.1") ); - result = brokerVersionFilter.filter(new HashMap<>(originalBrokers), getContext()); + result = brokerVersionFilter.filter(new HashMap<>(originalBrokers), null, getContext()); assertEquals(result, Map.of( "localhost:6652", getLookupData("2.10.1"), @@ -98,7 +99,7 @@ public void testFilter() throws BrokerFilterException { "localhost:6653", getLookupData("2.10.2-SNAPSHOT") ); - result = brokerVersionFilter.filter(new HashMap<>(originalBrokers), getContext()); + result = brokerVersionFilter.filter(new HashMap<>(originalBrokers), null, getContext()); assertEquals(result, Map.of( "localhost:6653", getLookupData("2.10.2-SNAPSHOT") )); @@ -111,6 +112,6 @@ public void testInvalidVersionString() throws BrokerFilterException { "localhost:6650", getLookupData("xxx") ); BrokerVersionFilter brokerVersionFilter = new BrokerVersionFilter(); - brokerVersionFilter.filter(new HashMap<>(originalBrokers), getContext()); + brokerVersionFilter.filter(new HashMap<>(originalBrokers), null, getContext()); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java index 709a1113f35e4..a668d85f0071c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java @@ -28,9 +28,15 @@ import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.OutDatedData; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Overloaded; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Underloaded; +import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Unknown; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; import static org.mockito.Mockito.spy; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; @@ -42,24 +48,37 @@ import java.util.Random; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeoutException; import java.util.function.BiConsumer; +import com.google.common.collect.BoundType; +import com.google.common.collect.Range; import org.apache.commons.lang.reflect.FieldUtils; import org.apache.commons.math3.stat.descriptive.moment.Mean; import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.extensions.BrokerRegistry; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData; import org.apache.pulsar.broker.loadbalance.extensions.models.TopKBundles; import org.apache.pulsar.broker.loadbalance.extensions.models.Unload; import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision; +import org.apache.pulsar.broker.loadbalance.extensions.policies.IsolationPoliciesHelper; import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore; import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies; +import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.resources.LocalPoliciesResources; import org.apache.pulsar.broker.resources.PulsarResources; +import org.apache.pulsar.common.naming.NamespaceBundle; +import org.apache.pulsar.common.naming.NamespaceBundleFactory; +import org.apache.pulsar.common.naming.NamespaceBundles; +import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.policies.data.LocalPolicies; +import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.api.MetadataStoreException; +import org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener; import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage; @@ -145,6 +164,19 @@ public TopBundlesLoadData getTopBundlesLoad(String bundlePrefix, int load1) { return topKBundles.getLoadData(); } + public TopBundlesLoadData getTopBundlesLoadWithOutSuffix(String namespace, + int load1, + int load2) { + var namespaceBundleStats1 = new NamespaceBundleStats(); + namespaceBundleStats1.msgThroughputOut = load1 * 1e6; + var namespaceBundleStats2 = new NamespaceBundleStats(); + namespaceBundleStats2.msgThroughputOut = load2 * 1e6; + var topKBundles = new TopKBundles(); + topKBundles.update(Map.of(namespace + "/0x00000000_0x7FFFFFF", namespaceBundleStats1, + namespace + "/0x7FFFFFF_0xFFFFFFF", namespaceBundleStats2), 2); + return topKBundles.getLoadData(); + } + public LoadManagerContext getContext(){ var ctx = mock(LoadManagerContext.class); var conf = new ServiceConfiguration(); @@ -236,6 +268,10 @@ public int size() { doReturn(conf).when(ctx).brokerConfiguration(); doReturn(brokerLoadDataStore).when(ctx).brokerLoadDataStore(); doReturn(topBundleLoadDataStore).when(ctx).topBundleLoadDataStore(); + var brokerRegister = mock(BrokerRegistry.class); + doReturn(brokerRegister).when(ctx).brokerRegistry(); + BrokerRegistry registry = ctx.brokerRegistry(); + doReturn(CompletableFuture.completedFuture(Map.of())).when(registry).getAvailableBrokerLookupDataAsync(); return ctx; } @@ -350,33 +386,187 @@ public void testRecentlyUnloadedBundles() { } @Test - public void testBundlesWithIsolationPolicies() throws IllegalAccessException { - var pulsar = mock(PulsarService.class); + public void testGetAvailableBrokersFailed() { + TransferShedder transferShedder = new TransferShedder(); + var ctx = setupContext(); + BrokerRegistry registry = ctx.brokerRegistry(); + doReturn(FutureUtil.failedFuture(new TimeoutException())).when(registry).getAvailableBrokerLookupDataAsync(); + var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); + + var expected = new UnloadDecision(); + expected.setLabel(Skip); + expected.skip(Unknown); + expected.setLoadAvg(setupLoadAvg); + expected.setLoadStd(setupLoadStd); + assertEquals(res, expected); + } + + @Test(timeOut = 30 * 1000) + public void testBundlesWithIsolationPolicies() throws IllegalAccessException, MetadataStoreException { + var pulsar = getMockPulsar(); + TransferShedder transferShedder = new TransferShedder(pulsar); + + var pulsarResourcesMock = mock(PulsarResources.class); + var localPoliciesResourcesMock = mock(LocalPoliciesResources.class); + doReturn(pulsarResourcesMock).when(pulsar).getPulsarResources(); + doReturn(localPoliciesResourcesMock).when(pulsarResourcesMock).getLocalPolicies(); + doReturn(Optional.empty()).when(localPoliciesResourcesMock).getLocalPolicies(any()); + var allocationPoliciesSpy = (SimpleResourceAllocationPolicies) spy(FieldUtils.readDeclaredField(transferShedder, "allocationPolicies", true)); - doReturn(true).when(allocationPoliciesSpy).areIsolationPoliciesPresent(any()); FieldUtils.writeDeclaredField(transferShedder, "allocationPolicies", allocationPoliciesSpy, true); + IsolationPoliciesHelper isolationPoliciesHelper = new IsolationPoliciesHelper(allocationPoliciesSpy); + FieldUtils.writeDeclaredField(transferShedder, "isolationPoliciesHelper", isolationPoliciesHelper, true); + + // Test transfer to a has isolation policies broker. + setIsolationPolicies(allocationPoliciesSpy, "my-tenant/my-namespaceE", + Set.of("broker5"), Set.of(), Set.of(), 1); var ctx = setupContext(); + BrokerRegistry registry = ctx.brokerRegistry(); + doReturn(CompletableFuture.completedFuture(Map.of( + "broker1", getLookupData(), + "broker2", getLookupData(), + "broker3", getLookupData(), + "broker4", getLookupData(), + "broker5", getLookupData()))).when(registry).getAvailableBrokerLookupDataAsync(); + var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); - topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA/0x00000000_0xFFFFFFF", 1, 3)); - topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB/0x00000000_0xFFFFFFF", 2, 8)); - topBundlesLoadDataStore.pushAsync("broker3", getTopBundlesLoad("my-tenant/my-namespaceC/0x00000000_0xFFFFFFF", 6, 10)); - topBundlesLoadDataStore.pushAsync("broker4", getTopBundlesLoad("my-tenant/my-namespaceD/0x00000000_0xFFFFFFF", 10, 20)); - topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE/0x00000000_0xFFFFFFF", 70, 20)); + topBundlesLoadDataStore.pushAsync("broker1", + getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceA", 1, 3)); + topBundlesLoadDataStore.pushAsync("broker2", + getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceB", 2, 8)); + topBundlesLoadDataStore.pushAsync("broker3", + getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceC", 6, 10)); + topBundlesLoadDataStore.pushAsync("broker4", + getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceD", 10, 20)); + topBundlesLoadDataStore.pushAsync("broker5", + getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceE", 70, 20)); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new UnloadDecision(); - expected.setLabel(Skip); - expected.skip(NoBundles); + var unloads = expected.getUnloads(); + unloads.put("broker4", + new Unload("broker4", "my-tenant/my-namespaceD/0x7FFFFFF_0xFFFFFFF", Optional.of("broker2"))); + expected.setLabel(Success); + expected.setReason(Overloaded); + expected.setLoadAvg(setupLoadAvg); + expected.setLoadStd(setupLoadStd); + assertEquals(res, expected); + + // Test unload a has isolation policies broker. + + setIsolationPolicies(allocationPoliciesSpy, "my-tenant/my-namespaceE", + Set.of("broker5"), Set.of(), Set.of(), 1); + ctx = setupContext(); + registry = ctx.brokerRegistry(); + doReturn(CompletableFuture.completedFuture(Map.of( + "broker1", getLookupData(), + "broker2", getLookupData(), + "broker3", getLookupData(), + "broker4", getLookupData(), + "broker5", getLookupData()))).when(registry).getAvailableBrokerLookupDataAsync(); + + ctx.brokerConfiguration().setLoadBalancerTransferEnabled(false); + + topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); + topBundlesLoadDataStore.pushAsync("broker1", + getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceA", 1, 3)); + topBundlesLoadDataStore.pushAsync("broker2", + getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceB", 2, 8)); + topBundlesLoadDataStore.pushAsync("broker3", + getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceC", 6, 10)); + topBundlesLoadDataStore.pushAsync("broker4", + getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceD", 10, 20)); + topBundlesLoadDataStore.pushAsync("broker5", + getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceE", 70, 20)); + res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); + + expected = new UnloadDecision(); + unloads = expected.getUnloads(); + unloads.put("broker4", + new Unload("broker4", "my-tenant/my-namespaceD/0x7FFFFFF_0xFFFFFFF", Optional.empty())); + expected.setLabel(Success); + expected.setReason(Overloaded); expected.setLoadAvg(setupLoadAvg); expected.setLoadStd(setupLoadStd); assertEquals(res, expected); } + public BrokerLookupData getLookupData() { + String webServiceUrl = "http://localhost:8080"; + String webServiceUrlTls = "https://localhoss:8081"; + String pulsarServiceUrl = "pulsar://localhost:6650"; + String pulsarServiceUrlTls = "pulsar+ssl://localhost:6651"; + Map advertisedListeners = new HashMap<>(); + Map protocols = new HashMap<>(){{ + put("kafka", "9092"); + }}; + return new BrokerLookupData( + webServiceUrl, webServiceUrlTls, pulsarServiceUrl, + pulsarServiceUrlTls, advertisedListeners, protocols, + true, true, "3.0.0"); + } + + private void setIsolationPolicies(SimpleResourceAllocationPolicies policies, + String namespace, + Set primary, + Set secondary, + Set shared, + int min_limit) { + reset(policies); + NamespaceName namespaceName = NamespaceName.get(namespace); + NamespaceBundle namespaceBundle = mock(NamespaceBundle.class); + doReturn(true).when(namespaceBundle).hasNonPersistentTopic(); + doReturn(namespaceName).when(namespaceBundle).getNamespaceObject(); + doReturn(false).when(policies).areIsolationPoliciesPresent(any()); + doReturn(false).when(policies).isPrimaryBroker(any(), any()); + doReturn(false).when(policies).isSecondaryBroker(any(), any()); + doReturn(true).when(policies).isSharedBroker(any()); + + doReturn(true).when(policies).areIsolationPoliciesPresent(eq(namespaceName)); + + primary.forEach(broker -> { + doReturn(true).when(policies).isPrimaryBroker(eq(namespaceName), eq(broker)); + }); + + secondary.forEach(broker -> { + doReturn(true).when(policies).isSecondaryBroker(eq(namespaceName), eq(broker)); + }); + + shared.forEach(broker -> { + doReturn(true).when(policies).isSharedBroker(eq(broker)); + }); + + doAnswer(invocationOnMock -> { + Integer totalPrimaryCandidates = invocationOnMock.getArgument(1, Integer.class); + return totalPrimaryCandidates < min_limit; + }).when(policies).shouldFailoverToSecondaries(eq(namespaceName), anyInt()); + } + + private PulsarService getMockPulsar() { + var pulsar = mock(PulsarService.class); + var namespaceService = mock(NamespaceService.class); + doReturn(namespaceService).when(pulsar).getNamespaceService(); + NamespaceBundleFactory factory = mock(NamespaceBundleFactory.class); + doReturn(factory).when(namespaceService).getNamespaceBundleFactory(); + doAnswer(answer -> { + String namespace = answer.getArgument(0, String.class); + String bundleRange = answer.getArgument(1, String.class); + String[] boundaries = bundleRange.split("_"); + Long lowerEndpoint = Long.decode(boundaries[0]); + Long upperEndpoint = Long.decode(boundaries[1]); + Range hashRange = Range.range(lowerEndpoint, BoundType.CLOSED, upperEndpoint, + (upperEndpoint.equals(NamespaceBundles.FULL_UPPER_BOUND)) ? BoundType.CLOSED : BoundType.OPEN); + return new NamespaceBundle(NamespaceName.get(namespace), hashRange, factory); + }).when(factory).getBundle(anyString(), anyString()); + return pulsar; + } + + @Test public void testBundlesWithAntiAffinityGroup() throws IllegalAccessException, MetadataStoreException { - var pulsar = mock(PulsarService.class); + var pulsar = getMockPulsar(); TransferShedder transferShedder = new TransferShedder(pulsar); var allocationPoliciesSpy = (SimpleResourceAllocationPolicies) spy(FieldUtils.readDeclaredField(transferShedder, "allocationPolicies", true)); @@ -392,11 +582,16 @@ public void testBundlesWithAntiAffinityGroup() throws IllegalAccessException, Me var ctx = setupContext(); var topBundlesLoadDataStore = ctx.topBundleLoadDataStore(); - topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA/0x00000000_0xFFFFFFF", 1, 3)); - topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB/0x00000000_0xFFFFFFF", 2, 8)); - topBundlesLoadDataStore.pushAsync("broker3", getTopBundlesLoad("my-tenant/my-namespaceC/0x00000000_0xFFFFFFF", 6, 10)); - topBundlesLoadDataStore.pushAsync("broker4", getTopBundlesLoad("my-tenant/my-namespaceD/0x00000000_0xFFFFFFF", 10, 20)); - topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE/0x00000000_0xFFFFFFF", 70, 20)); + topBundlesLoadDataStore.pushAsync("broker1", + getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceA", 1, 3)); + topBundlesLoadDataStore.pushAsync("broker2", + getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceB", 2, 8)); + topBundlesLoadDataStore.pushAsync("broker3", + getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceC", 6, 10)); + topBundlesLoadDataStore.pushAsync("broker4", + getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceD", 10, 20)); + topBundlesLoadDataStore.pushAsync("broker5", + getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceE", 70, 20)); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected = new UnloadDecision(); From 44993ce98280280a2cc9327faa0f1026acf742b3 Mon Sep 17 00:00:00 2001 From: Masahiro Sakamoto Date: Sun, 12 Mar 2023 14:39:30 +0900 Subject: [PATCH 101/174] [fix][broker] Fix issue where msgRateExpired may not refresh forever (#19759) --- .../persistent/PersistentReplicator.java | 2 + .../service/persistent/PersistentTopic.java | 1 + .../api/SimpleProducerConsumerStatTest.java | 41 +++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java index 2d341016f089e..f38bcc71582a1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java @@ -558,6 +558,8 @@ public void deleteFailed(ManagedLedgerException exception, Object ctx) { public void updateRates() { msgOut.calculateRate(); msgExpired.calculateRate(); + expiryMonitor.updateRates(); + stats.msgRateOut = msgOut.getRate(); stats.msgThroughputOut = msgOut.getValueRate(); stats.msgRateExpired = msgExpired.getRate() + expiryMonitor.getMessageExpiryRate(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 79717ed57303f..b3b6526eea54d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -1984,6 +1984,7 @@ public void updateRates(NamespaceStats nsStats, NamespaceBundleStats bundleStats // Populate subscription specific stats here topicStatsStream.writePair("msgBacklog", subscription.getNumberOfEntriesInBacklog(true)); + subscription.getExpiryMonitor().updateRates(); topicStatsStream.writePair("msgRateExpired", subscription.getExpiredMessageRate()); topicStatsStream.writePair("msgRateOut", subMsgRateOut); topicStatsStream.writePair("messageAckRate", subMsgAckRate); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerStatTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerStatTest.java index 2fa3ed9ad01c7..89e6c684ee17f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerStatTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerStatTest.java @@ -520,4 +520,45 @@ public void testPartitionTopicStats() throws Exception { log.info("-- Exiting {} test --", methodName); } + + @Test + public void testMsgRateExpired() throws Exception { + log.info("-- Starting {} test --", methodName); + + String topicName = "persistent://my-property/tp1/my-ns/" + methodName; + String subName = "my-sub"; + admin.topics().createSubscription(topicName, subName, MessageId.latest); + + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(topicName) + .enableBatching(false) + .create(); + + int numMessages = 100; + for (int i = 0; i < numMessages; i++) { + String message = "my-message-" + i; + producer.send(message.getBytes()); + } + + Thread.sleep(2000); + admin.topics().expireMessages(topicName, subName, 1); + pulsar.getBrokerService().updateRates(); + + Awaitility.await().ignoreExceptions().timeout(5, TimeUnit.SECONDS) + .until(() -> admin.topics().getStats(topicName).getSubscriptions().get(subName).getMsgRateExpired() > 0.001); + + Thread.sleep(2000); + pulsar.getBrokerService().updateRates(); + + Awaitility.await().ignoreExceptions().timeout(5, TimeUnit.SECONDS) + .until(() -> admin.topics().getStats(topicName).getSubscriptions().get(subName).getMsgRateExpired() < 0.001); + + assertEquals(admin.topics().getStats(topicName).getSubscriptions().get(subName).getMsgRateExpired(), 0.0, + 0.001); + assertEquals(admin.topics().getStats(topicName).getSubscriptions().get(subName).getTotalMsgExpired(), + numMessages); + + log.info("-- Exiting {} test --", methodName); + } } From eb3169937e93d87d7587331aab18077849e14ca7 Mon Sep 17 00:00:00 2001 From: ran Date: Sun, 12 Mar 2023 15:16:30 +0800 Subject: [PATCH 102/174] [fix][admin] Filter pending ack topic while deleting the namespace (#19719) ### Motivation A transaction system topic not found exception may occur while deleting the namespace. **How to happen?** 1. Make sure the topic has a pending ack system topic(`public/default/test-delete-ns-sub__transaction_pending_ack`). 2. Delete the namespace `public/default`. 3. Namespace deletion operation will try to delete the user-created topic `public/default/test-delete-ns` first, at this step, the topic will unsubscribe from all subscriptions, and delete the corresponding pending ack system topic. 4. After the namespace deletion operation delete all user-created topics, it will try to delete all system topics, which contain the pending ack topic `public/default/test-delete-ns-sub__transaction_pending_ack`. 5. The topicNotFound exception occurs. There are two ways to fix this problem. 1. Remove the pending ack topics from the pre-delete topic list. 2. Ignore the topicNotFound exception while deleting the namespace. ~~I think ignoring the exception is better because we don't know if there are other similar topics in the future.~~ After discussion, we decide to filter the pending ack topics while deleting the namespace. ### Modifications Ignore the topicNotFound exception while deleting the namespace. --- .../broker/admin/impl/NamespacesBase.java | 7 +++- .../broker/transaction/TransactionTest.java | 34 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 5be675f7b636c..cffc94b189225 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -249,7 +249,7 @@ private void internalRetryableDeleteNamespaceAsync0(boolean force, int retryTime } else { if (SystemTopicNames.isTopicPoliciesSystemTopic(topic)) { topicPolicy.add(topic); - } else { + } else if (!isDeletedAlongWithUserCreatedTopic(topic)) { allSystemTopics.add(topic); } } @@ -344,6 +344,11 @@ private void internalRetryableDeleteNamespaceAsync0(boolean force, int retryTime }); } + private boolean isDeletedAlongWithUserCreatedTopic(String topic) { + // The transaction pending ack topic will be deleted while topic unsubscribe corresponding subscription. + return topic.endsWith(SystemTopicNames.PENDING_ACK_STORE_SUFFIX); + } + private CompletableFuture internalDeletePartitionedTopicsAsync(List topicNames) { if (CollectionUtils.isEmpty(topicNames)) { return CompletableFuture.completedFuture(null); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index a568db3d9f143..20aeac0ed648f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -76,6 +76,7 @@ import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.commons.lang3.reflect.MethodUtils; import org.apache.pulsar.broker.PulsarService; @@ -114,6 +115,7 @@ import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.client.api.ReaderBuilder; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.transaction.Transaction; import org.apache.pulsar.client.api.transaction.TxnID; @@ -1642,4 +1644,36 @@ public void testEncryptionRequired() throws Exception { .send(); txn.commit(); } + + @Test + public void testDeleteNamespace() throws Exception { + String namespace = TENANT + "/ns-" + RandomStringUtils.randomAlphabetic(5); + String topic = namespace + "/test-delete-ns"; + admin.namespaces().createNamespace(namespace); + try (Producer producer = pulsarClient.newProducer() + .topic(topic) + .create()) { + producer.newMessage().value("test".getBytes()).send(); + + Transaction txn = this.pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.SECONDS) + .build().get(); + try (Consumer consumer = this.pulsarClient.newConsumer() + .topic(topic) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscriptionName("sub") + .subscribe()) { + Message message = consumer.receive(); + consumer.acknowledgeAsync(message.getMessageId(), txn).get(); + } + try (Producer outProducer = this.pulsarClient.newProducer() + .topic(topic + "-out") + .create()) { + outProducer.newMessage(txn).value("output".getBytes()).send(); + } + txn.commit(); + } + admin.namespaces().deleteNamespace(namespace, true); + } + } From 5caca1a995d316b73e3d6a9ba2ab4fdcc02211da Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Mon, 13 Mar 2023 09:13:11 +0800 Subject: [PATCH 103/174] [improve][meta] Fix busy-loop causes watcher can't acquire lock. (#19769) --- .../java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java | 3 ++- .../test/java/org/apache/pulsar/metadata/ZKSessionTest.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java index a0247e2231949..1ce01f57d4fbe 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java @@ -66,7 +66,8 @@ public ZKSessionWatcher(ZooKeeper zk, Consumer sessionListener) { this.scheduler = Executors .newSingleThreadScheduledExecutor(new DefaultThreadFactory("metadata-store-zk-session-watcher")); this.task = - scheduler.scheduleAtFixedRate(catchingAndLoggingThrowables(this::checkConnectionStatus), tickTimeMillis, + scheduler.scheduleWithFixedDelay( + catchingAndLoggingThrowables(this::checkConnectionStatus), tickTimeMillis, tickTimeMillis, TimeUnit.MILLISECONDS); this.currentStatus = SessionEvent.SessionReestablished; diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/ZKSessionTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/ZKSessionTest.java index 1757a6b991c0f..36cb0f132ba58 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/ZKSessionTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/ZKSessionTest.java @@ -40,7 +40,7 @@ import org.awaitility.Awaitility; import org.testng.annotations.Test; -@Test(groups = "quarantine") +@Test public class ZKSessionTest extends BaseMetadataStoreTest { @Test From 16a30a7037836641bf9453c7d07b3b6aab80ec26 Mon Sep 17 00:00:00 2001 From: Zhangao Date: Mon, 13 Mar 2023 14:13:23 +0800 Subject: [PATCH 104/174] [fix][broker] Fix index generator is not rollback after entries are failed added. (#19727) Co-authored-by: gavingaozhangmin --- .../mledger/impl/ManagedLedgerImpl.java | 7 +++ .../bookkeeper/mledger/impl/OpAddEntry.java | 1 + .../intercept/ManagedLedgerInterceptor.java | 8 ++++ .../ManagedLedgerInterceptorImpl.java | 9 ++++ .../MangedLedgerInterceptorImplTest.java | 43 +++++++++++++++++++ .../AppendIndexMetadataInterceptor.java | 4 ++ 6 files changed, 72 insertions(+) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 9c05fb7c1047e..8376ee1bb8467 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -866,6 +866,13 @@ protected synchronized void internalAsyncAddEntry(OpAddEntry addOperation) { lastAddEntryTimeMs = System.currentTimeMillis(); } + protected void afterFailedAddEntry(int numOfMessages) { + if (managedLedgerInterceptor == null) { + return; + } + managedLedgerInterceptor.afterFailedAddEntry(numOfMessages); + } + protected boolean beforeAddEntry(OpAddEntry addOperation) { // if no interceptor, just return true to make sure addOperation will be initiate() if (managedLedgerInterceptor == null) { diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java index c6c341fd921d1..c56123c24cac1 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java @@ -162,6 +162,7 @@ public void initiateShadowWrite() { public void failed(ManagedLedgerException e) { AddEntryCallback cb = callbackUpdater.getAndSet(this, null); + ml.afterFailedAddEntry(this.getNumberOfMessages()); if (cb != null) { ReferenceCountUtil.release(data); cb.addFailed(e, ctx); diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/intercept/ManagedLedgerInterceptor.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/intercept/ManagedLedgerInterceptor.java index 412655594c770..d26a5e15735aa 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/intercept/ManagedLedgerInterceptor.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/intercept/ManagedLedgerInterceptor.java @@ -41,6 +41,14 @@ public interface ManagedLedgerInterceptor { */ OpAddEntry beforeAddEntry(OpAddEntry op, int numberOfMessages); + /** + * Intercept When add entry failed. + * @param numberOfMessages + */ + default void afterFailedAddEntry(int numberOfMessages){ + + } + /** * Intercept when ManagedLedger is initialized. * @param propertiesMap map of properties. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/ManagedLedgerInterceptorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/ManagedLedgerInterceptorImpl.java index e3b94ec94a4c1..30713c91907ef 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/ManagedLedgerInterceptorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/ManagedLedgerInterceptorImpl.java @@ -79,6 +79,15 @@ public OpAddEntry beforeAddEntry(OpAddEntry op, int numberOfMessages) { return op; } + @Override + public void afterFailedAddEntry(int numberOfMessages) { + for (BrokerEntryMetadataInterceptor interceptor : brokerEntryMetadataInterceptors) { + if (interceptor instanceof AppendIndexMetadataInterceptor) { + ((AppendIndexMetadataInterceptor) interceptor).decreaseWithNumberOfMessages(numberOfMessages); + } + } + } + @Override public void onManagedLedgerPropertiesInitialize(Map propertiesMap) { if (propertiesMap == null || propertiesMap.size() == 0) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/MangedLedgerInterceptorImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/MangedLedgerInterceptorImplTest.java index 0cf0adb5181ec..7d164b68147ab 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/MangedLedgerInterceptorImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/MangedLedgerInterceptorImplTest.java @@ -305,6 +305,49 @@ public void testFindPositionByIndex() throws Exception { ledger.close(); } + @Test + public void testAddEntryFailed() throws Exception { + final int MOCK_BATCH_SIZE = 2; + final String ledgerAndCursorName = "testAddEntryFailed"; + + ManagedLedgerInterceptor interceptor = + new ManagedLedgerInterceptorImpl(getBrokerEntryMetadataInterceptors(), null); + + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setMaxEntriesPerLedger(2); + config.setManagedLedgerInterceptor(interceptor); + + ByteBuf buffer = Unpooled.wrappedBuffer("message".getBytes()); + ManagedLedger ledger = factory.open(ledgerAndCursorName, config); + + ledger.terminate(); + + ManagedLedgerInterceptorImpl interceptor1 = + (ManagedLedgerInterceptorImpl) ledger.getManagedLedgerInterceptor(); + + CountDownLatch countDownLatch = new CountDownLatch(1); + try { + ledger.asyncAddEntry(buffer, MOCK_BATCH_SIZE, new AsyncCallbacks.AddEntryCallback() { + @Override + public void addComplete(Position position, ByteBuf entryData, Object ctx) { + countDownLatch.countDown(); + } + + @Override + public void addFailed(ManagedLedgerException exception, Object ctx) { + countDownLatch.countDown(); + } + }, null); + + countDownLatch.await(); + assertEquals(interceptor1.getIndex(), -1); + } finally { + ledger.close(); + factory.shutdown(); + } + + } + @Test public void testBeforeAddEntryWithException() throws Exception { final int MOCK_BATCH_SIZE = 2; diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/intercept/AppendIndexMetadataInterceptor.java b/pulsar-common/src/main/java/org/apache/pulsar/common/intercept/AppendIndexMetadataInterceptor.java index de35cc4498d78..941ae16aef1eb 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/intercept/AppendIndexMetadataInterceptor.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/intercept/AppendIndexMetadataInterceptor.java @@ -50,4 +50,8 @@ public BrokerEntryMetadata interceptWithNumberOfMessages( public long getIndex() { return indexGenerator.get(); } + + public void decreaseWithNumberOfMessages(int numberOfMessages) { + indexGenerator.addAndGet(-numberOfMessages); + } } From b1a463abec0330d3788b1620e50c634c529148c0 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Mon, 13 Mar 2023 21:03:22 +0800 Subject: [PATCH 105/174] [fix][broker] Fix admin api status code compatibility (#19782) --- .../broker/admin/impl/PersistentTopicsBase.java | 2 +- .../pulsar/broker/admin/AdminApi2Test.java | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index 633c4747ee068..035e32542ed71 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -420,7 +420,7 @@ protected CompletableFuture internalCreateNonPartitionedTopicAsync(boolean .thenCompose(partitionedTopicMetadata -> { int currentMetadataPartitions = partitionedTopicMetadata.partitions; if (currentMetadataPartitions <= 0) { - throw new RestException(422 /* Unprocessable entity*/, + throw new RestException(Status.CONFLICT /* Unprocessable entity*/, String.format("Topic %s is not the partitioned topic.", topicName)); } if (expectPartitions < currentMetadataPartitions) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java index 90be220cfd8f7..fb4f880efff07 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java @@ -211,6 +211,23 @@ public Object[][] isV1() { return new Object[][] { { true }, { false } }; } + + /** + * It verifies http error code when updating partitions to ensure compatibility. + */ + @Test + public void testUpdatePartitionsErrorCode() { + final String nonPartitionedTopicName = "non-partitioned-topic-name" + UUID.randomUUID(); + try { + // Update a non-partitioned topic + admin.topics().updatePartitionedTopic(nonPartitionedTopicName, 2); + Assert.fail("Expect conflict exception."); + } catch (PulsarAdminException ex) { + Assert.assertEquals(ex.getStatusCode(), 409 /*Conflict*/); + Assert.assertTrue(ex instanceof PulsarAdminException.ConflictException); + } + } + /** * 
      * It verifies increasing partitions for partitioned-topic.

From 9a85dea53a6d0896cc4ba1ceb1ab11a47d5d65da Mon Sep 17 00:00:00 2001
From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com>
Date: Mon, 13 Mar 2023 18:39:07 -0700
Subject: [PATCH 106/174] [improve][broker] PIP-192 Added SplitScheduler and
 DefaultNamespaceBundleSplitStrategyImpl (#19622)

Master Issue: https://github.com/apache/pulsar/issues/16691

### Motivation

We will start raising PRs to implement PIP-192, https://github.com/apache/pulsar/issues/16691

### Modifications

This PR implemented
- SplitScheduler
- DefaultNamespaceBundleSplitStrategyImpl
- SplitManager
-  and their unit test.
---
 .../pulsar/broker/ServiceConfiguration.java   |  24 ++
 .../extensions/ExtensibleLoadManagerImpl.java |  22 +-
 .../extensions/manager/SplitManager.java      | 119 ++++++++
 .../extensions/models/SplitCounter.java       |  46 +--
 .../extensions/models/SplitDecision.java      |   9 -
 .../extensions/scheduler/SplitScheduler.java  | 177 +++++++++++
 ...faultNamespaceBundleSplitStrategyImpl.java | 171 +++++++++++
 .../NamespaceBundleSplitStrategy.java         |   7 +-
 .../ExtensibleLoadManagerImplTest.java        |  31 +-
 .../extensions/manager/SplitManagerTest.java  | 222 ++++++++++++++
 .../scheduler/SplitSchedulerTest.java         | 158 ++++++++++
 ...faultNamespaceBundleSplitStrategyTest.java | 284 ++++++++++++++++++
 12 files changed, 1208 insertions(+), 62 deletions(-)
 create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/SplitManager.java
 create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitScheduler.java
 create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java
 rename pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/{scheduler => strategy}/NamespaceBundleSplitStrategy.java (86%)
 create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/SplitManagerTest.java
 create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitSchedulerTest.java
 create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java

diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index ec5b0d4042bcd..3c00e905ac723 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -2557,6 +2557,30 @@ The delayed message index bucket time step(in seconds) in per bucket snapshot se
                     + "(only used in load balancer extension logics)"
     )
     private double loadBalancerBundleLoadReportPercentage = 10;
+    @FieldContext(
+            category = CATEGORY_LOAD_BALANCER,
+            doc = "Service units'(bundles) split interval. Broker periodically checks whether "
+                    + "some service units(e.g. bundles) should split if they become hot-spots. "
+                    + "(only used in load balancer extension logics)"
+    )
+    private int loadBalancerSplitIntervalMinutes = 1;
+    @FieldContext(
+            category = CATEGORY_LOAD_BALANCER,
+            dynamic = true,
+            doc = "Max number of bundles to split to per cycle. "
+                    + "(only used in load balancer extension logics)"
+    )
+    private int loadBalancerMaxNumberOfBundlesToSplitPerCycle = 10;
+    @FieldContext(
+            category = CATEGORY_LOAD_BALANCER,
+            dynamic = true,
+            doc = "Threshold to the consecutive count of fulfilled split conditions. "
+                    + "If the split scheduler consecutively finds bundles that meet split conditions "
+                    + "many times bigger than this threshold, the scheduler will trigger splits on the bundles "
+                    + "(if the number of bundles is less than loadBalancerNamespaceMaximumBundles). "
+                    + "(only used in load balancer extension logics)"
+    )
+    private int loadBalancerNamespaceBundleSplitConditionThreshold = 5;
 
     @FieldContext(
             category = CATEGORY_LOAD_BALANCER,
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
index 82790f44fcb28..c8ac8e4684506 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
@@ -45,16 +45,17 @@
 import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerIsolationPoliciesFilter;
 import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerMaxTopicCountFilter;
 import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerVersionFilter;
+import org.apache.pulsar.broker.loadbalance.extensions.manager.SplitManager;
 import org.apache.pulsar.broker.loadbalance.extensions.manager.UnloadManager;
 import org.apache.pulsar.broker.loadbalance.extensions.models.AssignCounter;
 import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter;
-import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision;
 import org.apache.pulsar.broker.loadbalance.extensions.models.Unload;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision;
 import org.apache.pulsar.broker.loadbalance.extensions.reporter.BrokerLoadDataReporter;
 import org.apache.pulsar.broker.loadbalance.extensions.reporter.TopBundleLoadDataReporter;
 import org.apache.pulsar.broker.loadbalance.extensions.scheduler.LoadManagerScheduler;
+import org.apache.pulsar.broker.loadbalance.extensions.scheduler.SplitScheduler;
 import org.apache.pulsar.broker.loadbalance.extensions.scheduler.UnloadScheduler;
 import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore;
 import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStoreException;
@@ -103,7 +104,6 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
 
     @Getter
     private final List brokerFilterPipeline;
-
     /**
      * The load data reporter.
      */
@@ -113,9 +113,12 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
 
     private ScheduledFuture brokerLoadDataReportTask;
     private ScheduledFuture topBundlesLoadDataReportTask;
+    private SplitScheduler splitScheduler;
 
     private UnloadManager unloadManager;
 
+    private SplitManager splitManager;
+
     private boolean started = false;
 
     private final AssignCounter assignCounter = new AssignCounter();
@@ -166,7 +169,9 @@ public void start() throws PulsarServerException {
         this.serviceUnitStateChannel = new ServiceUnitStateChannelImpl(pulsar);
         this.brokerRegistry.start();
         this.unloadManager = new UnloadManager();
+        this.splitManager = new SplitManager(splitCounter);
         this.serviceUnitStateChannel.listen(unloadManager);
+        this.serviceUnitStateChannel.listen(splitManager);
         this.serviceUnitStateChannel.start();
 
         try {
@@ -184,7 +189,6 @@ public void start() throws PulsarServerException {
                 .brokerLoadDataStore(brokerLoadDataStore)
                 .topBundleLoadDataStore(topBundlesLoadDataStore).build();
 
-
         this.brokerLoadDataReporter =
                 new BrokerLoadDataReporter(pulsar, brokerRegistry.getBrokerId(), brokerLoadDataStore);
 
@@ -216,10 +220,12 @@ public void start() throws PulsarServerException {
                         interval,
                         interval, TimeUnit.MILLISECONDS);
 
-        // TODO: Start bundle split scheduler.
         this.unloadScheduler = new UnloadScheduler(
                 pulsar.getLoadManagerExecutor(), unloadManager, context, serviceUnitStateChannel);
         this.unloadScheduler.start();
+        this.splitScheduler = new SplitScheduler(
+                pulsar, serviceUnitStateChannel, splitManager, splitCounter, splitMetrics, context);
+        this.splitScheduler.start();
         this.started = true;
     }
 
@@ -380,6 +386,7 @@ public void close() throws PulsarServerException {
             this.brokerLoadDataStore.close();
             this.topBundlesLoadDataStore.close();
             this.unloadScheduler.close();
+            this.splitScheduler.close();
         } catch (IOException ex) {
             throw new PulsarServerException(ex);
         } finally {
@@ -411,13 +418,6 @@ private void updateUnloadMetrics(UnloadDecision decision) {
         this.unloadMetrics.set(unloadCounter.toMetrics(pulsar.getAdvertisedAddress()));
     }
 
-    private void updateSplitMetrics(List decisions) {
-        for (var decision : decisions) {
-            splitCounter.update(decision);
-        }
-        this.splitMetrics.set(splitCounter.toMetrics(pulsar.getAdvertisedAddress()));
-    }
-
     public List getMetrics() {
         List metricsCollection = new ArrayList<>();
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/SplitManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/SplitManager.java
new file mode 100644
index 0000000000000..71ebbc92a87db
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/SplitManager.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.loadbalance.extensions.manager;
+
+import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Label.Failure;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Unknown;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData;
+import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter;
+import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision;
+
+/**
+ * Split manager.
+ */
+@Slf4j
+public class SplitManager implements StateChangeListener {
+
+
+    private final Map> inFlightSplitRequests;
+
+    private final SplitCounter counter;
+
+    public SplitManager(SplitCounter splitCounter) {
+        this.inFlightSplitRequests = new ConcurrentHashMap<>();
+        this.counter = splitCounter;
+    }
+
+    private void complete(String serviceUnit, Throwable ex) {
+        inFlightSplitRequests.computeIfPresent(serviceUnit, (__, future) -> {
+            if (!future.isDone()) {
+                if (ex != null) {
+                    future.completeExceptionally(ex);
+                } else {
+                    future.complete(null);
+                }
+            }
+            return null;
+        });
+    }
+
+    public CompletableFuture waitAsync(CompletableFuture eventPubFuture,
+                                             String bundle,
+                                             SplitDecision decision,
+                                             long timeout,
+                                             TimeUnit timeoutUnit) {
+        return eventPubFuture
+                .thenCompose(__ -> inFlightSplitRequests.computeIfAbsent(bundle, ignore -> {
+                    log.info("Published the bundle split event for bundle:{}. "
+                                    + "Waiting the split event to complete. Timeout: {} {}",
+                            bundle, timeout, timeoutUnit);
+                    CompletableFuture future = new CompletableFuture<>();
+                    future.orTimeout(timeout, timeoutUnit).whenComplete((v, ex) -> {
+                        if (ex != null) {
+                            inFlightSplitRequests.remove(bundle);
+                            log.warn("Timed out while waiting for the bundle split event: {}", bundle, ex);
+                        }
+                    });
+                    return future;
+                }))
+                .whenComplete((__, ex) -> {
+                    if (ex != null) {
+                        log.error("Failed the bundle split event for bundle:{}", bundle, ex);
+                        counter.update(Failure, Unknown);
+                    } else {
+                        log.info("Completed the bundle split event for bundle:{}", bundle);
+                        counter.update(decision);
+                    }
+                });
+    }
+
+    @Override
+    public void handleEvent(String serviceUnit, ServiceUnitStateData data, Throwable t) {
+        ServiceUnitState state = ServiceUnitStateData.state(data);
+        if (t != null && inFlightSplitRequests.containsKey(serviceUnit)) {
+            this.complete(serviceUnit, t);
+            return;
+        }
+        switch (state) {
+            case Deleted, Owned, Init -> this.complete(serviceUnit, t);
+            default -> {
+                if (log.isDebugEnabled()) {
+                    log.debug("Handling {} for service unit {}", data, serviceUnit);
+                }
+            }
+        }
+    }
+
+    public void close() {
+        inFlightSplitRequests.forEach((bundle, future) -> {
+            if (!future.isDone()) {
+                String msg = String.format("Splitting bundle: %s, but the manager already closed.", bundle);
+                log.warn(msg);
+                future.completeExceptionally(new IllegalStateException(msg));
+            }
+        });
+        inFlightSplitRequests.clear();
+    }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/SplitCounter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/SplitCounter.java
index 99406412cee2b..ed72b5f586331 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/SplitCounter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/SplitCounter.java
@@ -19,10 +19,8 @@
 package org.apache.pulsar.broker.loadbalance.extensions.models;
 
 import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Label.Failure;
-import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Label.Skip;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Label.Success;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Admin;
-import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Balanced;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Bandwidth;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.MsgRate;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Sessions;
@@ -32,7 +30,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import org.apache.commons.lang3.mutable.MutableLong;
+import java.util.concurrent.atomic.AtomicLong;
 import org.apache.pulsar.common.stats.Metrics;
 
 /**
@@ -40,23 +38,20 @@
  */
 public class SplitCounter {
 
-    long splitCount = 0;
-
-    final Map> breakdownCounters;
+    private long splitCount = 0;
+    private final Map> breakdownCounters;
+    private volatile long updatedAt = 0;
 
     public SplitCounter() {
         breakdownCounters = Map.of(
                 Success, Map.of(
-                        Topics, new MutableLong(),
-                        Sessions, new MutableLong(),
-                        MsgRate, new MutableLong(),
-                        Bandwidth, new MutableLong(),
-                        Admin, new MutableLong()),
-                Skip, Map.of(
-                        Balanced, new MutableLong()
-                        ),
+                        Topics, new AtomicLong(),
+                        Sessions, new AtomicLong(),
+                        MsgRate, new AtomicLong(),
+                        Bandwidth, new AtomicLong(),
+                        Admin, new AtomicLong()),
                 Failure, Map.of(
-                        Unknown, new MutableLong())
+                        Unknown, new AtomicLong())
         );
     }
 
@@ -64,7 +59,16 @@ public void update(SplitDecision decision) {
         if (decision.label == Success) {
             splitCount++;
         }
-        breakdownCounters.get(decision.getLabel()).get(decision.getReason()).increment();
+        breakdownCounters.get(decision.getLabel()).get(decision.getReason()).incrementAndGet();
+        updatedAt = System.currentTimeMillis();
+    }
+
+    public void update(SplitDecision.Label label, SplitDecision.Reason reason) {
+        if (label == Success) {
+            splitCount++;
+        }
+        breakdownCounters.get(label).get(reason).incrementAndGet();
+        updatedAt = System.currentTimeMillis();
     }
 
     public List toMetrics(String advertisedBrokerAddress) {
@@ -77,17 +81,18 @@ public List toMetrics(String advertisedBrokerAddress) {
         m.put("brk_lb_bundles_split_total", splitCount);
         metrics.add(m);
 
-        for (Map.Entry> etr
+
+        for (Map.Entry> etr
                 : breakdownCounters.entrySet()) {
             var result = etr.getKey();
-            for (Map.Entry counter : etr.getValue().entrySet()) {
+            for (Map.Entry counter : etr.getValue().entrySet()) {
                 var reason = counter.getKey();
                 var count = counter.getValue();
                 Map breakdownDims = new HashMap<>(dimensions);
                 breakdownDims.put("result", result.toString());
                 breakdownDims.put("reason", reason.toString());
                 Metrics breakdownMetric = Metrics.create(breakdownDims);
-                breakdownMetric.put("brk_lb_bundles_split_breakdown_total", count);
+                breakdownMetric.put("brk_lb_bundles_split_breakdown_total", count.get());
                 metrics.add(breakdownMetric);
             }
         }
@@ -95,4 +100,7 @@ public List toMetrics(String advertisedBrokerAddress) {
         return metrics;
     }
 
+    public long updatedAt() {
+        return updatedAt;
+    }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/SplitDecision.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/SplitDecision.java
index a3dede50c1cd8..433d21a5a613e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/SplitDecision.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/SplitDecision.java
@@ -19,9 +19,7 @@
 package org.apache.pulsar.broker.loadbalance.extensions.models;
 
 import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Label.Failure;
-import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Label.Skip;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Label.Success;
-import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Balanced;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Unknown;
 import lombok.Data;
 
@@ -36,7 +34,6 @@ public class SplitDecision {
 
     public enum Label {
         Success,
-        Skip,
         Failure
     }
 
@@ -46,7 +43,6 @@ public enum Reason {
         MsgRate,
         Bandwidth,
         Admin,
-        Balanced,
         Unknown
     }
 
@@ -62,11 +58,6 @@ public void clear() {
         reason = null;
     }
 
-    public void skip() {
-        label = Skip;
-        reason = Balanced;
-    }
-
     public void succeed(Reason reason) {
         label = Success;
         this.reason = reason;
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitScheduler.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitScheduler.java
new file mode 100644
index 0000000000000..589df80fc5c14
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitScheduler.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.loadbalance.extensions.scheduler;
+
+import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Label.Success;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel;
+import org.apache.pulsar.broker.loadbalance.extensions.manager.SplitManager;
+import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter;
+import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision;
+import org.apache.pulsar.broker.loadbalance.extensions.strategy.DefaultNamespaceBundleSplitStrategyImpl;
+import org.apache.pulsar.broker.loadbalance.extensions.strategy.NamespaceBundleSplitStrategy;
+import org.apache.pulsar.common.stats.Metrics;
+import org.apache.pulsar.common.util.FutureUtil;
+
+/**
+ * Service Unit(e.g. bundles) Split scheduler.
+ */
+@Slf4j
+public class SplitScheduler implements LoadManagerScheduler {
+
+    private final PulsarService pulsar;
+
+    private final ScheduledExecutorService loadManagerExecutor;
+
+    private final LoadManagerContext context;
+
+    private final ServiceConfiguration conf;
+
+    private final ServiceUnitStateChannel serviceUnitStateChannel;
+
+    private final NamespaceBundleSplitStrategy bundleSplitStrategy;
+
+    private final SplitCounter counter;
+
+    private final SplitManager splitManager;
+
+    private final AtomicReference> splitMetrics;
+
+    private volatile ScheduledFuture task;
+
+    private long counterLastUpdatedAt = 0;
+
+    public SplitScheduler(PulsarService pulsar,
+                          ServiceUnitStateChannel serviceUnitStateChannel,
+                          SplitManager splitManager,
+                          SplitCounter counter,
+                          AtomicReference> splitMetrics,
+                          LoadManagerContext context,
+                          NamespaceBundleSplitStrategy bundleSplitStrategy) {
+        this.pulsar = pulsar;
+        this.loadManagerExecutor = pulsar.getLoadManagerExecutor();
+        this.splitManager = splitManager;
+        this.counter = counter;
+        this.splitMetrics = splitMetrics;
+        this.context = context;
+        this.conf = pulsar.getConfiguration();
+        this.bundleSplitStrategy = bundleSplitStrategy;
+        this.serviceUnitStateChannel = serviceUnitStateChannel;
+    }
+
+    public SplitScheduler(PulsarService pulsar,
+                          ServiceUnitStateChannel serviceUnitStateChannel,
+                          SplitManager splitManager,
+                          SplitCounter counter,
+                          AtomicReference> splitMetrics,
+                          LoadManagerContext context) {
+        this(pulsar, serviceUnitStateChannel, splitManager, counter, splitMetrics, context,
+                new DefaultNamespaceBundleSplitStrategyImpl(counter));
+    }
+
+    @Override
+    public void execute() {
+        boolean debugMode = conf.isLoadBalancerDebugModeEnabled() || log.isDebugEnabled();
+        if (debugMode) {
+            log.info("Load balancer enabled: {}, Split enabled: {}.",
+                    conf.isLoadBalancerEnabled(), conf.isLoadBalancerAutoBundleSplitEnabled());
+        }
+
+        if (!isLoadBalancerAutoBundleSplitEnabled()) {
+            if (debugMode) {
+                log.info("The load balancer or load balancer split already disabled. Skipping.");
+            }
+            return;
+        }
+
+        synchronized (bundleSplitStrategy) {
+            final Set decisions = bundleSplitStrategy.findBundlesToSplit(context, pulsar);
+            if (!decisions.isEmpty()) {
+
+                // currently following the unloading timeout
+                var asyncOpTimeoutMs = conf.getNamespaceBundleUnloadingTimeoutMs();
+                List> futures = new ArrayList<>();
+                for (SplitDecision decision : decisions) {
+                    if (decision.getLabel() == Success) {
+                        var split = decision.getSplit();
+                        futures.add(
+                                splitManager.waitAsync(
+                                        serviceUnitStateChannel.publishSplitEventAsync(split),
+                                        split.serviceUnit(),
+                                        decision,
+                                        asyncOpTimeoutMs, TimeUnit.MILLISECONDS)
+                        );
+                    }
+                }
+                try {
+                    FutureUtil.waitForAll(futures)
+                            .get(asyncOpTimeoutMs, TimeUnit.MILLISECONDS);
+                } catch (Throwable e) {
+                    log.error("Failed to wait for split events to persist.", e);
+                }
+            } else {
+                if (debugMode) {
+                    log.info("BundleSplitStrategy returned no bundles to split.");
+                }
+            }
+        }
+
+        if (counter.updatedAt() > counterLastUpdatedAt) {
+            splitMetrics.set(counter.toMetrics(pulsar.getAdvertisedAddress()));
+            counterLastUpdatedAt = counter.updatedAt();
+        }
+    }
+
+    @Override
+    public void start() {
+        long interval = TimeUnit.MINUTES
+                .toMillis(conf.getLoadBalancerSplitIntervalMinutes());
+        task = loadManagerExecutor.scheduleAtFixedRate(() -> {
+            try {
+                execute();
+            } catch (Throwable e) {
+                log.error("Failed to run the split job.", e);
+            }
+        }, interval, interval, TimeUnit.MILLISECONDS);
+    }
+
+    @Override
+    public void close() {
+        if (task != null) {
+            task.cancel(false);
+            task = null;
+        }
+    }
+
+    private boolean isLoadBalancerAutoBundleSplitEnabled() {
+        return conf.isLoadBalancerEnabled() && conf.isLoadBalancerAutoBundleSplitEnabled();
+    }
+
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java
new file mode 100644
index 0000000000000..e572fd4161bdb
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.loadbalance.extensions.strategy;
+
+import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Label.Failure;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Bandwidth;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.MsgRate;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Sessions;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Topics;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Unknown;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
+import org.apache.pulsar.broker.loadbalance.extensions.models.Split;
+import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter;
+import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision;
+import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared;
+import org.apache.pulsar.common.naming.NamespaceBundleFactory;
+import org.apache.pulsar.common.naming.NamespaceName;
+import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats;
+
+/**
+ * Determines which bundles should be split based on various thresholds.
+ *
+ * Migrate from {@link org.apache.pulsar.broker.loadbalance.impl.BundleSplitterTask}
+ */
+@Slf4j
+public class DefaultNamespaceBundleSplitStrategyImpl implements NamespaceBundleSplitStrategy {
+    private final Set decisionCache;
+    private final Map namespaceBundleCount;
+    private final Map bundleHighTrafficFrequency;
+    private final SplitCounter counter;
+
+    public DefaultNamespaceBundleSplitStrategyImpl(SplitCounter counter) {
+        decisionCache = new HashSet<>();
+        namespaceBundleCount = new HashMap<>();
+        bundleHighTrafficFrequency = new HashMap<>();
+        this.counter = counter;
+
+    }
+
+    @Override
+    public Set findBundlesToSplit(LoadManagerContext context, PulsarService pulsar) {
+        decisionCache.clear();
+        namespaceBundleCount.clear();
+        final ServiceConfiguration conf = pulsar.getConfiguration();
+        int maxBundleCount = conf.getLoadBalancerNamespaceMaximumBundles();
+        long maxBundleTopics = conf.getLoadBalancerNamespaceBundleMaxTopics();
+        long maxBundleSessions = conf.getLoadBalancerNamespaceBundleMaxSessions();
+        long maxBundleMsgRate = conf.getLoadBalancerNamespaceBundleMaxMsgRate();
+        long maxBundleBandwidth = conf.getLoadBalancerNamespaceBundleMaxBandwidthMbytes() * LoadManagerShared.MIBI;
+        long maxSplitCount = conf.getLoadBalancerMaxNumberOfBundlesToSplitPerCycle();
+        long splitConditionThreshold = conf.getLoadBalancerNamespaceBundleSplitConditionThreshold();
+        boolean debug = log.isDebugEnabled() || conf.isLoadBalancerDebugModeEnabled();
+
+        Map bundleStatsMap = pulsar.getBrokerService().getBundleStats();
+        NamespaceBundleFactory namespaceBundleFactory =
+                pulsar.getNamespaceService().getNamespaceBundleFactory();
+
+        // clean bundleHighTrafficFrequency
+        bundleHighTrafficFrequency.keySet().retainAll(bundleStatsMap.keySet());
+
+        for (var entry : bundleStatsMap.entrySet()) {
+            final String bundle = entry.getKey();
+            final NamespaceBundleStats stats = entry.getValue();
+            if (stats.topics < 2) {
+                if (debug) {
+                    log.info("The count of topics on the bundle {} is less than 2, skip split!", bundle);
+                }
+                continue;
+            }
+
+            final String namespaceName = LoadManagerShared.getNamespaceNameFromBundleName(bundle);
+            final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle);
+            if (!namespaceBundleFactory
+                    .canSplitBundle(namespaceBundleFactory.getBundle(namespaceName, bundleRange))) {
+                if (debug) {
+                    log.info("Can't split the bundle:{}. invalid bundle range:{}. ", bundle, bundleRange);
+                }
+                counter.update(Failure, Unknown);
+                continue;
+            }
+
+            double totalMessageRate = stats.msgRateIn + stats.msgRateOut;
+            double totalMessageThroughput = stats.msgThroughputIn + stats.msgThroughputOut;
+            int totalSessionCount = stats.consumerCount + stats.producerCount;
+            SplitDecision.Reason reason = Unknown;
+            if (stats.topics > maxBundleTopics) {
+                reason = Topics;
+            } else if (maxBundleSessions > 0 && (totalSessionCount > maxBundleSessions)) {
+                reason = Sessions;
+            } else if (totalMessageRate > maxBundleMsgRate) {
+                reason = MsgRate;
+            } else if (totalMessageThroughput > maxBundleBandwidth) {
+                reason = Bandwidth;
+            }
+
+            if (reason != Unknown) {
+                bundleHighTrafficFrequency.put(bundle, bundleHighTrafficFrequency.getOrDefault(bundle, 0) + 1);
+            } else {
+                bundleHighTrafficFrequency.remove(bundle);
+            }
+
+            if (bundleHighTrafficFrequency.getOrDefault(bundle, 0) > splitConditionThreshold) {
+                final String namespace = LoadManagerShared.getNamespaceNameFromBundleName(bundle);
+                try {
+                    final int bundleCount = pulsar.getNamespaceService()
+                            .getBundleCount(NamespaceName.get(namespace));
+                    if ((bundleCount + namespaceBundleCount.getOrDefault(namespace, 0))
+                            < maxBundleCount) {
+                        if (debug) {
+                            log.info("The bundle {} is considered to split. Topics: {}/{}, Sessions: ({}+{})/{}, "
+                                            + "Message Rate: {}/{} (msgs/s), Message Throughput: {}/{} (MB/s)",
+                                    bundle, stats.topics, maxBundleTopics, stats.producerCount, stats.consumerCount,
+                                    maxBundleSessions, totalMessageRate, maxBundleMsgRate,
+                                    totalMessageThroughput / LoadManagerShared.MIBI,
+                                    maxBundleBandwidth / LoadManagerShared.MIBI);
+                        }
+                        var decision = new SplitDecision();
+                        decision.setSplit(new Split(bundle, context.brokerRegistry().getBrokerId(), new HashMap<>()));
+                        decision.succeed(reason);
+                        decisionCache.add(decision);
+                        int bundleNum = namespaceBundleCount.getOrDefault(namespace, 0);
+                        namespaceBundleCount.put(namespace, bundleNum + 1);
+                        bundleHighTrafficFrequency.remove(bundle);
+                        // Clear namespace bundle-cache
+                        namespaceBundleFactory.invalidateBundleCache(NamespaceName.get(namespaceName));
+                        if (decisionCache.size() == maxSplitCount) {
+                            if (debug) {
+                                log.info("Too many bundles to split in this split cycle {} / {}. Stop.",
+                                        decisionCache.size(), maxSplitCount);
+                            }
+                            break;
+                        }
+                    } else {
+                        if (debug) {
+                            log.info(
+                                    "Could not split namespace bundle {} because namespace {} has too many bundles:"
+                                            + "{}", bundle, namespace, bundleCount);
+                        }
+                    }
+                } catch (Exception e) {
+                    counter.update(Failure, Unknown);
+                    log.warn("Error while computing bundle splits for namespace {}", namespace, e);
+                }
+            }
+        }
+        return decisionCache;
+    }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/NamespaceBundleSplitStrategy.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/NamespaceBundleSplitStrategy.java
similarity index 86%
rename from pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/NamespaceBundleSplitStrategy.java
rename to pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/NamespaceBundleSplitStrategy.java
index 88bd7f0b08780..14023f1b5b01d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/NamespaceBundleSplitStrategy.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/NamespaceBundleSplitStrategy.java
@@ -16,11 +16,12 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.pulsar.broker.loadbalance.extensions.scheduler;
+package org.apache.pulsar.broker.loadbalance.extensions.strategy;
 
 import java.util.Set;
+import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
-import org.apache.pulsar.broker.loadbalance.extensions.models.Split;
+import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision;
 
 /**
  * Determines which bundles should be split based on various thresholds.
@@ -35,5 +36,5 @@ public interface NamespaceBundleSplitStrategy {
      * @param context The context used for decisions.
      * @return A set of the bundles that should be split.
      */
-    Set findBundlesToSplit(LoadManagerContext context);
+    Set findBundlesToSplit(LoadManagerContext context, PulsarService pulsar);
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
index 441415a9d35e5..17fcb9952618b 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
@@ -464,24 +464,16 @@ Unknown, new MutableLong(10))
                     FieldUtils.readDeclaredField(primaryLoadManager, "splitMetrics", true);
             SplitCounter splitCounter = new SplitCounter();
             FieldUtils.writeDeclaredField(splitCounter, "splitCount", 35l, true);
-            FieldUtils.writeDeclaredField(splitCounter, "breakdownCounters", new LinkedHashMap<>() {
-                {
-                    put(SplitDecision.Label.Success, new LinkedHashMap<>() {
-                        {
-                            put(Topics, new MutableLong(1));
-                            put(Sessions, new MutableLong(2));
-                            put(MsgRate, new MutableLong(3));
-                            put(Bandwidth, new MutableLong(4));
-                            put(Admin, new MutableLong(5));
-                        }
-                    });
-                    put(SplitDecision.Label.Skip, Map.of(
-                            SplitDecision.Reason.Balanced, new MutableLong(6)
-                    ));
-                    put(SplitDecision.Label.Failure, Map.of(
-                            SplitDecision.Reason.Unknown, new MutableLong(7)));
-                }
-            }, true);
+            FieldUtils.writeDeclaredField(splitCounter, "breakdownCounters", Map.of(
+                    SplitDecision.Label.Success, Map.of(
+                            Topics, new AtomicLong(1),
+                            Sessions, new AtomicLong(2),
+                            MsgRate, new AtomicLong(3),
+                            Bandwidth, new AtomicLong(4),
+                            Admin, new AtomicLong(5)),
+                    SplitDecision.Label.Failure, Map.of(
+                            SplitDecision.Reason.Unknown, new AtomicLong(6))
+            ), true);
             splitMetrics.set(splitCounter.toMetrics(pulsar.getAdvertisedAddress()));
         }
 
@@ -556,8 +548,7 @@ SplitDecision.Reason.Balanced, new MutableLong(6)
                         dimensions=[{broker=localhost, metric=bundlesSplit, reason=MsgRate, result=Success}], metrics=[{brk_lb_bundles_split_breakdown_total=3}]
                         dimensions=[{broker=localhost, metric=bundlesSplit, reason=Bandwidth, result=Success}], metrics=[{brk_lb_bundles_split_breakdown_total=4}]
                         dimensions=[{broker=localhost, metric=bundlesSplit, reason=Admin, result=Success}], metrics=[{brk_lb_bundles_split_breakdown_total=5}]
-                        dimensions=[{broker=localhost, metric=bundlesSplit, reason=Balanced, result=Skip}], metrics=[{brk_lb_bundles_split_breakdown_total=6}]
-                        dimensions=[{broker=localhost, metric=bundlesSplit, reason=Unknown, result=Failure}], metrics=[{brk_lb_bundles_split_breakdown_total=7}]
+                        dimensions=[{broker=localhost, metric=bundlesSplit, reason=Unknown, result=Failure}], metrics=[{brk_lb_bundles_split_breakdown_total=6}]
                         dimensions=[{broker=localhost, metric=assign, result=Empty}], metrics=[{brk_lb_assign_broker_breakdown_total=2}]
                         dimensions=[{broker=localhost, metric=assign, result=Skip}], metrics=[{brk_lb_assign_broker_breakdown_total=3}]
                         dimensions=[{broker=localhost, metric=assign, result=Success}], metrics=[{brk_lb_assign_broker_breakdown_total=1}]
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/SplitManagerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/SplitManagerTest.java
new file mode 100644
index 0000000000000..3287306ab48ba
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/SplitManagerTest.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.loadbalance.extensions.manager;
+
+import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.VERSION_ID_INIT;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Sessions;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Unknown;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.reflect.FieldUtils;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData;
+import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter;
+import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision;
+import org.apache.pulsar.common.util.FutureUtil;
+import org.testng.annotations.Test;
+
+@Slf4j
+@Test(groups = "broker")
+public class SplitManagerTest {
+    
+    String bundle = "bundle-1";
+
+    String dstBroker = "broker-1";
+
+    @Test
+    public void testEventPubFutureHasException() {
+        var counter = new SplitCounter();
+        SplitManager manager = new SplitManager(counter);
+        var decision = new SplitDecision();
+        CompletableFuture future =
+                manager.waitAsync(FutureUtil.failedFuture(new Exception("test")),
+                        bundle, decision, 10, TimeUnit.SECONDS);
+
+        assertTrue(future.isCompletedExceptionally());
+        try {
+            future.get();
+            fail();
+        } catch (Exception ex) {
+            assertEquals(ex.getCause().getMessage(), "test");
+        }
+        var counterExpected = new SplitCounter();
+        counterExpected.update(SplitDecision.Label.Failure, Unknown);
+        assertEquals(counter.toMetrics(null).toString(),
+                counterExpected.toMetrics(null).toString());
+    }
+
+    @Test
+    public void testTimeout() throws IllegalAccessException {
+        var counter = new SplitCounter();
+        SplitManager manager = new SplitManager(counter);
+        var decision = new SplitDecision();
+        CompletableFuture future =
+                manager.waitAsync(CompletableFuture.completedFuture(null),
+                        bundle, decision, 3, TimeUnit.SECONDS);
+        var inFlightUnloadRequests = getinFlightUnloadRequests(manager);
+
+        assertEquals(inFlightUnloadRequests.size(), 1);
+
+        try {
+            future.get();
+            fail();
+        } catch (Exception ex) {
+            assertTrue(ex.getCause() instanceof TimeoutException);
+        }
+
+        assertEquals(inFlightUnloadRequests.size(), 0);
+        var counterExpected = new SplitCounter();
+        counterExpected.update(SplitDecision.Label.Failure, Unknown);
+        assertEquals(counter.toMetrics(null).toString(),
+                counterExpected.toMetrics(null).toString());
+    }
+
+    @Test
+    public void testSuccess() throws IllegalAccessException, ExecutionException, InterruptedException {
+        var counter = new SplitCounter();
+        SplitManager manager = new SplitManager(counter);
+        var counterExpected = new SplitCounter();
+        var decision = new SplitDecision();
+        decision.succeed(Sessions);
+        CompletableFuture future =
+                manager.waitAsync(CompletableFuture.completedFuture(null),
+                        bundle, decision, 5, TimeUnit.SECONDS);
+        var inFlightUnloadRequests = getinFlightUnloadRequests(manager);
+
+        assertEquals(inFlightUnloadRequests.size(), 1);
+
+        manager.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Assigning, dstBroker, VERSION_ID_INIT), null);
+        assertEquals(inFlightUnloadRequests.size(), 1);
+
+        manager.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Splitting, dstBroker, VERSION_ID_INIT), null);
+        assertEquals(inFlightUnloadRequests.size(), 1);
+
+        manager.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Releasing, dstBroker, VERSION_ID_INIT), null);
+        assertEquals(inFlightUnloadRequests.size(), 1);
+
+        manager.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Free, dstBroker, VERSION_ID_INIT), null);
+        assertEquals(inFlightUnloadRequests.size(), 1);
+        assertEquals(counter.toMetrics(null).toString(),
+                counterExpected.toMetrics(null).toString());
+
+        manager.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Deleted, dstBroker, VERSION_ID_INIT), null);
+        counterExpected.update(SplitDecision.Label.Success, Sessions);
+        assertEquals(inFlightUnloadRequests.size(), 0);
+        assertEquals(counter.toMetrics(null).toString(),
+                counterExpected.toMetrics(null).toString());
+
+        // Success with Init state.
+        future = manager.waitAsync(CompletableFuture.completedFuture(null),
+                bundle, decision, 5, TimeUnit.SECONDS);
+        inFlightUnloadRequests = getinFlightUnloadRequests(manager);
+        assertEquals(inFlightUnloadRequests.size(), 1);
+        manager.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Init, dstBroker, VERSION_ID_INIT), null);
+        assertEquals(inFlightUnloadRequests.size(), 0);
+        counterExpected.update(SplitDecision.Label.Success, Sessions);
+        assertEquals(counter.toMetrics(null).toString(),
+                counterExpected.toMetrics(null).toString());
+        future.get();
+
+        // Success with Owned state.
+        future = manager.waitAsync(CompletableFuture.completedFuture(null),
+                bundle, decision, 5, TimeUnit.SECONDS);
+        inFlightUnloadRequests = getinFlightUnloadRequests(manager);
+        assertEquals(inFlightUnloadRequests.size(), 1);
+        manager.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Owned, dstBroker, VERSION_ID_INIT), null);
+        assertEquals(inFlightUnloadRequests.size(), 0);
+        counterExpected.update(SplitDecision.Label.Success, Sessions);
+        assertEquals(counter.toMetrics(null).toString(),
+                counterExpected.toMetrics(null).toString());
+        future.get();
+    }
+
+    @Test
+    public void testFailedStage() throws IllegalAccessException {
+        var counter = new SplitCounter();
+        SplitManager manager = new SplitManager(counter);
+        var decision = new SplitDecision();
+        CompletableFuture future =
+                manager.waitAsync(CompletableFuture.completedFuture(null),
+                        bundle, decision, 5, TimeUnit.SECONDS);
+        var inFlightUnloadRequests = getinFlightUnloadRequests(manager);
+
+        assertEquals(inFlightUnloadRequests.size(), 1);
+
+        manager.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Owned, dstBroker, VERSION_ID_INIT),
+                new IllegalStateException("Failed stage."));
+
+        try {
+            future.get();
+            fail();
+        } catch (Exception ex) {
+            assertTrue(ex.getCause() instanceof IllegalStateException);
+            assertEquals(ex.getCause().getMessage(), "Failed stage.");
+        }
+
+        assertEquals(inFlightUnloadRequests.size(), 0);
+        var counterExpected = new SplitCounter();
+        counterExpected.update(SplitDecision.Label.Failure, Unknown);
+        assertEquals(counter.toMetrics(null).toString(),
+                counterExpected.toMetrics(null).toString());
+    }
+
+    @Test
+    public void testClose() throws IllegalAccessException {
+        SplitManager manager = new SplitManager(new SplitCounter());
+        var decision = new SplitDecision();
+        CompletableFuture future =
+                manager.waitAsync(CompletableFuture.completedFuture(null),
+                        bundle, decision, 5, TimeUnit.SECONDS);
+        var inFlightUnloadRequests = getinFlightUnloadRequests(manager);
+        assertEquals(inFlightUnloadRequests.size(), 1);
+        manager.close();
+        assertEquals(inFlightUnloadRequests.size(), 0);
+
+        try {
+            future.get();
+            fail();
+        } catch (Exception ex) {
+            assertTrue(ex.getCause() instanceof IllegalStateException);
+        }
+    }
+
+    private Map> getinFlightUnloadRequests(SplitManager manager)
+            throws IllegalAccessException {
+        var inFlightUnloadRequest =
+                (Map>) FieldUtils.readField(manager, "inFlightSplitRequests", true);
+
+        return inFlightUnloadRequest;
+    }
+
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitSchedulerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitSchedulerTest.java
new file mode 100644
index 0000000000000..7988aa413366f
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitSchedulerTest.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.loadbalance.extensions.scheduler;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.testng.Assert.assertEquals;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel;
+import org.apache.pulsar.broker.loadbalance.extensions.manager.SplitManager;
+import org.apache.pulsar.broker.loadbalance.extensions.models.Split;
+import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision;
+import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter;
+import org.apache.pulsar.broker.loadbalance.extensions.strategy.NamespaceBundleSplitStrategy;
+import org.apache.pulsar.common.naming.NamespaceBundleFactory;
+import org.apache.pulsar.common.stats.Metrics;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+@Test(groups = "broker")
+public class SplitSchedulerTest {
+
+    PulsarService pulsar;
+    ServiceConfiguration config;
+    NamespaceBundleFactory namespaceBundleFactory;
+    LoadManagerContext context;
+    ServiceUnitStateChannel channel;
+    NamespaceBundleSplitStrategy strategy;
+    String bundle1 = "tenant/namespace/0x00000000_0xFFFFFFFF";
+    String bundle2 = "tenant/namespace/0x00000000_0x0FFFFFFF";
+    String broker = "broker-1";
+    SplitDecision decision1;
+    SplitDecision decision2;
+
+    @BeforeMethod
+    public void setUp() {
+
+        config = new ServiceConfiguration();
+        config.setLoadBalancerDebugModeEnabled(true);
+
+        pulsar = mock(PulsarService.class);
+        namespaceBundleFactory = mock(NamespaceBundleFactory.class);
+        context = mock(LoadManagerContext.class);
+        channel = mock(ServiceUnitStateChannel.class);
+        strategy = mock(NamespaceBundleSplitStrategy.class);
+
+        doReturn(config).when(pulsar).getConfiguration();
+        doReturn(true).when(namespaceBundleFactory).canSplitBundle(any());
+        doReturn(CompletableFuture.completedFuture(null)).when(channel).publishSplitEventAsync(any());
+
+        decision1 = new SplitDecision();
+        decision1.setSplit(new Split(bundle1, broker, new HashMap<>()));
+        decision1.succeed(SplitDecision.Reason.MsgRate);
+
+        decision2 = new SplitDecision();
+        decision2.setSplit(new Split(bundle2, broker, new HashMap<>()));
+        decision2.succeed(SplitDecision.Reason.Sessions);
+        Set decisions = Set.of(decision1, decision2);
+        doReturn(decisions).when(strategy).findBundlesToSplit(any(), any());
+    }
+
+    @Test(timeOut = 30 * 1000)
+    public void testExecuteSuccess() {
+        AtomicReference> reference = new AtomicReference();
+        SplitCounter counter = new SplitCounter();
+        SplitManager manager = mock(SplitManager.class);
+        SplitScheduler scheduler = new SplitScheduler(pulsar, channel, manager, counter, reference, context, strategy);
+        doAnswer((invocation)->{
+            var decision = invocation.getArgument(2, SplitDecision.class);
+            counter.update(decision);
+            return CompletableFuture.completedFuture(null);
+        }).when(manager).waitAsync(any(), any(), any(), anyLong(), any());
+        scheduler.execute();
+
+        var counterExpected = new SplitCounter();
+        counterExpected.update(decision1);
+        counterExpected.update(decision2);
+        verify(channel, times(1)).publishSplitEventAsync(eq(decision1.getSplit()));
+        verify(channel, times(1)).publishSplitEventAsync(eq(decision2.getSplit()));
+
+        assertEquals(reference.get().toString(), counterExpected.toMetrics(pulsar.getAdvertisedAddress()).toString());
+
+        // Test empty splits.
+        Set emptyUnload = Set.of();
+        doReturn(emptyUnload).when(strategy).findBundlesToSplit(any(), any());
+
+        scheduler.execute();
+        verify(channel, times(2)).publishSplitEventAsync(any());
+        assertEquals(reference.get().toString(), counterExpected.toMetrics(pulsar.getAdvertisedAddress()).toString());
+    }
+
+    @Test(timeOut = 30 * 1000)
+    public void testExecuteFailure() {
+        AtomicReference> reference = new AtomicReference();
+        SplitCounter counter = new SplitCounter();
+        SplitManager manager = new SplitManager(counter);
+        SplitScheduler scheduler = new SplitScheduler(pulsar, channel, manager, counter, reference, context, strategy);
+        doReturn(CompletableFuture.failedFuture(new RuntimeException())).when(channel).publishSplitEventAsync(any());
+
+        scheduler.execute();
+
+
+        var counterExpected = new SplitCounter();
+        counterExpected.update(SplitDecision.Label.Failure, SplitDecision.Reason.Unknown);
+        counterExpected.update(SplitDecision.Label.Failure, SplitDecision.Reason.Unknown);
+        verify(channel, times(1)).publishSplitEventAsync(eq(decision1.getSplit()));
+        verify(channel, times(1)).publishSplitEventAsync(eq(decision2.getSplit()));
+
+        assertEquals(reference.get().toString(), counterExpected.toMetrics(pulsar.getAdvertisedAddress()).toString());
+    }
+
+
+    @Test(timeOut = 30 * 1000)
+    public void testDisableLoadBalancer() {
+
+        config.setLoadBalancerEnabled(false);
+        SplitScheduler scheduler = new SplitScheduler(pulsar, channel, null, null, null, context, strategy);
+
+        scheduler.execute();
+
+        verify(strategy, times(0)).findBundlesToSplit(any(), any());
+
+        config.setLoadBalancerEnabled(true);
+        config.setLoadBalancerAutoBundleSplitEnabled(false);
+        scheduler.execute();
+
+        verify(strategy, times(0)).findBundlesToSplit(any(), any());
+    }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java
new file mode 100644
index 0000000000000..71606bb85a3fe
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.loadbalance.extensions.strategy;
+
+import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Unknown;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.testng.Assert.assertEquals;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.loadbalance.extensions.BrokerRegistry;
+import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
+import org.apache.pulsar.broker.loadbalance.extensions.models.Split;
+import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision;
+import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter;
+import org.apache.pulsar.broker.namespace.NamespaceService;
+import org.apache.pulsar.broker.service.BrokerService;
+import org.apache.pulsar.broker.service.PulsarStats;
+import org.apache.pulsar.common.naming.NamespaceBundleFactory;
+import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+@Test(groups = "broker")
+public class DefaultNamespaceBundleSplitStrategyTest {
+
+    PulsarService pulsar;
+    BrokerService brokerService;
+    PulsarStats pulsarStats;
+    Map bundleStats;
+    ServiceConfiguration config;
+    NamespaceBundleFactory namespaceBundleFactory;
+    NamespaceService namespaceService;
+
+    LoadManagerContext loadManagerContext;
+
+    BrokerRegistry brokerRegistry;
+
+    String bundle1 = "tenant/namespace/0x00000000_0xFFFFFFFF";
+    String bundle2 = "tenant/namespace/0x00000000_0x0FFFFFFF";
+
+    String broker = "broker-1";
+
+    @BeforeMethod
+    void setup() {
+        config = new ServiceConfiguration();
+        config.setLoadBalancerDebugModeEnabled(true);
+        config.setLoadBalancerNamespaceMaximumBundles(100);
+        config.setLoadBalancerNamespaceBundleMaxTopics(100);
+        config.setLoadBalancerNamespaceBundleMaxSessions(100);
+        config.setLoadBalancerNamespaceBundleMaxMsgRate(100);
+        config.setLoadBalancerNamespaceBundleMaxBandwidthMbytes(100);
+        config.setLoadBalancerMaxNumberOfBundlesToSplitPerCycle(1);
+        config.setLoadBalancerNamespaceBundleSplitConditionThreshold(3);
+
+        pulsar = mock(PulsarService.class);
+        brokerService = mock(BrokerService.class);
+        pulsarStats = mock(PulsarStats.class);
+        namespaceService = mock(NamespaceService.class);
+        namespaceBundleFactory = mock(NamespaceBundleFactory.class);
+        loadManagerContext = mock(LoadManagerContext.class);
+        brokerRegistry = mock(BrokerRegistry.class);
+
+
+
+        doReturn(brokerService).when(pulsar).getBrokerService();
+        doReturn(config).when(pulsar).getConfiguration();
+        doReturn(pulsarStats).when(brokerService).getPulsarStats();
+        doReturn(namespaceService).when(pulsar).getNamespaceService();
+        doReturn(namespaceBundleFactory).when(namespaceService).getNamespaceBundleFactory();
+        doReturn(true).when(namespaceBundleFactory).canSplitBundle(any());
+        doReturn(brokerRegistry).when(loadManagerContext).brokerRegistry();
+        doReturn(broker).when(brokerRegistry).getBrokerId();
+
+
+        bundleStats = new LinkedHashMap<>();
+        NamespaceBundleStats stats1 = new NamespaceBundleStats();
+        stats1.topics = 5;
+        bundleStats.put(bundle1, stats1);
+        NamespaceBundleStats stats2 = new NamespaceBundleStats();
+        stats2.topics = 5;
+        bundleStats.put(bundle2, stats2);
+        doReturn(bundleStats).when(brokerService).getBundleStats();
+    }
+
+    public void testNamespaceBundleSplitConditionThreshold() {
+        config.setLoadBalancerNamespaceBundleSplitConditionThreshold(0);
+        bundleStats.values().forEach(v -> v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() + 1);
+        var strategy = new DefaultNamespaceBundleSplitStrategyImpl(new SplitCounter());
+        var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
+        assertEquals(actual.size(), 1);
+    }
+
+
+    public void testNotEnoughTopics() {
+        config.setLoadBalancerNamespaceBundleSplitConditionThreshold(0);
+        bundleStats.values().forEach(v -> v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() + 1);
+        var strategy = new DefaultNamespaceBundleSplitStrategyImpl(new SplitCounter());
+        bundleStats.values().forEach(v -> v.topics = 1);
+        var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
+        var expected = Set.of();
+        assertEquals(actual, expected);
+    }
+
+    public void testNamespaceMaximumBundles() throws Exception {
+        config.setLoadBalancerNamespaceBundleSplitConditionThreshold(0);
+        bundleStats.values().forEach(v -> v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() + 1);
+        var strategy = new DefaultNamespaceBundleSplitStrategyImpl(new SplitCounter());
+        doReturn(config.getLoadBalancerNamespaceMaximumBundles()).when(namespaceService).getBundleCount(any());
+        var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
+        var expected = Set.of();
+        assertEquals(actual, expected);
+    }
+
+    public void testEmptyBundleStats() {
+        config.setLoadBalancerNamespaceBundleSplitConditionThreshold(0);
+        bundleStats.values().forEach(v -> v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() + 1);
+        var strategy = new DefaultNamespaceBundleSplitStrategyImpl(new SplitCounter());
+        bundleStats.clear();
+        var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
+        var expected = Set.of();
+        assertEquals(actual, expected);
+    }
+
+    public void testError() throws Exception {
+        var counter = spy(new SplitCounter());
+        config.setLoadBalancerNamespaceBundleSplitConditionThreshold(0);
+        bundleStats.values().forEach(v -> v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() + 1);
+        var strategy = new DefaultNamespaceBundleSplitStrategyImpl(counter);
+        doThrow(new RuntimeException()).when(namespaceService).getBundleCount(any());
+        var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
+        var expected = Set.of();
+        assertEquals(actual, expected);
+        verify(counter, times(2)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
+    }
+
+    public void testMaxMsgRate() {
+        var counter = spy(new SplitCounter());
+        var strategy = new DefaultNamespaceBundleSplitStrategyImpl(counter);
+        int threshold = config.getLoadBalancerNamespaceBundleSplitConditionThreshold();
+        bundleStats.values().forEach(v -> {
+            v.msgRateOut = config.getLoadBalancerNamespaceBundleMaxMsgRate() / 2 + 1;
+            v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() / 2 + 1;
+        });
+        for (int i = 0; i < threshold + 2; i++) {
+            var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
+            if (i == threshold) {
+                SplitDecision decision1 = new SplitDecision();
+                decision1.setSplit(new Split(bundle1, broker, new HashMap<>()));
+                decision1.succeed(SplitDecision.Reason.MsgRate);
+
+                assertEquals(actual, Set.of(decision1));
+                verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
+            } else if (i == threshold + 1) {
+                SplitDecision decision1 = new SplitDecision();
+                decision1.setSplit(new Split(bundle2, broker, new HashMap<>()));
+                decision1.succeed(SplitDecision.Reason.MsgRate);
+
+                assertEquals(actual, Set.of(decision1));
+                verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
+            } else {
+                assertEquals(actual, Set.of());
+                verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
+            }
+        }
+    }
+
+    public void testMaxTopics() {
+        var counter = spy(new SplitCounter());
+        var strategy = new DefaultNamespaceBundleSplitStrategyImpl(counter);
+        int threshold = config.getLoadBalancerNamespaceBundleSplitConditionThreshold();
+        bundleStats.values().forEach(v -> v.topics = config.getLoadBalancerNamespaceBundleMaxTopics() + 1);
+        for (int i = 0; i < threshold + 2; i++) {
+            var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
+            if (i == threshold) {
+                SplitDecision decision1 = new SplitDecision();
+                decision1.setSplit(new Split(bundle1, broker, new HashMap<>()));
+                decision1.succeed(SplitDecision.Reason.Topics);
+
+                assertEquals(actual, Set.of(decision1));
+                verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
+            } else if (i == threshold + 1) {
+                SplitDecision decision1 = new SplitDecision();
+                decision1.setSplit(new Split(bundle2, broker, new HashMap<>()));
+                decision1.succeed(SplitDecision.Reason.Topics);
+
+                assertEquals(actual, Set.of(decision1));
+                verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
+            } else {
+                assertEquals(actual, Set.of());
+                verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
+            }
+        }
+    }
+
+    public void testMaxSessions() {
+        var counter = spy(new SplitCounter());
+        var strategy = new DefaultNamespaceBundleSplitStrategyImpl(counter);
+        int threshold = config.getLoadBalancerNamespaceBundleSplitConditionThreshold();
+        bundleStats.values().forEach(v -> {
+            v.producerCount = config.getLoadBalancerNamespaceBundleMaxSessions() / 2 + 1;
+            v.consumerCount = config.getLoadBalancerNamespaceBundleMaxSessions() / 2 + 1;
+        });
+        for (int i = 0; i < threshold + 2; i++) {
+            var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
+            if (i == threshold) {
+                SplitDecision decision1 = new SplitDecision();
+                decision1.setSplit(new Split(bundle1, broker, new HashMap<>()));
+                decision1.succeed(SplitDecision.Reason.Sessions);
+
+                assertEquals(actual, Set.of(decision1));
+                verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
+            } else if (i == threshold + 1) {
+                SplitDecision decision1 = new SplitDecision();
+                decision1.setSplit(new Split(bundle2, broker, new HashMap<>()));
+                decision1.succeed(SplitDecision.Reason.Sessions);
+
+                assertEquals(actual, Set.of(decision1));
+                verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
+            } else {
+                assertEquals(actual, Set.of());
+                verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
+            }
+        }
+    }
+
+    public void testMaxBandwidthMbytes() {
+        var counter = spy(new SplitCounter());
+        var strategy = new DefaultNamespaceBundleSplitStrategyImpl(counter);
+        int threshold = config.getLoadBalancerNamespaceBundleSplitConditionThreshold();
+        bundleStats.values().forEach(v -> {
+            v.msgThroughputOut = config.getLoadBalancerNamespaceBundleMaxBandwidthMbytes() * 1024 * 1024 / 2 + 1;
+            v.msgThroughputIn = config.getLoadBalancerNamespaceBundleMaxBandwidthMbytes() * 1024 * 1024 / 2 + 1;
+        });
+        for (int i = 0; i < threshold + 2; i++) {
+            var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
+            if (i == threshold) {
+                SplitDecision decision1 = new SplitDecision();
+                decision1.setSplit(new Split(bundle1, broker, new HashMap<>()));
+                decision1.succeed(SplitDecision.Reason.Bandwidth);
+
+                assertEquals(actual, Set.of(decision1));
+                verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
+            } else if (i == threshold + 1) {
+                SplitDecision decision1 = new SplitDecision();
+                decision1.setSplit(new Split(bundle2, broker, new HashMap<>()));
+                decision1.succeed(SplitDecision.Reason.Bandwidth);
+
+                assertEquals(actual, Set.of(decision1));
+                verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
+            } else {
+                assertEquals(actual, Set.of());
+                verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
+            }
+        }
+    }
+
+}

From 2ddfbfe8ce11f3ff66df53a164570335e573bef4 Mon Sep 17 00:00:00 2001
From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com>
Date: Tue, 14 Mar 2023 00:42:47 -0700
Subject: [PATCH 107/174] [improve][broker] PIP-192 Supports
 AntiAffinityGroupPolicy (#19708)

Master Issue: https://github.com/apache/pulsar/issues/16691

### Motivation

Raising a PR to implement https://github.com/apache/pulsar/issues/16691.

We need to support AntiAffinityGroupPolicy in Load Manager Extension as well.

### Modifications
This PR
- added AntiAffinityGroupPolicyFilter used in the broker assignment logic.
- added AntiAffinityGroupPolicyHelper that is passed to TransferShedder and AntiAffinityGroupPolicyFilter
- modified LoadManagerShared.filterAntiAffinityGroupOwnedBrokers and LoadManagerShared.getAntiAffinityNamespaceOwnedBrokers to accept the bundle ownership data from the Load Manager Extension.
- moved ModularLoadManagerImpl.refreshBrokerToFailureDomainMap(..) to LoadManager.refreshBrokerToFailureDomainMap(..) to reuse it in ExtensibleLoadManagerImpl.
- modified AntiAffinityNamespaceGroupTest to reuse the test cases in AntiAffinityNamespaceGroupExtensionTest
---
 .../extensions/ExtensibleLoadManagerImpl.java |  12 +
 .../channel/ServiceUnitStateChannel.java      |   8 +
 .../channel/ServiceUnitStateChannelImpl.java  |   6 +
 .../filter/AntiAffinityGroupPolicyFilter.java |  58 ++++
 .../AntiAffinityGroupPolicyHelper.java        |  97 ++++++
 .../extensions/scheduler/TransferShedder.java |  28 +-
 .../loadbalance/impl/LoadManagerShared.java   | 179 +++++++++-
 .../impl/ModularLoadManagerImpl.java          |  43 +--
 .../AntiAffinityNamespaceGroupTest.java       | 310 ++++++++++--------
 ...tiAffinityNamespaceGroupExtensionTest.java | 196 +++++++++++
 .../scheduler/TransferShedderTest.java        | 216 ++++++------
 11 files changed, 841 insertions(+), 312 deletions(-)
 create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java
 create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java
 create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
index c8ac8e4684506..7895f06fd5088 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
@@ -41,6 +41,7 @@
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData;
+import org.apache.pulsar.broker.loadbalance.extensions.filter.AntiAffinityGroupPolicyFilter;
 import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerFilter;
 import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerIsolationPoliciesFilter;
 import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerMaxTopicCountFilter;
@@ -52,6 +53,7 @@
 import org.apache.pulsar.broker.loadbalance.extensions.models.Unload;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision;
+import org.apache.pulsar.broker.loadbalance.extensions.policies.AntiAffinityGroupPolicyHelper;
 import org.apache.pulsar.broker.loadbalance.extensions.reporter.BrokerLoadDataReporter;
 import org.apache.pulsar.broker.loadbalance.extensions.reporter.TopBundleLoadDataReporter;
 import org.apache.pulsar.broker.loadbalance.extensions.scheduler.LoadManagerScheduler;
@@ -91,6 +93,10 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
 
     private ServiceUnitStateChannel serviceUnitStateChannel;
 
+    private AntiAffinityGroupPolicyFilter antiAffinityGroupPolicyFilter;
+
+    private AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper;
+
     private LoadDataStore brokerLoadDataStore;
     private LoadDataStore topBundlesLoadDataStore;
 
@@ -137,6 +143,7 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
                     CompletableFuture>>newBuilder()
             .build();
 
+
     /**
      * Life cycle: Constructor -> initialize -> start -> close.
      */
@@ -173,6 +180,11 @@ public void start() throws PulsarServerException {
         this.serviceUnitStateChannel.listen(unloadManager);
         this.serviceUnitStateChannel.listen(splitManager);
         this.serviceUnitStateChannel.start();
+        this.antiAffinityGroupPolicyHelper =
+                new AntiAffinityGroupPolicyHelper(pulsar, serviceUnitStateChannel);
+        antiAffinityGroupPolicyHelper.listenFailureDomainUpdate();
+        this.antiAffinityGroupPolicyFilter = new AntiAffinityGroupPolicyFilter(antiAffinityGroupPolicyHelper);
+        this.brokerFilterPipeline.add(antiAffinityGroupPolicyFilter);
 
         try {
             this.brokerLoadDataStore = LoadDataStoreFactory
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java
index dc4d582ddb081..4e92ad791ab63 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java
@@ -20,7 +20,9 @@
 
 import java.io.Closeable;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import org.apache.pulsar.broker.PulsarServerException;
 import org.apache.pulsar.broker.loadbalance.extensions.manager.StateChangeListener;
@@ -164,4 +166,10 @@ public interface ServiceUnitStateChannel extends Closeable {
      */
     void listen(StateChangeListener listener);
 
+    /**
+     * Returns service unit ownership entry set.
+     * @return a set of service unit ownership entries
+     */
+    Set> getOwnershipEntrySet();
+
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
index 5f24e41dda931..f8686c07f05fa 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
@@ -1317,5 +1317,11 @@ public List getMetrics() {
     @Override
     public void listen(StateChangeListener listener) {
         this.stateChangeListeners.addListener(listener);
+
+    }
+
+    @Override
+    public Set> getOwnershipEntrySet() {
+        return tableview.entrySet();
     }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java
new file mode 100644
index 0000000000000..358f985f83e12
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.loadbalance.extensions.filter;
+
+import java.util.Map;
+import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
+import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
+import org.apache.pulsar.broker.loadbalance.extensions.policies.AntiAffinityGroupPolicyHelper;
+import org.apache.pulsar.common.naming.ServiceUnitId;
+
+/**
+ * Filter by anti-affinity-group-policy.
+ */
+public class AntiAffinityGroupPolicyFilter implements BrokerFilter {
+
+    public static final String FILTER_NAME = "broker_anti_affinity_group_filter";
+
+    private final AntiAffinityGroupPolicyHelper helper;
+
+    public AntiAffinityGroupPolicyFilter(AntiAffinityGroupPolicyHelper helper) {
+        this.helper = helper;
+    }
+
+    @Override
+    public Map filter(
+            Map brokers, ServiceUnitId serviceUnitId, LoadManagerContext context) {
+        helper.filter(brokers, serviceUnitId.toString());
+        return brokers;
+    }
+
+
+    @Override
+    public String name() {
+        return FILTER_NAME;
+    }
+
+    @Override
+    public void initialize(PulsarService pulsar) {
+        return;
+    }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java
new file mode 100644
index 0000000000000..28acf5fba0ea1
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.loadbalance.extensions.policies;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel;
+import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
+import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared;
+import org.apache.pulsar.metadata.api.MetadataStoreException;
+
+@Slf4j
+public class AntiAffinityGroupPolicyHelper {
+    PulsarService pulsar;
+    Map brokerToFailureDomainMap;
+    ServiceUnitStateChannel channel;
+
+    public AntiAffinityGroupPolicyHelper(PulsarService pulsar,
+                                  ServiceUnitStateChannel channel){
+
+        this.pulsar = pulsar;
+        this.brokerToFailureDomainMap = new HashMap<>();
+        this.channel = channel;
+    }
+
+    public void filter(
+            Map brokers, String bundle) {
+        LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar, bundle,
+                brokers.keySet(),
+                channel.getOwnershipEntrySet(), brokerToFailureDomainMap);
+    }
+
+    public boolean canUnload(
+            Map brokers,
+            String bundle,
+            String srcBroker,
+            Optional dstBroker) {
+
+
+
+        try {
+            var antiAffinityGroupOptional = LoadManagerShared.getNamespaceAntiAffinityGroup(
+                    pulsar, LoadManagerShared.getNamespaceNameFromBundleName(bundle));
+            if (antiAffinityGroupOptional.isPresent()) {
+
+                // copy to retain the input brokers
+                Map candidates = new HashMap<>(brokers);
+
+                filter(candidates, bundle);
+
+                candidates.remove(srcBroker);
+
+                // unload case
+                if (dstBroker.isEmpty()) {
+                    return !candidates.isEmpty();
+                }
+
+                // transfer case
+                return candidates.containsKey(dstBroker.get());
+            }
+        } catch (MetadataStoreException e) {
+            log.error("Failed to check unload candidates. Assumes that bundle:{} cannot unload ", bundle, e);
+            return false;
+        }
+
+        return true;
+    }
+
+    public void listenFailureDomainUpdate() {
+        LoadManagerShared.refreshBrokerToFailureDomainMap(pulsar, brokerToFailureDomainMap);
+        // register listeners for domain changes
+        pulsar.getPulsarResources().getClusterResources().getFailureDomainResources()
+                .registerListener(__ -> {
+                    pulsar.getLoadManagerExecutor().execute(() ->
+                            LoadManagerShared.refreshBrokerToFailureDomainMap(pulsar, brokerToFailureDomainMap));
+                });
+    }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java
index 810fda320af68..9f9582df2cc28 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java
@@ -32,7 +32,6 @@
 import java.util.concurrent.TimeoutException;
 import lombok.Getter;
 import lombok.experimental.Accessors;
-import org.apache.commons.lang3.StringUtils;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
@@ -41,13 +40,12 @@
 import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData;
 import org.apache.pulsar.broker.loadbalance.extensions.models.Unload;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision;
+import org.apache.pulsar.broker.loadbalance.extensions.policies.AntiAffinityGroupPolicyHelper;
 import org.apache.pulsar.broker.loadbalance.extensions.policies.IsolationPoliciesHelper;
 import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore;
 import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared;
 import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies;
 import org.apache.pulsar.common.naming.NamespaceBundle;
-import org.apache.pulsar.common.naming.NamespaceName;
-import org.apache.pulsar.metadata.api.MetadataStoreException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -80,6 +78,7 @@ public class TransferShedder implements NamespaceUnloadStrategy {
     private final PulsarService pulsar;
     private final SimpleResourceAllocationPolicies allocationPolicies;
     private final IsolationPoliciesHelper isolationPoliciesHelper;
+    private final AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper;
 
     private final UnloadDecision decision = new UnloadDecision();
 
@@ -88,12 +87,14 @@ public TransferShedder(){
         this.pulsar = null;
         this.allocationPolicies = null;
         this.isolationPoliciesHelper = null;
+        this.antiAffinityGroupPolicyHelper = null;
     }
 
-    public TransferShedder(PulsarService pulsar){
+    public TransferShedder(PulsarService pulsar, AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper) {
         this.pulsar = pulsar;
         this.allocationPolicies = new SimpleResourceAllocationPolicies(pulsar);
         this.isolationPoliciesHelper = new IsolationPoliciesHelper(allocationPolicies);
+        this.antiAffinityGroupPolicyHelper = antiAffinityGroupPolicyHelper;
     }
 
 
@@ -438,32 +439,25 @@ private boolean hasMsgThroughput(LoadManagerContext context, String broker) {
     private boolean isTransferable(LoadManagerContext context,
                                    Map availableBrokers,
                                    String bundle,
-                                   String maxBroker,
-                                   Optional broker) {
+                                   String srcBroker,
+                                   Optional dstBroker) {
         if (pulsar == null || allocationPolicies == null) {
             return true;
         }
         String namespace = LoadManagerShared.getNamespaceNameFromBundleName(bundle);
-        NamespaceName namespaceName = NamespaceName.get(namespace);
         final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle);
         NamespaceBundle namespaceBundle =
                 pulsar.getNamespaceService().getNamespaceBundleFactory().getBundle(namespace, bundleRange);
 
-        if (!canTransferWithIsolationPoliciesToBroker(context, availableBrokers, namespaceBundle, maxBroker, broker)) {
+        if (!canTransferWithIsolationPoliciesToBroker(
+                context, availableBrokers, namespaceBundle, srcBroker, dstBroker)) {
             return false;
         }
 
-        try {
-            var localPoliciesOptional = pulsar
-                    .getPulsarResources().getLocalPolicies().getLocalPolicies(namespaceName);
-            if (localPoliciesOptional.isPresent() && StringUtils.isNotBlank(
-                    localPoliciesOptional.get().namespaceAntiAffinityGroup)) {
-                return false;
-            }
-        } catch (MetadataStoreException e) {
-            log.error("Failed to get localPolicies. Assumes that bundle:{} is not transferable.", bundle, e);
+        if (!antiAffinityGroupPolicyHelper.canUnload(availableBrokers, bundle, srcBroker, dstBroker)) {
             return false;
         }
+
         return true;
     }
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java
index 2c90b8a4047a2..6818ae03b5280 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java
@@ -29,6 +29,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
@@ -38,13 +39,18 @@
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.loadbalance.BrokerHostUsage;
 import org.apache.pulsar.broker.loadbalance.LoadData;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData;
+import org.apache.pulsar.broker.resources.ClusterResources;
 import org.apache.pulsar.common.naming.NamespaceBundle;
 import org.apache.pulsar.common.naming.NamespaceName;
 import org.apache.pulsar.common.naming.ServiceUnitId;
+import org.apache.pulsar.common.policies.data.FailureDomainImpl;
 import org.apache.pulsar.common.util.DirectMemoryUtils;
 import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap;
 import org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet;
+import org.apache.pulsar.metadata.api.MetadataStoreException;
 import org.apache.pulsar.policies.data.loadbalancer.BrokerData;
 import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage;
 import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage;
@@ -329,6 +335,18 @@ public static void filterAntiAffinityGroupOwnedBrokers(
         try {
             final Map brokerToAntiAffinityNamespaceCount = getAntiAffinityNamespaceOwnedBrokers(pulsar,
                     namespaceName, brokerToNamespaceToBundleRange).get(30, TimeUnit.SECONDS);
+            filterAntiAffinityGroupOwnedBrokers(pulsar, candidates, brokerToDomainMap,
+                    brokerToAntiAffinityNamespaceCount);
+        } catch (Exception e) {
+            LOG.error("Failed to filter anti-affinity group namespace {}", e.getMessage());
+        }
+    }
+
+    private static void filterAntiAffinityGroupOwnedBrokers(
+            final PulsarService pulsar,
+            final Set candidates,
+            Map brokerToDomainMap,
+            Map brokerToAntiAffinityNamespaceCount) {
             if (brokerToAntiAffinityNamespaceCount == null) {
                 // none of the broker owns anti-affinity-namespace so, none of the broker will be filtered
                 return;
@@ -368,6 +386,23 @@ public static void filterAntiAffinityGroupOwnedBrokers(
                 candidates
                         .removeIf(broker -> brokerToAntiAffinityNamespaceCount.get(broker) != finalLeastNamespaceCount);
             }
+    }
+
+    public static void filterAntiAffinityGroupOwnedBrokers(
+            final PulsarService pulsar, final String assignedBundleName,
+            final Set candidates,
+            Set> bundleOwnershipData,
+            Map brokerToDomainMap) {
+        if (candidates.isEmpty()) {
+            return;
+        }
+        final String namespaceName = getNamespaceNameFromBundleName(assignedBundleName);
+        try {
+            final Map brokerToAntiAffinityNamespaceCount = getAntiAffinityNamespaceOwnedBrokers(
+                    pulsar, namespaceName, bundleOwnershipData)
+                    .get(30, TimeUnit.SECONDS);
+            filterAntiAffinityGroupOwnedBrokers(pulsar, candidates, brokerToDomainMap,
+                    brokerToAntiAffinityNamespaceCount);
         } catch (Exception e) {
             LOG.error("Failed to filter anti-affinity group namespace {}", e.getMessage());
         }
@@ -424,14 +459,13 @@ public static CompletableFuture> getAntiAffinityNamespaceOw
                     brokerToNamespaceToBundleRange) {
 
         CompletableFuture> antiAffinityNsBrokersResult = new CompletableFuture<>();
-
-        pulsar.getPulsarResources().getLocalPolicies().getLocalPoliciesAsync(NamespaceName.get(namespaceName))
-                .thenAccept(policies -> {
-            if (!policies.isPresent() || StringUtils.isBlank(policies.get().namespaceAntiAffinityGroup)) {
+        getNamespaceAntiAffinityGroupAsync(pulsar, namespaceName)
+                .thenAccept(antiAffinityGroupOptional -> {
+            if (antiAffinityGroupOptional.isEmpty()) {
                 antiAffinityNsBrokersResult.complete(null);
                 return;
             }
-            final String antiAffinityGroup = policies.get().namespaceAntiAffinityGroup;
+            final String antiAffinityGroup = antiAffinityGroupOptional.get();
             final Map brokerToAntiAffinityNamespaceCount = new ConcurrentHashMap<>();
             final List> futures = new ArrayList<>();
             brokerToNamespaceToBundleRange.forEach((broker, nsToBundleRange) -> {
@@ -442,11 +476,27 @@ public static CompletableFuture> getAntiAffinityNamespaceOw
 
                     CompletableFuture future = new CompletableFuture<>();
                     futures.add(future);
+                    countAntiAffinityNamespaceOwnedBrokers(broker, ns, future,
+                            pulsar, antiAffinityGroup, brokerToAntiAffinityNamespaceCount);
+                });
+            });
+            FutureUtil.waitForAll(futures)
+                    .thenAccept(r -> antiAffinityNsBrokersResult.complete(brokerToAntiAffinityNamespaceCount));
+        }).exceptionally(ex -> {
+            // namespace-policies has not been created yet
+            antiAffinityNsBrokersResult.complete(null);
+            return null;
+        });
+        return antiAffinityNsBrokersResult;
+    }
 
-                    pulsar.getPulsarResources().getLocalPolicies().getLocalPoliciesAsync(NamespaceName.get(ns))
-                            .thenAccept(nsPolicies -> {
-                        if (nsPolicies.isPresent()
-                                && antiAffinityGroup.equalsIgnoreCase(nsPolicies.get().namespaceAntiAffinityGroup)) {
+    private static void countAntiAffinityNamespaceOwnedBrokers(
+            String broker, String ns, CompletableFuture future, PulsarService pulsar,
+            String targetAntiAffinityGroup, Map brokerToAntiAffinityNamespaceCount) {
+                    getNamespaceAntiAffinityGroupAsync(pulsar, ns)
+                            .thenAccept(antiAffinityGroupOptional -> {
+                        if (antiAffinityGroupOptional.isPresent()
+                                && targetAntiAffinityGroup.equalsIgnoreCase(antiAffinityGroupOptional.get())) {
                             brokerToAntiAffinityNamespaceCount.compute(broker,
                                     (brokerName, count) -> count == null ? 1 : count + 1);
                         }
@@ -455,8 +505,39 @@ public static CompletableFuture> getAntiAffinityNamespaceOw
                         future.complete(null);
                         return null;
                     });
-                });
-            });
+    }
+
+    public static CompletableFuture> getAntiAffinityNamespaceOwnedBrokers(
+            final PulsarService pulsar, final String namespaceName,
+            Set> bundleOwnershipData) {
+
+        CompletableFuture> antiAffinityNsBrokersResult = new CompletableFuture<>();
+        getNamespaceAntiAffinityGroupAsync(pulsar, namespaceName)
+                .thenAccept(antiAffinityGroupOptional -> {
+            if (antiAffinityGroupOptional.isEmpty()) {
+                antiAffinityNsBrokersResult.complete(null);
+                return;
+            }
+            final String antiAffinityGroup = antiAffinityGroupOptional.get();
+            final Map brokerToAntiAffinityNamespaceCount = new ConcurrentHashMap<>();
+            final List> futures = new ArrayList<>();
+
+            bundleOwnershipData
+                    .forEach(etr -> {
+                        var stateData = etr.getValue();
+                        var bundle = etr.getKey();
+                        if (stateData.state() == ServiceUnitState.Owned
+                                && StringUtils.isNotBlank(stateData.dstBroker())) {
+                            CompletableFuture future = new CompletableFuture<>();
+                            futures.add(future);
+                            countAntiAffinityNamespaceOwnedBrokers
+                                    (stateData.dstBroker(),
+                                            LoadManagerShared.getNamespaceNameFromBundleName(bundle),
+                                            future, pulsar,
+                                            antiAffinityGroup, brokerToAntiAffinityNamespaceCount);
+                        }
+                    });
+
             FutureUtil.waitForAll(futures)
                     .thenAccept(r -> antiAffinityNsBrokersResult.complete(brokerToAntiAffinityNamespaceCount));
         }).exceptionally(ex -> {
@@ -467,6 +548,31 @@ public static CompletableFuture> getAntiAffinityNamespaceOw
         return antiAffinityNsBrokersResult;
     }
 
+    public static CompletableFuture> getNamespaceAntiAffinityGroupAsync(
+            PulsarService pulsar, String namespaceName) {
+        return pulsar.getPulsarResources().getLocalPolicies().getLocalPoliciesAsync(NamespaceName.get(namespaceName))
+                .thenApply(localPoliciesOptional -> {
+                    if (localPoliciesOptional.isPresent()
+                            && StringUtils.isNotBlank(localPoliciesOptional.get().namespaceAntiAffinityGroup)) {
+                        return Optional.of(localPoliciesOptional.get().namespaceAntiAffinityGroup);
+                    }
+                    return Optional.empty();
+                });
+    }
+
+
+    public static Optional getNamespaceAntiAffinityGroup(
+            PulsarService pulsar, String namespaceName) throws MetadataStoreException {
+        var localPoliciesOptional =
+                pulsar.getPulsarResources().getLocalPolicies().getLocalPolicies(NamespaceName.get(namespaceName));
+        if (localPoliciesOptional.isPresent()
+                && StringUtils.isNotBlank(localPoliciesOptional.get().namespaceAntiAffinityGroup)) {
+            return Optional.of(localPoliciesOptional.get().namespaceAntiAffinityGroup);
+        }
+        return Optional.empty();
+    }
+
+
     /**
      *
      * It checks if given anti-affinity namespace should be unloaded by broker due to load-shedding. If all the brokers
@@ -492,6 +598,13 @@ public static boolean shouldAntiAffinityNamespaceUnload(
 
         Map brokerNamespaceCount = getAntiAffinityNamespaceOwnedBrokers(pulsar, namespace,
                 brokerToNamespaceToBundleRange).get(10, TimeUnit.SECONDS);
+        return shouldAntiAffinityNamespaceUnload(currentBroker, candidateBrokers, brokerNamespaceCount);
+    }
+
+    private static boolean shouldAntiAffinityNamespaceUnload(
+            String currentBroker,
+            Set candidateBrokers,
+            Map brokerNamespaceCount) {
         if (brokerNamespaceCount != null && !brokerNamespaceCount.isEmpty()) {
             int leastNsCount = Integer.MAX_VALUE;
             int currentBrokerNsCount = 0;
@@ -522,6 +635,18 @@ public static boolean shouldAntiAffinityNamespaceUnload(
         return true;
     }
 
+    public static boolean shouldAntiAffinityNamespaceUnload(
+            String namespace, String bundle, String currentBroker,
+            final PulsarService pulsar,
+            Set> bundleOwnershipData,
+            Set candidateBrokers) throws Exception {
+
+        Map brokerNamespaceCount = getAntiAffinityNamespaceOwnedBrokers(
+                pulsar, namespace, bundleOwnershipData)
+                .get(10, TimeUnit.SECONDS);
+        return shouldAntiAffinityNamespaceUnload(currentBroker, candidateBrokers, brokerNamespaceCount);
+    }
+
     public interface BrokerTopicLoadingPredicate {
         boolean isEnablePersistentTopics(String brokerUrl);
 
@@ -554,4 +679,36 @@ public static void filterBrokersWithLargeTopicCount(Set brokerCandidateC
             brokerCandidateCache.addAll(filteredBrokerCandidates);
         }
     }
+
+    public static void refreshBrokerToFailureDomainMap(PulsarService pulsar,
+                                                       Map brokerToFailureDomainMap) {
+        if (!pulsar.getConfiguration().isFailureDomainsEnabled()) {
+            return;
+        }
+        ClusterResources.FailureDomainResources fdr =
+                pulsar.getPulsarResources().getClusterResources().getFailureDomainResources();
+        String clusterName = pulsar.getConfiguration().getClusterName();
+        try {
+            synchronized (brokerToFailureDomainMap) {
+                Map tempBrokerToFailureDomainMap = new HashMap<>();
+                for (String domainName : fdr.listFailureDomains(clusterName)) {
+                    try {
+                        Optional domain = fdr.getFailureDomain(clusterName, domainName);
+                        if (domain.isPresent()) {
+                            for (String broker : domain.get().brokers) {
+                                tempBrokerToFailureDomainMap.put(broker, domainName);
+                            }
+                        }
+                    } catch (Exception e) {
+                        LOG.warn("Failed to get domain {}", domainName, e);
+                    }
+                }
+                brokerToFailureDomainMap.clear();
+                brokerToFailureDomainMap.putAll(tempBrokerToFailureDomainMap);
+            }
+            LOG.info("Cluster domain refreshed {}", brokerToFailureDomainMap);
+        } catch (Exception e) {
+            LOG.warn("Failed to get domain-list for cluster {}", e.getMessage());
+        }
+    }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java
index 59f9836b30975..f135840d60e59 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java
@@ -57,7 +57,6 @@
 import org.apache.pulsar.broker.loadbalance.ModularLoadManager;
 import org.apache.pulsar.broker.loadbalance.ModularLoadManagerStrategy;
 import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared.BrokerTopicLoadingPredicate;
-import org.apache.pulsar.broker.resources.ClusterResources;
 import org.apache.pulsar.broker.stats.prometheus.metrics.Summary;
 import org.apache.pulsar.client.admin.PulsarAdminException;
 import org.apache.pulsar.client.util.ExecutorProvider;
@@ -65,8 +64,6 @@
 import org.apache.pulsar.common.naming.NamespaceBundleFactory;
 import org.apache.pulsar.common.naming.NamespaceName;
 import org.apache.pulsar.common.naming.ServiceUnitId;
-import org.apache.pulsar.common.policies.data.FailureDomainImpl;
-import org.apache.pulsar.common.policies.data.LocalPolicies;
 import org.apache.pulsar.common.policies.data.ResourceQuota;
 import org.apache.pulsar.common.stats.Metrics;
 import org.apache.pulsar.common.util.FutureUtil;
@@ -272,11 +269,12 @@ public void initialize(final PulsarService pulsar) {
         policies = new SimpleResourceAllocationPolicies(pulsar);
         filterPipeline.add(new BrokerVersionFilter());
 
-        refreshBrokerToFailureDomainMap();
+        LoadManagerShared.refreshBrokerToFailureDomainMap(pulsar, brokerToFailureDomainMap);
         // register listeners for domain changes
         pulsar.getPulsarResources().getClusterResources().getFailureDomainResources()
                 .registerListener(__ -> {
-                    executors.execute(() -> refreshBrokerToFailureDomainMap());
+                    executors.execute(
+                            () -> LoadManagerShared.refreshBrokerToFailureDomainMap(pulsar, brokerToFailureDomainMap));
                 });
 
         loadSheddingPipeline.add(createLoadSheddingStrategy());
@@ -713,9 +711,8 @@ public boolean shouldNamespacePoliciesUnload(String namespace, String bundle, St
 
     public boolean shouldAntiAffinityNamespaceUnload(String namespace, String bundle, String currentBroker) {
         try {
-            Optional nsPolicies = pulsar.getPulsarResources().getLocalPolicies()
-                    .getLocalPolicies(NamespaceName.get(namespace));
-            if (!nsPolicies.isPresent() || StringUtils.isBlank(nsPolicies.get().namespaceAntiAffinityGroup)) {
+            var antiAffinityGroupOptional = LoadManagerShared.getNamespaceAntiAffinityGroup(pulsar, namespace);
+            if (antiAffinityGroupOptional.isEmpty()) {
                 return true;
             }
 
@@ -1159,36 +1156,6 @@ private void deleteTimeAverageDataFromMetadataStoreAsync(String broker) {
         });
     }
 
-    private void refreshBrokerToFailureDomainMap() {
-        if (!pulsar.getConfiguration().isFailureDomainsEnabled()) {
-            return;
-        }
-        ClusterResources.FailureDomainResources fdr =
-                pulsar.getPulsarResources().getClusterResources().getFailureDomainResources();
-        String clusterName = pulsar.getConfiguration().getClusterName();
-        try {
-            synchronized (brokerToFailureDomainMap) {
-                Map tempBrokerToFailureDomainMap = new HashMap<>();
-                for (String domainName : fdr.listFailureDomains(clusterName)) {
-                    try {
-                        Optional domain = fdr.getFailureDomain(clusterName, domainName);
-                        if (domain.isPresent()) {
-                            for (String broker : domain.get().brokers) {
-                                tempBrokerToFailureDomainMap.put(broker, domainName);
-                            }
-                        }
-                    } catch (Exception e) {
-                        log.warn("Failed to get domain {}", domainName, e);
-                    }
-                }
-                this.brokerToFailureDomainMap = tempBrokerToFailureDomainMap;
-            }
-            log.info("Cluster domain refreshed {}", brokerToFailureDomainMap);
-        } catch (Exception e) {
-            log.warn("Failed to get domain-list for cluster {}", e.getMessage());
-        }
-    }
-
     @Override
     public LocalBrokerData getBrokerLocalData(String broker) {
         String key = String.format("%s/%s", LoadManager.LOADBALANCE_BROKERS_ROOT, broker);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AntiAffinityNamespaceGroupTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AntiAffinityNamespaceGroupTest.java
index 869fee487ab19..560cfa9216a02 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AntiAffinityNamespaceGroupTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AntiAffinityNamespaceGroupTest.java
@@ -27,25 +27,25 @@
 import com.google.common.collect.Sets;
 import com.google.common.hash.Hashing;
 import java.lang.reflect.Field;
-import java.net.URL;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
-import java.util.Optional;
 import java.util.Set;
 import java.util.UUID;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
 import lombok.Cleanup;
-import org.apache.bookkeeper.util.ZkUtils;
+import org.apache.commons.lang.reflect.FieldUtils;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData;
 import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared;
 import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl;
 import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerWrapper;
+import org.apache.pulsar.broker.resources.NamespaceResources;
+import org.apache.pulsar.broker.resources.PulsarResources;
+import org.apache.pulsar.broker.resources.TenantResources;
+import org.apache.pulsar.broker.testcontext.PulsarTestContext;
 import org.apache.pulsar.client.admin.PulsarAdmin;
 import org.apache.pulsar.client.api.Producer;
 import org.apache.pulsar.client.api.PulsarClient;
@@ -54,42 +54,33 @@
 import org.apache.pulsar.common.naming.NamespaceBundles;
 import org.apache.pulsar.common.naming.NamespaceName;
 import org.apache.pulsar.common.naming.ServiceUnitId;
-import org.apache.pulsar.common.policies.data.ClusterData;
 import org.apache.pulsar.common.policies.data.FailureDomain;
+import org.apache.pulsar.common.policies.data.Policies;
+import org.apache.pulsar.common.policies.data.TenantInfo;
 import org.apache.pulsar.common.policies.data.TenantInfoImpl;
-import org.apache.pulsar.common.util.ObjectMapperFactory;
 import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap;
 import org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet;
-import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble;
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.ZooDefs.Ids;
-import org.apache.zookeeper.ZooKeeper;
+import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
 import org.awaitility.Awaitility;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
 @Test(groups = "broker")
-public class AntiAffinityNamespaceGroupTest {
-    private LocalBookkeeperEnsemble bkEnsemble;
+public class AntiAffinityNamespaceGroupTest extends MockedPulsarServiceBaseTest {
 
-    private URL url1;
+    private PulsarTestContext additionalPulsarTestContext;
     private PulsarService pulsar1;
     private PulsarAdmin admin1;
-
-    private URL url2;
     private PulsarService pulsar2;
-    private PulsarAdmin admin2;
-
-    private String primaryHost;
-    private String secondaryHost;
-
+    protected String primaryHost;
+    protected String secondaryHost;
     private NamespaceBundleFactory nsFactory;
+    protected Object primaryLoadManager;
+    private Object secondaryLoadManager;
 
-    private ModularLoadManagerImpl primaryLoadManager;
-    private ModularLoadManagerImpl secondaryLoadManager;
 
-    private ExecutorService executor;
+    private PulsarResources resources;
 
     private static Object getField(final Object instance, final String fieldName) throws Exception {
         final Field field = instance.getClass().getDeclaredField(fieldName);
@@ -97,87 +88,94 @@ private static Object getField(final Object instance, final String fieldName) th
         return field.get(instance);
     }
 
-    @BeforeMethod
-    void setup() throws Exception {
-        executor = new ThreadPoolExecutor(5, 20, 30, TimeUnit.SECONDS,
-                new LinkedBlockingQueue());
-        // Start local bookkeeper ensemble
-        bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0);
-        bkEnsemble.start();
-
-        // Start broker 1
-        ServiceConfiguration config1 = new ServiceConfiguration();
-        config1.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
-        config1.setClusterName("use");
-        config1.setWebServicePort(Optional.of(0));
-        config1.setMetadataStoreUrl("zk:127.0.0.1:" + bkEnsemble.getZookeeperPort());
-        config1.setBrokerShutdownTimeoutMs(0L);
-        config1.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d));
-        config1.setBrokerServicePort(Optional.of(0));
-        config1.setFailureDomainsEnabled(true);
-        config1.setLoadBalancerEnabled(true);
-        config1.setAdvertisedAddress("localhost");
+    void setupConfigs(ServiceConfiguration conf){
+        conf.setAllowAutoTopicCreation(true);
+        conf.setLoadManagerClassName(getLoadManagerClassName());
+        conf.setFailureDomainsEnabled(true);
+        conf.setLoadBalancerEnabled(true);
         // Don't want overloaded threshold to affect namespace placement
-        config1.setLoadBalancerBrokerOverloadedThresholdPercentage(400);
-        createCluster(bkEnsemble.getZkClient(), config1);
-        pulsar1 = new PulsarService(config1);
-        pulsar1.start();
+        conf.setLoadBalancerBrokerOverloadedThresholdPercentage(400);
+    }
 
+    @BeforeClass
+    @Override
+    public void setup() throws Exception {
+        setupConfigs(conf);
+        super.internalSetup(conf);
+        pulsar1 = pulsar;
         primaryHost = String.format("%s:%d", "localhost", pulsar1.getListenPortHTTP().get());
-        url1 = new URL("http://127.0.0.1" + ":" + pulsar1.getListenPortHTTP().get());
-        admin1 = PulsarAdmin.builder().serviceHttpUrl(url1.toString()).build();
-
-        // Start broker 2
-        ServiceConfiguration config2 = new ServiceConfiguration();
-        config2.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
-        config2.setClusterName("use");
-        config2.setWebServicePort(Optional.of(0));
-        config2.setMetadataStoreUrl("zk:127.0.0.1:" + bkEnsemble.getZookeeperPort());
-        config2.setBrokerShutdownTimeoutMs(0L);
-        config2.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d));
-        config2.setBrokerServicePort(Optional.of(0));
-        config2.setFailureDomainsEnabled(true);
-        config2.setAdvertisedAddress("localhost");
-        // Don't want overloaded threshold to affect namespace placement
-        config2.setLoadBalancerBrokerOverloadedThresholdPercentage(400);
-        pulsar2 = new PulsarService(config2);
-        pulsar2.start();
+        admin1 = admin;
 
+        var config2 = getDefaultConf();
+        setupConfigs(config2);
+        additionalPulsarTestContext = createAdditionalPulsarTestContext(config2);
+        pulsar2 = additionalPulsarTestContext.getPulsarService();
         secondaryHost = String.format("%s:%d", "localhost", pulsar2.getListenPortHTTP().get());
 
-        url2 = new URL("http://127.0.0.1" + ":" + config2.getWebServicePort().get());
-        admin2 = PulsarAdmin.builder().serviceHttpUrl(url2.toString()).build();
-
-        primaryLoadManager = (ModularLoadManagerImpl) getField(pulsar1.getLoadManager().get(), "loadManager");
-        secondaryLoadManager = (ModularLoadManagerImpl) getField(pulsar2.getLoadManager().get(), "loadManager");
+        primaryLoadManager = getField(pulsar1.getLoadManager().get(), "loadManager");
+        secondaryLoadManager = getField(pulsar2.getLoadManager().get(), "loadManager");
         nsFactory = new NamespaceBundleFactory(pulsar1, Hashing.crc32());
 
         Awaitility.await().untilAsserted(() -> {
             assertEquals(pulsar1.getState(), PulsarService.State.Started);
             assertEquals(pulsar2.getState(), PulsarService.State.Started);
         });
+
+        admin1.tenants().createTenant("my-tenant",
+                createDefaultTenantInfo());
+    }
+
+    @Override
+    @AfterClass
+    protected void cleanup() throws Exception {
+        pulsar1 = null;
+        pulsar2.close();
+        super.internalCleanup();
+        this.additionalPulsarTestContext.close();
+    }
+
+    protected void beforePulsarStart(PulsarService pulsar) throws Exception {
+        if (resources == null) {
+            MetadataStoreExtended localStore = pulsar.createLocalMetadataStore(null);
+            MetadataStoreExtended configStore = (MetadataStoreExtended) pulsar.createConfigurationMetadataStore(null);
+            resources = new PulsarResources(localStore, configStore);
+        }
+        this.createNamespaceIfNotExists(resources, NamespaceName.SYSTEM_NAMESPACE.getTenant(),
+                NamespaceName.SYSTEM_NAMESPACE);
     }
 
-    @AfterMethod(alwaysRun = true)
-    void shutdown() throws Exception {
-        executor.shutdownNow();
+    protected void createNamespaceIfNotExists(PulsarResources resources,
+                                              String publicTenant,
+                                              NamespaceName ns) throws Exception {
+        TenantResources tr = resources.getTenantResources();
+        NamespaceResources nsr = resources.getNamespaceResources();
+
+        if (!tr.tenantExists(publicTenant)) {
+            tr.createTenant(publicTenant,
+                    TenantInfo.builder()
+                            .adminRoles(Sets.newHashSet(conf.getSuperUserRoles()))
+                            .allowedClusters(Sets.newHashSet(conf.getClusterName()))
+                            .build());
+        }
 
-        admin1.close();
-        admin2.close();
+        if (!nsr.namespaceExists(ns)) {
+            Policies nsp = new Policies();
+            nsp.replication_clusters = Collections.singleton(conf.getClusterName());
+            nsr.createPolicies(ns, nsp);
+        }
+    }
 
-        pulsar2.close();
-        pulsar1.close();
 
-        bkEnsemble.stop();
+    protected Object getBundleOwnershipData(){
+        return ConcurrentOpenHashMap.>>newBuilder().build();
     }
 
-    private void createCluster(ZooKeeper zk, ServiceConfiguration config) throws Exception {
-        ZkUtils.createFullPathOptimistic(zk, "/admin/clusters/" + config.getClusterName(),
-                ObjectMapperFactory.getMapper().writer().writeValueAsBytes(
-                        ClusterData.builder().serviceUrl("http://" + config.getAdvertisedAddress() + ":" + config.getWebServicePort().get()).build()),
-                Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+
+    protected String getLoadManagerClassName() {
+        return ModularLoadManagerImpl.class.getName();
     }
 
+
     @Test
     public void testClusterDomain() {
 
@@ -209,15 +207,13 @@ public void testClusterDomain() {
     @Test
     public void testAntiAffinityNamespaceFilteringWithDomain() throws Exception {
 
-        final String namespace = "my-tenant/use/my-ns";
+        final String namespace = "my-tenant/test/my-ns";
         final int totalNamespaces = 5;
         final String namespaceAntiAffinityGroup = "my-antiaffinity";
         final String bundle = "/0x00000000_0xffffffff";
         final int totalBrokers = 4;
 
         pulsar1.getConfiguration().setFailureDomainsEnabled(true);
-        admin1.tenants().createTenant("my-tenant",
-                new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("use")));
 
         for (int i = 0; i < totalNamespaces; i++) {
             final String ns = namespace + i;
@@ -237,8 +233,7 @@ public void testAntiAffinityNamespaceFilteringWithDomain() throws Exception {
         brokerToDomainMap.put("brokerName-3", "domain-1");
 
         Set candidate = new HashSet<>();
-        ConcurrentOpenHashMap>> brokerToNamespaceToBundleRange =
-                ConcurrentOpenHashMap.>>newBuilder().build();
+        Object brokerToNamespaceToBundleRange = getBundleOwnershipData();
 
         assertEquals(brokers.size(), totalBrokers);
 
@@ -246,7 +241,7 @@ public void testAntiAffinityNamespaceFilteringWithDomain() throws Exception {
         candidate.addAll(brokers);
 
         // for namespace-0 all brokers available
-        LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, brokers,
+        filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, brokers,
                 brokerToNamespaceToBundleRange, brokerToDomainMap);
         assertEquals(brokers.size(), totalBrokers);
 
@@ -255,7 +250,7 @@ public void testAntiAffinityNamespaceFilteringWithDomain() throws Exception {
         candidate.addAll(brokers);
         // for namespace-1 only domain-1 brokers are available as broker-0 already owns namespace-0
         assignedNamespace = namespace + "1" + bundle;
-        LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate,
+        filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate,
                 brokerToNamespaceToBundleRange, brokerToDomainMap);
         assertEquals(candidate.size(), 2);
         candidate.forEach(broker -> assertEquals(brokerToDomainMap.get(broker), "domain-1"));
@@ -265,7 +260,7 @@ public void testAntiAffinityNamespaceFilteringWithDomain() throws Exception {
         candidate.addAll(brokers);
         // for namespace-2 only brokers available are : broker-1 and broker-3
         assignedNamespace = namespace + "2" + bundle;
-        LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate,
+        filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate,
                 brokerToNamespaceToBundleRange, brokerToDomainMap);
         assertEquals(candidate.size(), 2);
         assertTrue(candidate.contains("brokerName-1"));
@@ -276,7 +271,7 @@ public void testAntiAffinityNamespaceFilteringWithDomain() throws Exception {
         candidate.addAll(brokers);
         // for namespace-3 only brokers available are : broker-3
         assignedNamespace = namespace + "3" + bundle;
-        LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate,
+        filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate,
                 brokerToNamespaceToBundleRange, brokerToDomainMap);
         assertEquals(candidate.size(), 1);
         assertTrue(candidate.contains("brokerName-3"));
@@ -285,7 +280,7 @@ public void testAntiAffinityNamespaceFilteringWithDomain() throws Exception {
         candidate.addAll(brokers);
         // for namespace-4 only brokers available are : all 4 brokers
         assignedNamespace = namespace + "4" + bundle;
-        LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate,
+        filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate,
                 brokerToNamespaceToBundleRange, brokerToDomainMap);
         assertEquals(candidate.size(), 4);
     }
@@ -308,14 +303,11 @@ public void testAntiAffinityNamespaceFilteringWithDomain() throws Exception {
     @Test
     public void testAntiAffinityNamespaceFilteringWithoutDomain() throws Exception {
 
-        final String namespace = "my-tenant/use/my-ns";
+        final String namespace = "my-tenant/test/my-ns-wo-domain";
         final int totalNamespaces = 5;
-        final String namespaceAntiAffinityGroup = "my-antiaffinity";
+        final String namespaceAntiAffinityGroup = "my-antiaffinity-wo-domain";
         final String bundle = "/0x00000000_0xffffffff";
 
-        admin1.tenants().createTenant("my-tenant",
-                new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("use")));
-
         for (int i = 0; i < totalNamespaces; i++) {
             final String ns = namespace + i;
             admin1.namespaces().createNamespace(ns);
@@ -324,8 +316,7 @@ public void testAntiAffinityNamespaceFilteringWithoutDomain() throws Exception {
 
         Set brokers = new HashSet<>();
         Set candidate = new HashSet<>();
-        ConcurrentOpenHashMap>> brokerToNamespaceToBundleRange =
-                ConcurrentOpenHashMap.>>newBuilder().build();
+        Object brokerToNamespaceToBundleRange = getBundleOwnershipData();
         brokers.add("broker-0");
         brokers.add("broker-1");
         brokers.add("broker-2");
@@ -334,7 +325,7 @@ public void testAntiAffinityNamespaceFilteringWithoutDomain() throws Exception {
 
         // all brokers available so, candidate will be all 3 brokers
         candidate.addAll(brokers);
-        LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, brokers,
+        filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, brokers,
                 brokerToNamespaceToBundleRange, null);
         assertEquals(brokers.size(), 3);
 
@@ -343,7 +334,7 @@ public void testAntiAffinityNamespaceFilteringWithoutDomain() throws Exception {
         candidate.addAll(brokers);
         assignedNamespace = namespace + "1" + bundle;
         // available brokers for ns-1 => broker-1, broker-2
-        LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate,
+        filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate,
                 brokerToNamespaceToBundleRange, null);
         assertEquals(candidate.size(), 2);
         assertTrue(candidate.contains("broker-1"));
@@ -354,7 +345,7 @@ public void testAntiAffinityNamespaceFilteringWithoutDomain() throws Exception {
         candidate.addAll(brokers);
         // available brokers for ns-2 => broker-2
         assignedNamespace = namespace + "2" + bundle;
-        LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate,
+        filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate,
                 brokerToNamespaceToBundleRange, null);
         assertEquals(candidate.size(), 1);
         assertTrue(candidate.contains("broker-2"));
@@ -364,14 +355,19 @@ public void testAntiAffinityNamespaceFilteringWithoutDomain() throws Exception {
         candidate.addAll(brokers);
         // available brokers for ns-3 => broker-0, broker-1, broker-2
         assignedNamespace = namespace + "3" + bundle;
-        LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate,
+        filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate,
                 brokerToNamespaceToBundleRange, null);
         assertEquals(candidate.size(), 3);
     }
 
-    private void selectBrokerForNamespace(
-            ConcurrentOpenHashMap>> brokerToNamespaceToBundleRange,
+    protected void selectBrokerForNamespace(
+            Object ownershipData,
             String broker, String namespace, String assignedBundleName) {
+
+        ConcurrentOpenHashMap>>
+                brokerToNamespaceToBundleRange =
+                (ConcurrentOpenHashMap>>) ownershipData;
         ConcurrentOpenHashSet bundleSet =
                 ConcurrentOpenHashSet.newBuilder().build();
         bundleSet.add(assignedBundleName);
@@ -436,15 +432,19 @@ public void testBrokerSelectionForAntiAffinityGroup() throws Exception {
         });
 
         ServiceUnitId serviceUnit1 = makeBundle(tenant, cluster, "ns1");
-        String selectedBroker1 = primaryLoadManager.selectBrokerForAssignment(serviceUnit1).get();
+        String selectedBroker1 = selectBroker(serviceUnit1, primaryLoadManager);
 
         ServiceUnitId serviceUnit2 = makeBundle(tenant, cluster, "ns2");
-        String selectedBroker2 = primaryLoadManager.selectBrokerForAssignment(serviceUnit2).get();
+        String selectedBroker2 = selectBroker(serviceUnit2, primaryLoadManager);
 
         assertNotEquals(selectedBroker1, selectedBroker2);
 
     }
 
+    protected String selectBroker(ServiceUnitId serviceUnit, Object loadManager) {
+        return ((ModularLoadManager) loadManager).selectBrokerForAssignment(serviceUnit).get();
+    }
+
     /**
      * It verifies that load-shedding task should unload namespace only if there is a broker available which doesn't
      * cause uneven anti-affinity namespace distribution.
@@ -460,14 +460,11 @@ public void testBrokerSelectionForAntiAffinityGroup() throws Exception {
     @Test
     public void testLoadSheddingUtilWithAntiAffinityNamespace() throws Exception {
 
-        final String namespace = "my-tenant/use/my-ns";
+        final String namespace = "my-tenant/test/my-ns-load-shedding-util";
         final int totalNamespaces = 5;
-        final String namespaceAntiAffinityGroup = "my-antiaffinity";
+        final String namespaceAntiAffinityGroup = "my-antiaffinity-load-shedding-util";
         final String bundle = "/0x00000000_0xffffffff";
 
-        admin1.tenants().createTenant("my-tenant",
-                new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("use")));
-
         for (int i = 0; i < totalNamespaces; i++) {
             final String ns = namespace + i;
             admin1.namespaces().createNamespace(ns);
@@ -476,8 +473,7 @@ public void testLoadSheddingUtilWithAntiAffinityNamespace() throws Exception {
 
         Set brokers = new HashSet<>();
         Set candidate = new HashSet<>();
-        ConcurrentOpenHashMap>> brokerToNamespaceToBundleRange =
-                ConcurrentOpenHashMap.>>newBuilder().build();
+        Object brokerToNamespaceToBundleRange = getBundleOwnershipData();
         brokers.add("broker-0");
         brokers.add("broker-1");
         brokers.add("broker-2");
@@ -489,17 +485,17 @@ public void testLoadSheddingUtilWithAntiAffinityNamespace() throws Exception {
         // add ns-0 to broker-0
         selectBrokerForNamespace(brokerToNamespaceToBundleRange, "broker-0", namespace + "0", assignedNamespace);
         String currentBroker = "broker-0";
-        boolean shouldUnload = LoadManagerShared.shouldAntiAffinityNamespaceUnload(namespace + "0", bundle,
+        boolean shouldUnload = shouldAntiAffinityNamespaceUnload(namespace + "0", bundle,
                 currentBroker, pulsar1, brokerToNamespaceToBundleRange, candidate);
         assertTrue(shouldUnload);
         // add ns-1 to broker-1
         selectBrokerForNamespace(brokerToNamespaceToBundleRange, "broker-1", namespace + "1", assignedNamespace);
-        shouldUnload = LoadManagerShared.shouldAntiAffinityNamespaceUnload(namespace + "0", bundle, currentBroker,
+        shouldUnload = shouldAntiAffinityNamespaceUnload(namespace + "0", bundle, currentBroker,
                 pulsar1, brokerToNamespaceToBundleRange, candidate);
         assertTrue(shouldUnload);
         // add ns-2 to broker-2
         selectBrokerForNamespace(brokerToNamespaceToBundleRange, "broker-2", namespace + "2", assignedNamespace);
-        shouldUnload = LoadManagerShared.shouldAntiAffinityNamespaceUnload(namespace + "0", bundle, currentBroker,
+        shouldUnload = shouldAntiAffinityNamespaceUnload(namespace + "0", bundle, currentBroker,
                 pulsar1, brokerToNamespaceToBundleRange, candidate);
         assertFalse(shouldUnload);
 
@@ -514,14 +510,11 @@ public void testLoadSheddingUtilWithAntiAffinityNamespace() throws Exception {
     @Test
     public void testLoadSheddingWithAntiAffinityNamespace() throws Exception {
 
-        final String namespace = "my-tenant/use/my-ns";
+        final String namespace = "my-tenant/test/my-ns-load-shedding";
         final int totalNamespaces = 5;
-        final String namespaceAntiAffinityGroup = "my-antiaffinity";
+        final String namespaceAntiAffinityGroup = "my-antiaffinity-load-shedding";
         final String bundle = "0x00000000_0xffffffff";
 
-        admin1.tenants().createTenant("my-tenant",
-                new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("use")));
-
         for (int i = 0; i < totalNamespaces; i++) {
             final String ns = namespace + i;
             admin1.namespaces().createNamespace(ns);
@@ -532,22 +525,24 @@ public void testLoadSheddingWithAntiAffinityNamespace() throws Exception {
         PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(pulsar1.getSafeWebServiceAddress()).build();
         Producer producer = pulsarClient.newProducer().topic("persistent://" + namespace + "0/my-topic1")
                 .create();
+        pulsar1.getBrokerService().updateRates();
+        verifyLoadSheddingWithAntiAffinityNamespace(namespace + "0", bundle);
+        producer.close();
+    }
+
+    protected void verifyLoadSheddingWithAntiAffinityNamespace(String namespace, String bundle) {
+
         ModularLoadManagerImpl loadManager = (ModularLoadManagerImpl) ((ModularLoadManagerWrapper) pulsar1
                 .getLoadManager().get()).getLoadManager();
-
-        pulsar1.getBrokerService().updateRates();
         loadManager.updateAll();
-
-        assertTrue(loadManager.shouldAntiAffinityNamespaceUnload(namespace + "0", bundle, primaryHost));
-        producer.close();
+        assertTrue(loadManager.shouldAntiAffinityNamespaceUnload(namespace, bundle, primaryHost));
     }
 
-    private boolean isLoadManagerUpdatedDomainCache(ModularLoadManagerImpl loadManager) throws Exception {
-        Field mapField = ModularLoadManagerImpl.class.getDeclaredField("brokerToFailureDomainMap");
-        mapField.setAccessible(true);
+    protected boolean isLoadManagerUpdatedDomainCache(Object loadManager) throws Exception {
         @SuppressWarnings("unchecked")
-        Map map = (Map) mapField.get(loadManager);
-        return !map.isEmpty();
+        var brokerToFailureDomainMap = (Map)
+                FieldUtils.readDeclaredField(loadManager, "brokerToFailureDomainMap", true);
+        return !brokerToFailureDomainMap.isEmpty();
     }
 
     private NamespaceBundle makeBundle(final String property, final String cluster, final String namespace) {
@@ -556,4 +551,43 @@ private NamespaceBundle makeBundle(final String property, final String cluster,
                         BoundType.CLOSED));
     }
 
+    private static void filterAntiAffinityGroupOwnedBrokers(
+            PulsarService pulsar,
+            String assignedNamespace,
+            Set brokers,
+            Object ownershipData,
+            Map brokerToDomainMap) {
+        if (ownershipData instanceof Set) {
+            LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar, assignedNamespace, brokers,
+                    (Set>) ownershipData, brokerToDomainMap);
+        } else if (ownershipData instanceof ConcurrentOpenHashMap) {
+            LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar, assignedNamespace, brokers,
+                    (ConcurrentOpenHashMap>>)
+                            ownershipData, brokerToDomainMap);
+        } else {
+            throw new RuntimeException("Unknown ownershipData class type");
+        }
+    }
+
+    private static boolean shouldAntiAffinityNamespaceUnload(
+            String namespace,
+            String bundle,
+            String currentBroker,
+            PulsarService pulsar,
+            Object ownershipData,
+            Set candidate) throws Exception {
+
+        if (ownershipData instanceof Set) {
+            return LoadManagerShared.shouldAntiAffinityNamespaceUnload(namespace, bundle,
+                    currentBroker, pulsar, (Set>) ownershipData, candidate);
+        } else if (ownershipData instanceof ConcurrentOpenHashMap) {
+            return LoadManagerShared.shouldAntiAffinityNamespaceUnload(namespace, bundle,
+                    currentBroker, pulsar,
+                    (ConcurrentOpenHashMap>>)
+                            ownershipData, candidate);
+        } else {
+            throw new RuntimeException("Unknown ownershipData class type");
+        }
+    }
+
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java
new file mode 100644
index 0000000000000..32822c0f5b524
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.loadbalance.extensions;
+
+import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Owned;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+import java.util.AbstractMap;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import lombok.Cleanup;
+import org.apache.pulsar.broker.loadbalance.AntiAffinityNamespaceGroupTest;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData;
+import org.apache.pulsar.broker.loadbalance.extensions.filter.AntiAffinityGroupPolicyFilter;
+import org.apache.pulsar.broker.loadbalance.extensions.policies.AntiAffinityGroupPolicyHelper;
+import org.apache.pulsar.client.admin.PulsarAdminException;
+import org.apache.pulsar.client.api.Producer;
+import org.apache.pulsar.client.api.PulsarClient;
+import org.apache.pulsar.client.api.PulsarClientException;
+import org.apache.pulsar.common.naming.ServiceUnitId;
+import org.testcontainers.shaded.org.apache.commons.lang3.reflect.FieldUtils;
+import org.testng.annotations.Test;
+
+@Test(groups = "broker")
+public class AntiAffinityNamespaceGroupExtensionTest extends AntiAffinityNamespaceGroupTest {
+
+    final String bundle = "0x00000000_0xffffffff";
+    final String nsSuffix = "-antiaffinity-enabled";
+
+    protected Object getBundleOwnershipData() {
+        return new HashSet>();
+    }
+
+    protected String getLoadManagerClassName() {
+        return ExtensibleLoadManagerImpl.class.getName();
+    }
+
+    protected String selectBroker(ServiceUnitId serviceUnit, Object loadManager) {
+        try {
+            return ((ExtensibleLoadManagerImpl) loadManager).assign(Optional.empty(), serviceUnit).get()
+                    .get().getPulsarServiceUrl();
+        } catch (Throwable e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    protected void selectBrokerForNamespace(
+            Object ownershipData,
+            String broker, String namespace, String assignedBundleName) {
+
+        Set> ownershipDataSet =
+                (Set>) ownershipData;
+        ownershipDataSet.add(
+                new AbstractMap.SimpleEntry(
+                        assignedBundleName,
+                        new ServiceUnitStateData(Owned, broker, 1)));
+
+    }
+
+    protected void verifyLoadSheddingWithAntiAffinityNamespace(String namespace, String bundle) {
+        try {
+            String namespaceBundle = namespace + "/" + bundle;
+            var antiAffinityGroupPolicyHelper =
+                    (AntiAffinityGroupPolicyHelper)
+                            FieldUtils.readDeclaredField(
+                                    primaryLoadManager, "antiAffinityGroupPolicyHelper", true);
+            var brokerRegistry =
+                    (BrokerRegistry)
+                            FieldUtils.readDeclaredField(
+                                    primaryLoadManager, "brokerRegistry", true);
+            var brokers = brokerRegistry
+                    .getAvailableBrokerLookupDataAsync().get(5, TimeUnit.SECONDS);
+            var serviceUnitStateChannel = (ServiceUnitStateChannel)
+                    FieldUtils.readDeclaredField(
+                            primaryLoadManager, "serviceUnitStateChannel", true);
+            var srcBroker = serviceUnitStateChannel.getOwnerAsync(namespaceBundle)
+                    .get(5, TimeUnit.SECONDS).get();
+            var brokersCopy = new HashMap<>(brokers);
+            brokersCopy.remove(srcBroker);
+            var dstBroker = brokersCopy.entrySet().iterator().next().getKey();
+
+            assertTrue(antiAffinityGroupPolicyHelper.canUnload(brokers,
+                    "not-enabled-" + namespace + "/" + bundle,
+                    srcBroker, Optional.of(dstBroker)));
+
+            assertTrue(antiAffinityGroupPolicyHelper.canUnload(brokers,
+                    "not-enabled-" + namespace + "/" + bundle,
+                    srcBroker, Optional.empty()));
+
+            assertTrue(antiAffinityGroupPolicyHelper.canUnload(brokers,
+                    namespaceBundle,
+                    srcBroker, Optional.of(dstBroker)));
+
+            assertFalse(antiAffinityGroupPolicyHelper.canUnload(brokers,
+                    namespaceBundle,
+                    dstBroker, Optional.of(srcBroker)));
+
+            assertTrue(antiAffinityGroupPolicyHelper.canUnload(brokers,
+                    namespaceBundle,
+                    srcBroker, Optional.empty()));
+
+            assertFalse(antiAffinityGroupPolicyHelper.canUnload(brokers,
+                    namespaceBundle,
+                    dstBroker, Optional.empty()));
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    protected boolean isLoadManagerUpdatedDomainCache(Object loadManager) throws Exception {
+        @SuppressWarnings("unchecked")
+        var antiAffinityGroupPolicyHelper =
+                (AntiAffinityGroupPolicyHelper)
+                        FieldUtils.readDeclaredField(
+                                loadManager, "antiAffinityGroupPolicyHelper", true);
+        var brokerToFailureDomainMap = (Map)
+                org.apache.commons.lang.reflect.FieldUtils.readDeclaredField(antiAffinityGroupPolicyHelper,
+                        "brokerToFailureDomainMap", true);
+        return !brokerToFailureDomainMap.isEmpty();
+    }
+
+    @Test
+    public void testAntiAffinityGroupPolicyFilter()
+            throws IllegalAccessException, ExecutionException, InterruptedException,
+            TimeoutException, PulsarAdminException, PulsarClientException {
+
+        final String namespace = "my-tenant/test/my-ns-filter";
+        final String namespaceAntiAffinityGroup = "my-antiaffinity-filter";
+
+
+        final String antiAffinityEnabledNameSpace = namespace + nsSuffix;
+        admin.namespaces().createNamespace(antiAffinityEnabledNameSpace);
+        admin.namespaces().setNamespaceAntiAffinityGroup(antiAffinityEnabledNameSpace, namespaceAntiAffinityGroup);
+        PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(pulsar.getSafeWebServiceAddress()).build();
+        @Cleanup
+        Producer producer = pulsarClient.newProducer().topic(
+                        "persistent://" + antiAffinityEnabledNameSpace + "/my-topic1")
+                .create();
+        pulsar.getBrokerService().updateRates();
+        var brokerRegistry =
+                (BrokerRegistry)
+                        FieldUtils.readDeclaredField(
+                                primaryLoadManager, "brokerRegistry", true);
+        var antiAffinityGroupPolicyFilter =
+                (AntiAffinityGroupPolicyFilter)
+                        FieldUtils.readDeclaredField(
+                                primaryLoadManager, "antiAffinityGroupPolicyFilter", true);
+        var context = ((ExtensibleLoadManagerImpl) primaryLoadManager).getContext();
+        var brokers = brokerRegistry
+                .getAvailableBrokerLookupDataAsync().get(5, TimeUnit.SECONDS);
+        ServiceUnitId namespaceBundle = mock(ServiceUnitId.class);
+        doReturn(namespace + "/" + bundle).when(namespaceBundle).toString();
+
+        var expected = new HashMap<>(brokers);
+        var actual = antiAffinityGroupPolicyFilter.filter(
+                brokers, namespaceBundle, context);
+        assertEquals(actual, expected);
+
+        doReturn(antiAffinityEnabledNameSpace + "/" + bundle).when(namespaceBundle).toString();
+        var serviceUnitStateChannel = (ServiceUnitStateChannel)
+                FieldUtils.readDeclaredField(
+                        primaryLoadManager, "serviceUnitStateChannel", true);
+        var srcBroker = serviceUnitStateChannel.getOwnerAsync(namespaceBundle.toString())
+                .get(5, TimeUnit.SECONDS).get();
+        expected.remove(srcBroker);
+        actual = antiAffinityGroupPolicyFilter.filter(
+                brokers, namespaceBundle, context);
+        assertEquals(actual, expected);
+    }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
index a668d85f0071c..47bf1ad2e7ed1 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
@@ -40,6 +40,8 @@
 import static org.mockito.Mockito.spy;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertTrue;
+import com.google.common.collect.BoundType;
+import com.google.common.collect.Range;
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -50,8 +52,6 @@
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeoutException;
 import java.util.function.BiConsumer;
-import com.google.common.collect.BoundType;
-import com.google.common.collect.Range;
 import org.apache.commons.lang.reflect.FieldUtils;
 import org.apache.commons.math3.stat.descriptive.moment.Mean;
 import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation;
@@ -65,11 +65,13 @@
 import org.apache.pulsar.broker.loadbalance.extensions.models.TopKBundles;
 import org.apache.pulsar.broker.loadbalance.extensions.models.Unload;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision;
+import org.apache.pulsar.broker.loadbalance.extensions.policies.AntiAffinityGroupPolicyHelper;
 import org.apache.pulsar.broker.loadbalance.extensions.policies.IsolationPoliciesHelper;
 import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore;
 import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies;
 import org.apache.pulsar.broker.namespace.NamespaceService;
 import org.apache.pulsar.broker.resources.LocalPoliciesResources;
+import org.apache.pulsar.broker.resources.NamespaceResources;
 import org.apache.pulsar.broker.resources.PulsarResources;
 import org.apache.pulsar.common.naming.NamespaceBundle;
 import org.apache.pulsar.common.naming.NamespaceBundleFactory;
@@ -82,6 +84,7 @@
 import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats;
 import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage;
 import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage;
+import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
 
@@ -89,6 +92,49 @@
 public class TransferShedderTest {
     double setupLoadAvg = 0.36400000000000005;
     double setupLoadStd = 0.3982762860126121;
+
+    PulsarService pulsar;
+    AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper;
+    ServiceConfiguration conf;
+    LocalPoliciesResources localPoliciesResources;
+    String bundleD1 = "my-tenant/my-namespaceD/0x00000000_0x0FFFFFFF";
+    String bundleD2 = "my-tenant/my-namespaceD/0x0FFFFFFF_0xFFFFFFFF";
+    String bundleE1 = "my-tenant/my-namespaceE/0x00000000_0x0FFFFFFF";
+    String bundleE2 = "my-tenant/my-namespaceE/0x0FFFFFFF_0xFFFFFFFF";
+
+    @BeforeMethod
+    public void init() throws MetadataStoreException {
+        pulsar = mock(PulsarService.class);
+        conf = new ServiceConfiguration();
+        doReturn(conf).when(pulsar).getConfiguration();
+
+        var pulsarResources = mock(PulsarResources.class);
+        var namespaceResources = mock(NamespaceResources.class);
+        var isolationPolicyResources = mock(NamespaceResources.IsolationPolicyResources.class);
+        var factory = mock(NamespaceBundleFactory.class);
+        var namespaceService = mock(NamespaceService.class);
+        localPoliciesResources = mock(LocalPoliciesResources.class);
+        antiAffinityGroupPolicyHelper = mock(AntiAffinityGroupPolicyHelper.class);
+        doReturn(namespaceService).when(pulsar).getNamespaceService();
+        doReturn(pulsarResources).when(pulsar).getPulsarResources();
+        doReturn(localPoliciesResources).when(pulsarResources).getLocalPolicies();
+        doReturn(namespaceResources).when(pulsarResources).getNamespaceResources();
+        doReturn(isolationPolicyResources).when(namespaceResources).getIsolationPolicies();
+        doReturn(Optional.empty()).when(localPoliciesResources).getLocalPolicies(any());
+        doReturn(Optional.empty()).when(isolationPolicyResources).getIsolationDataPolicies(any());
+        doReturn(factory).when(namespaceService).getNamespaceBundleFactory();
+        doAnswer(answer -> {
+            String namespace = answer.getArgument(0, String.class);
+            String bundleRange = answer.getArgument(1, String.class);
+            String[] boundaries = bundleRange.split("_");
+            Long lowerEndpoint = Long.decode(boundaries[0]);
+            Long upperEndpoint = Long.decode(boundaries[1]);
+            Range hashRange = Range.range(lowerEndpoint, BoundType.CLOSED, upperEndpoint,
+                    (upperEndpoint.equals(NamespaceBundles.FULL_UPPER_BOUND)) ? BoundType.CLOSED : BoundType.OPEN);
+            return new NamespaceBundle(NamespaceName.get(namespace), hashRange, factory);
+        }).when(factory).getBundle(anyString(), anyString());
+        doReturn(true).when(antiAffinityGroupPolicyHelper).canUnload(any(), any(), any(), any());
+    }
     public LoadManagerContext setupContext(){
         var ctx = getContext();
         ctx.brokerConfiguration().setLoadBalancerDebugModeEnabled(true);
@@ -101,11 +147,11 @@ public LoadManagerContext setupContext(){
         brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx,  90));
 
         var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
-        topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("bundleA", 2000000, 1000000));
-        topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("bundleB", 3000000, 1000000));
-        topBundlesLoadDataStore.pushAsync("broker3", getTopBundlesLoad("bundleC", 4000000, 2000000));
-        topBundlesLoadDataStore.pushAsync("broker4", getTopBundlesLoad("bundleD", 6000000, 2000000));
-        topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("bundleE", 7000000, 2000000));
+        topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 2000000, 1000000));
+        topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB", 3000000, 1000000));
+        topBundlesLoadDataStore.pushAsync("broker3", getTopBundlesLoad("my-tenant/my-namespaceC", 4000000, 2000000));
+        topBundlesLoadDataStore.pushAsync("broker4", getTopBundlesLoad("my-tenant/my-namespaceD", 6000000, 2000000));
+        topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 7000000, 2000000));
         return ctx;
     }
 
@@ -151,8 +197,8 @@ public TopBundlesLoadData getTopBundlesLoad(String bundlePrefix, int load1, int
         var namespaceBundleStats2 = new NamespaceBundleStats();
         namespaceBundleStats2.msgThroughputOut = load2;
         var topKBundles = new TopKBundles();
-        topKBundles.update(Map.of(bundlePrefix + "-1", namespaceBundleStats1,
-                bundlePrefix + "-2", namespaceBundleStats2), 2);
+        topKBundles.update(Map.of(bundlePrefix + "/0x00000000_0x0FFFFFFF", namespaceBundleStats1,
+                bundlePrefix + "/0x0FFFFFFF_0xFFFFFFFF", namespaceBundleStats2), 2);
         return topKBundles.getLoadData();
     }
 
@@ -160,7 +206,7 @@ public TopBundlesLoadData getTopBundlesLoad(String bundlePrefix, int load1) {
         var namespaceBundleStats1 = new NamespaceBundleStats();
         namespaceBundleStats1.msgThroughputOut = load1;
         var topKBundles = new TopKBundles();
-        topKBundles.update(Map.of(bundlePrefix + "-1", namespaceBundleStats1), 2);
+        topKBundles.update(Map.of(bundlePrefix + "/0x00000000_0x0FFFFFFF", namespaceBundleStats1), 2);
         return topKBundles.getLoadData();
     }
 
@@ -265,13 +311,20 @@ public int size() {
                 return map.size();
             }
         };
+
+        BrokerRegistry brokerRegistry = mock(BrokerRegistry.class);
+        doReturn(CompletableFuture.completedFuture(Map.of(
+                "broker1", mock(BrokerLookupData.class),
+                "broker2", mock(BrokerLookupData.class),
+                "broker3", mock(BrokerLookupData.class),
+                "broker4", mock(BrokerLookupData.class),
+                "broker5", mock(BrokerLookupData.class)
+        )))
+                .when(brokerRegistry).getAvailableBrokerLookupDataAsync();
         doReturn(conf).when(ctx).brokerConfiguration();
         doReturn(brokerLoadDataStore).when(ctx).brokerLoadDataStore();
         doReturn(topBundleLoadDataStore).when(ctx).topBundleLoadDataStore();
-        var brokerRegister = mock(BrokerRegistry.class);
-        doReturn(brokerRegister).when(ctx).brokerRegistry();
-        BrokerRegistry registry = ctx.brokerRegistry();
-        doReturn(CompletableFuture.completedFuture(Map.of())).when(registry).getAvailableBrokerLookupDataAsync();
+        doReturn(brokerRegistry).when(ctx).brokerRegistry();
         return ctx;
     }
 
@@ -345,9 +398,9 @@ public void testRecentlyUnloadedBrokers() {
         var expected = new UnloadDecision();
         var unloads = expected.getUnloads();
         unloads.put("broker5",
-                new Unload("broker5", "bundleE-1", Optional.of("broker1")));
+                new Unload("broker5", bundleE1, Optional.of("broker1")));
         unloads.put("broker4",
-                new Unload("broker4", "bundleD-1", Optional.of("broker2")));
+                new Unload("broker4", bundleD1, Optional.of("broker2")));
 
         expected.setLabel(Success);
         expected.setReason(Overloaded);
@@ -371,10 +424,10 @@ public void testRecentlyUnloadedBundles() {
         var ctx = setupContext();
         Map recentlyUnloadedBundles = new HashMap<>();
         var now = System.currentTimeMillis();
-        recentlyUnloadedBundles.put("bundleE-1", now);
-        recentlyUnloadedBundles.put("bundleE-2", now);
-        recentlyUnloadedBundles.put("bundleD-1", now);
-        recentlyUnloadedBundles.put("bundleD-2", now);
+        recentlyUnloadedBundles.put(bundleE1, now);
+        recentlyUnloadedBundles.put(bundleE2, now);
+        recentlyUnloadedBundles.put(bundleD1, now);
+        recentlyUnloadedBundles.put(bundleD2, now);
         var res = transferShedder.findBundlesForUnloading(ctx, recentlyUnloadedBundles, Map.of());
 
         var expected = new UnloadDecision();
@@ -402,16 +455,10 @@ public void testGetAvailableBrokersFailed() {
     }
 
     @Test(timeOut = 30 * 1000)
-    public void testBundlesWithIsolationPolicies() throws IllegalAccessException, MetadataStoreException {
-        var pulsar = getMockPulsar();
+    public void testBundlesWithIsolationPolicies() throws IllegalAccessException {
 
-        TransferShedder transferShedder = new TransferShedder(pulsar);
 
-        var pulsarResourcesMock = mock(PulsarResources.class);
-        var localPoliciesResourcesMock = mock(LocalPoliciesResources.class);
-        doReturn(pulsarResourcesMock).when(pulsar).getPulsarResources();
-        doReturn(localPoliciesResourcesMock).when(pulsarResourcesMock).getLocalPolicies();
-        doReturn(Optional.empty()).when(localPoliciesResourcesMock).getLocalPolicies(any());
+        TransferShedder transferShedder = new TransferShedder(pulsar, antiAffinityGroupPolicyHelper);
 
         var allocationPoliciesSpy = (SimpleResourceAllocationPolicies)
                 spy(FieldUtils.readDeclaredField(transferShedder, "allocationPolicies", true));
@@ -423,31 +470,11 @@ public void testBundlesWithIsolationPolicies() throws IllegalAccessException, Me
         setIsolationPolicies(allocationPoliciesSpy, "my-tenant/my-namespaceE",
                 Set.of("broker5"), Set.of(), Set.of(), 1);
         var ctx = setupContext();
-        BrokerRegistry registry = ctx.brokerRegistry();
-        doReturn(CompletableFuture.completedFuture(Map.of(
-                "broker1", getLookupData(),
-                "broker2", getLookupData(),
-                "broker3", getLookupData(),
-                "broker4", getLookupData(),
-                "broker5", getLookupData()))).when(registry).getAvailableBrokerLookupDataAsync();
-
-        var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
-        topBundlesLoadDataStore.pushAsync("broker1",
-                getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceA", 1, 3));
-        topBundlesLoadDataStore.pushAsync("broker2",
-                getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceB", 2, 8));
-        topBundlesLoadDataStore.pushAsync("broker3",
-                getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceC", 6, 10));
-        topBundlesLoadDataStore.pushAsync("broker4",
-                getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceD", 10, 20));
-        topBundlesLoadDataStore.pushAsync("broker5",
-                getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceE", 70, 20));
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
-
         var expected = new UnloadDecision();
         var unloads = expected.getUnloads();
         unloads.put("broker4",
-                new Unload("broker4", "my-tenant/my-namespaceD/0x7FFFFFF_0xFFFFFFF", Optional.of("broker2")));
+                new Unload("broker4", bundleD1, Optional.of("broker2")));
         expected.setLabel(Success);
         expected.setReason(Overloaded);
         expected.setLoadAvg(setupLoadAvg);
@@ -455,37 +482,12 @@ public void testBundlesWithIsolationPolicies() throws IllegalAccessException, Me
         assertEquals(res, expected);
 
         // Test unload a has isolation policies broker.
-
-        setIsolationPolicies(allocationPoliciesSpy, "my-tenant/my-namespaceE",
-                Set.of("broker5"), Set.of(), Set.of(), 1);
-        ctx = setupContext();
-        registry = ctx.brokerRegistry();
-        doReturn(CompletableFuture.completedFuture(Map.of(
-                "broker1", getLookupData(),
-                "broker2", getLookupData(),
-                "broker3", getLookupData(),
-                "broker4", getLookupData(),
-                "broker5", getLookupData()))).when(registry).getAvailableBrokerLookupDataAsync();
-
         ctx.brokerConfiguration().setLoadBalancerTransferEnabled(false);
-
-        topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
-        topBundlesLoadDataStore.pushAsync("broker1",
-                getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceA", 1, 3));
-        topBundlesLoadDataStore.pushAsync("broker2",
-                getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceB", 2, 8));
-        topBundlesLoadDataStore.pushAsync("broker3",
-                getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceC", 6, 10));
-        topBundlesLoadDataStore.pushAsync("broker4",
-                getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceD", 10, 20));
-        topBundlesLoadDataStore.pushAsync("broker5",
-                getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceE", 70, 20));
         res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
-
         expected = new UnloadDecision();
         unloads = expected.getUnloads();
         unloads.put("broker4",
-                new Unload("broker4", "my-tenant/my-namespaceD/0x7FFFFFF_0xFFFFFFF", Optional.empty()));
+                new Unload("broker4", bundleD1, Optional.empty()));
         expected.setLabel(Success);
         expected.setReason(Overloaded);
         expected.setLoadAvg(setupLoadAvg);
@@ -567,31 +569,17 @@ private PulsarService getMockPulsar() {
     @Test
     public void testBundlesWithAntiAffinityGroup() throws IllegalAccessException, MetadataStoreException {
         var pulsar = getMockPulsar();
-        TransferShedder transferShedder = new TransferShedder(pulsar);
+        TransferShedder transferShedder = new TransferShedder(pulsar, antiAffinityGroupPolicyHelper);
         var allocationPoliciesSpy = (SimpleResourceAllocationPolicies)
                 spy(FieldUtils.readDeclaredField(transferShedder, "allocationPolicies", true));
         doReturn(false).when(allocationPoliciesSpy).areIsolationPoliciesPresent(any());
         FieldUtils.writeDeclaredField(transferShedder, "allocationPolicies", allocationPoliciesSpy, true);
 
-        var pulsarResourcesMock = mock(PulsarResources.class);
-        var localPoliciesResourcesMock = mock(LocalPoliciesResources.class);
-        doReturn(pulsarResourcesMock).when(pulsar).getPulsarResources();
-        doReturn(localPoliciesResourcesMock).when(pulsarResourcesMock).getLocalPolicies();
         LocalPolicies localPolicies = new LocalPolicies(null, null, "namespaceAntiAffinityGroup");
-        doReturn(Optional.of(localPolicies)).when(localPoliciesResourcesMock).getLocalPolicies(any());
+        doReturn(Optional.of(localPolicies)).when(localPoliciesResources).getLocalPolicies(any());
 
         var ctx = setupContext();
-        var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
-        topBundlesLoadDataStore.pushAsync("broker1",
-                getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceA", 1, 3));
-        topBundlesLoadDataStore.pushAsync("broker2",
-                getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceB", 2, 8));
-        topBundlesLoadDataStore.pushAsync("broker3",
-                getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceC", 6, 10));
-        topBundlesLoadDataStore.pushAsync("broker4",
-                getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceD", 10, 20));
-        topBundlesLoadDataStore.pushAsync("broker5",
-                getTopBundlesLoadWithOutSuffix("my-tenant/my-namespaceE", 70, 20));
+        doReturn(false).when(antiAffinityGroupPolicyHelper).canUnload(any(), any(), any(), any());
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
 
         var expected = new UnloadDecision();
@@ -600,6 +588,18 @@ public void testBundlesWithAntiAffinityGroup() throws IllegalAccessException, Me
         expected.setLoadAvg(setupLoadAvg);
         expected.setLoadStd(setupLoadStd);
         assertEquals(res, expected);
+
+        doReturn(true).when(antiAffinityGroupPolicyHelper).canUnload(any(), eq(bundleE1), any(), any());
+        var res2 = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
+        var expected2 = new UnloadDecision();
+        var unloads = expected2.getUnloads();
+        unloads.put("broker5",
+                new Unload("broker5", bundleE1, Optional.of("broker1")));
+        expected2.setLabel(Success);
+        expected2.setReason(Overloaded);
+        expected2.setLoadAvg(setupLoadAvg);
+        expected2.setLoadStd(setupLoadStd);
+        assertEquals(res2, expected2);
     }
 
     @Test
@@ -614,9 +614,9 @@ public void testTargetStd() {
 
         var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
 
-        topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("bundleA", 30, 30));
-        topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("bundleB", 40, 40));
-        topBundlesLoadDataStore.pushAsync("broker3", getTopBundlesLoad("bundleC", 50, 50));
+        topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 30, 30));
+        topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB", 40, 40));
+        topBundlesLoadDataStore.pushAsync("broker3", getTopBundlesLoad("my-tenant/my-namespaceC", 50, 50));
 
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
 
@@ -633,11 +633,11 @@ public void testSingleTopBundlesLoadData() {
         TransferShedder transferShedder = new TransferShedder();
         var ctx = setupContext();
         var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
-        topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("bundleA", 1));
-        topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("bundleB", 2));
-        topBundlesLoadDataStore.pushAsync("broker3", getTopBundlesLoad("bundleC", 6));
-        topBundlesLoadDataStore.pushAsync("broker4", getTopBundlesLoad("bundleD", 10));
-        topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("bundleE", 70));
+        topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 1));
+        topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB", 2));
+        topBundlesLoadDataStore.pushAsync("broker3", getTopBundlesLoad("my-tenant/my-namespaceC", 6));
+        topBundlesLoadDataStore.pushAsync("broker4", getTopBundlesLoad("my-tenant/my-namespaceD", 10));
+        topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 70));
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
 
         var expected = new UnloadDecision();
@@ -661,7 +661,7 @@ public void testTargetStdAfterTransfer() {
         var expected = new UnloadDecision();
         var unloads = expected.getUnloads();
         unloads.put("broker5",
-                new Unload("broker5", "bundleE-1", Optional.of("broker1")));
+                new Unload("broker5", bundleE1, Optional.of("broker1")));
         expected.setLabel(Success);
         expected.setReason(Overloaded);
         expected.setLoadAvg(0.26400000000000007);
@@ -687,9 +687,9 @@ public void testMinBrokerWithZeroTraffic() throws IllegalAccessException {
         var expected = new UnloadDecision();
         var unloads = expected.getUnloads();
         unloads.put("broker5",
-                new Unload("broker5", "bundleE-1", Optional.of("broker1")));
+                new Unload("broker5", bundleE1, Optional.of("broker1")));
         unloads.put("broker4",
-                new Unload("broker4", "bundleD-1", Optional.of("broker2")));
+                new Unload("broker4", bundleD1, Optional.of("broker2")));
         expected.setLabel(Success);
         expected.setReason(Underloaded);
         expected.setLoadAvg(0.26400000000000007);
@@ -708,9 +708,9 @@ public void testMaxNumberOfTransfersPerShedderCycle() {
         var expected = new UnloadDecision();
         var unloads = expected.getUnloads();
         unloads.put("broker5",
-                new Unload("broker5", "bundleE-1", Optional.of("broker1")));
+                new Unload("broker5", bundleE1, Optional.of("broker1")));
         unloads.put("broker4",
-                new Unload("broker4", "bundleD-1", Optional.of("broker2")));
+                new Unload("broker4", bundleD1, Optional.of("broker2")));
         expected.setLabel(Success);
         expected.setReason(Overloaded);
         expected.setLoadAvg(setupLoadAvg);
@@ -723,15 +723,15 @@ public void testRemainingTopBundles() {
         TransferShedder transferShedder = new TransferShedder();
         var ctx = setupContext();
         var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
-        topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("bundleE", 3000000, 2000000));
+        topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 3000000, 2000000));
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
 
         var expected = new UnloadDecision();
         var unloads = expected.getUnloads();
         unloads.put("broker5",
-                new Unload("broker5", "bundleE-1", Optional.of("broker1")));
+                new Unload("broker5", bundleE1, Optional.of("broker1")));
         unloads.put("broker4",
-                new Unload("broker4", "bundleD-1", Optional.of("broker2")));
+                new Unload("broker4", bundleD1, Optional.of("broker2")));
         expected.setLabel(Success);
         expected.setReason(Overloaded);
         expected.setLoadAvg(setupLoadAvg);

From 710cea6f6fa2ed11ac67280b000aaa067a767ea0 Mon Sep 17 00:00:00 2001
From: Qiang Zhao 
Date: Tue, 14 Mar 2023 16:48:21 +0800
Subject: [PATCH 108/174] [fix][meta] Fix close borrowed executor (#19761)

---
 .../pulsar/metadata/coordination/impl/LeaderElectionImpl.java   | 2 --
 1 file changed, 2 deletions(-)

diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java
index ad2a5bef70610..c1121b1309c2c 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java
@@ -253,8 +253,6 @@ public synchronized CompletableFuture asyncClose() {
 
         internalState = InternalState.Closed;
 
-        executor.shutdownNow();
-
         if (leaderElectionState != LeaderElectionState.Leading) {
             return CompletableFuture.completedFuture(null);
         }

From 4f6b5fee94eee39c7013cf79263d7c31c8c8793b Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Tue, 14 Mar 2023 03:53:45 -0500
Subject: [PATCH 109/174] [improve][misc] Add security section to PIP issue
 template (#19803)

---
 .github/ISSUE_TEMPLATE/pip.yml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/.github/ISSUE_TEMPLATE/pip.yml b/.github/ISSUE_TEMPLATE/pip.yml
index cd9aac33194e2..d494d69947252 100644
--- a/.github/ISSUE_TEMPLATE/pip.yml
+++ b/.github/ISSUE_TEMPLATE/pip.yml
@@ -60,6 +60,15 @@ body:
         This should also serve as documentation for any person that is trying to understand or debug the behavior of a certain feature.
     validations:
       required: true
+  - type: textarea
+    attributes:
+      label: Security Considerations
+      description: |
+        A detailed description of the security details that ought to be considered for the PIP. This is most relevant for any new HTTP endpoints, new Pulsar Protocol Commands, and new security features. The goal is to describe details like which role will have permission to perform an action.
+
+        If there is uncertainty for this section, please submit the PIP and request for feedback on the mailing list.
+    validations:
+      required: true
   - type: textarea
     attributes:
       label: Alternatives

From da3cab5289c41cfe1e064b02ce4794224aeaea04 Mon Sep 17 00:00:00 2001
From: Cong Zhao 
Date: Tue, 14 Mar 2023 22:06:21 +0800
Subject: [PATCH 110/174] [improve][broker][PIP-195] Cut off snapshot segment
 according to maxIndexesPerBucketSnapshotSegment (#19706)

---
 conf/broker.conf                              |  7 ++--
 conf/standalone.conf                          | 42 +++++++++++++++++++
 .../pulsar/broker/ServiceConfiguration.java   | 12 +++---
 .../BucketDelayedDeliveryTrackerFactory.java  |  8 +++-
 .../bucket/BucketDelayedDeliveryTracker.java  | 19 ++++++---
 .../broker/delayed/bucket/MutableBucket.java  | 10 +++--
 .../BucketDelayedDeliveryTrackerTest.java     | 41 ++++++++++++------
 .../persistent/BucketDelayedDeliveryTest.java |  1 +
 8 files changed, 109 insertions(+), 31 deletions(-)

diff --git a/conf/broker.conf b/conf/broker.conf
index 4b7c108be5f11..d52adb254563d 100644
--- a/conf/broker.conf
+++ b/conf/broker.conf
@@ -571,13 +571,14 @@ delayedDeliveryMinIndexCountPerBucket=50000
 # after reaching the max time step limitation, the snapshot segment will be cut off.
 delayedDeliveryMaxTimeStepPerBucketSnapshotSegmentSeconds=300
 
+# The max number of delayed message index in per bucket snapshot segment, -1 means no limitation
+# after reaching the max number limitation, the snapshot segment will be cut off.
+delayedDeliveryMaxIndexesPerBucketSnapshotSegment=5000
+
 # The max number of delayed message index bucket,
 # after reaching the max buckets limitation, the adjacent buckets will be merged.
 delayedDeliveryMaxNumBuckets=50
 
-# Enable share the delayed message index across subscriptions
-delayedDeliverySharedIndexEnabled=false
-
 # Size of the lookahead window to use when detecting if all the messages in the topic
 # have a fixed delay.
 # Default is 50,000. Setting the lookahead window to 0 will disable the logic to handle
diff --git a/conf/standalone.conf b/conf/standalone.conf
index ed883406883ed..f141946c29f4e 100644
--- a/conf/standalone.conf
+++ b/conf/standalone.conf
@@ -1223,3 +1223,45 @@ configurationStoreServers=
 # zookeeper.
 # Deprecated: use managedLedgerMaxUnackedRangesToPersistInMetadataStore
 managedLedgerMaxUnackedRangesToPersistInZooKeeper=-1
+
+# Whether to enable the delayed delivery for messages.
+# If disabled, messages will be immediately delivered and there will
+# be no tracking overhead.
+delayedDeliveryEnabled=true
+
+# Class name of the factory that implements the delayed deliver tracker.
+# If value is "org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTrackerFactory",
+# will create bucket based delayed message index tracker.
+delayedDeliveryTrackerFactoryClassName=org.apache.pulsar.broker.delayed.InMemoryDelayedDeliveryTrackerFactory
+
+# Control the tick time for when retrying on delayed delivery,
+# affecting the accuracy of the delivery time compared to the scheduled time.
+# Note that this time is used to configure the HashedWheelTimer's tick time for the
+# InMemoryDelayedDeliveryTrackerFactory (the default DelayedDeliverTrackerFactory).
+# Default is 1 second.
+delayedDeliveryTickTimeMillis=1000
+
+# When using the InMemoryDelayedDeliveryTrackerFactory (the default DelayedDeliverTrackerFactory), whether
+# the deliverAt time is strictly followed. When false (default), messages may be sent to consumers before the deliverAt
+# time by as much as the tickTimeMillis. This can reduce the overhead on the broker of maintaining the delayed index
+# for a potentially very short time period. When true, messages will not be sent to consumer until the deliverAt time
+# has passed, and they may be as late as the deliverAt time plus the tickTimeMillis for the topic plus the
+# delayedDeliveryTickTimeMillis.
+isDelayedDeliveryDeliverAtTimeStrict=false
+
+# The delayed message index bucket min index count.
+# When the index count of the current bucket is more than this value and all message indexes of current ledger
+# have already been added to the tracker we will seal the bucket.
+delayedDeliveryMinIndexCountPerBucket=50000
+
+# The delayed message index bucket time step(in seconds) in per bucket snapshot segment,
+# after reaching the max time step limitation, the snapshot segment will be cut off.
+delayedDeliveryMaxTimeStepPerBucketSnapshotSegmentSeconds=300
+
+# The max number of delayed message index in per bucket snapshot segment, -1 means no limitation
+# after reaching the max number limitation, the snapshot segment will be cut off.
+delayedDeliveryMaxIndexesPerBucketSnapshotSegment=5000
+
+# The max number of delayed message index bucket,
+# after reaching the max buckets limitation, the adjacent buckets will be merged.
+delayedDeliveryMaxNumBuckets=50
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index 3c00e905ac723..ff242888ae0b6 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -361,18 +361,20 @@ public class ServiceConfiguration implements PulsarConfiguration {
     private long delayedDeliveryMinIndexCountPerBucket = 50000;
 
     @FieldContext(category = CATEGORY_SERVER, doc = """
-            The delayed message index bucket time step(in seconds) in per bucket snapshot segment, \
+            The delayed message index time step(in seconds) in per bucket snapshot segment, \
             after reaching the max time step limitation, the snapshot segment will be cut off.""")
-    private long delayedDeliveryMaxTimeStepPerBucketSnapshotSegmentSeconds = 300;
+    private int delayedDeliveryMaxTimeStepPerBucketSnapshotSegmentSeconds = 300;
+
+    @FieldContext(category = CATEGORY_SERVER, doc = """
+            The max number of delayed message index in per bucket snapshot segment, -1 means no limitation\
+            after reaching the max number limitation, the snapshot segment will be cut off.""")
+    private int delayedDeliveryMaxIndexesPerBucketSnapshotSegment = 5000;
 
     @FieldContext(category = CATEGORY_SERVER, doc = """
             The max number of delayed message index bucket, \
             after reaching the max buckets limitation, the adjacent buckets will be merged.""")
     private int delayedDeliveryMaxNumBuckets = 50;
 
-    @FieldContext(category = CATEGORY_SERVER, doc = "Enable share the delayed message index across subscriptions")
-    private boolean delayedDeliverySharedIndexEnabled = false;
-
     @FieldContext(category = CATEGORY_SERVER, doc = "Size of the lookahead window to use "
             + "when detecting if all the messages in the topic have a fixed delay. "
             + "Default is 50,000. Setting the lookahead window to 0 will disable the "
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerFactory.java
index ae9cb23ceb922..f0feb8b27d6a1 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerFactory.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerFactory.java
@@ -43,7 +43,9 @@ public class BucketDelayedDeliveryTrackerFactory implements DelayedDeliveryTrack
 
     private long delayedDeliveryMinIndexCountPerBucket;
 
-    private long delayedDeliveryMaxTimeStepPerBucketSnapshotSegmentSeconds;
+    private int delayedDeliveryMaxTimeStepPerBucketSnapshotSegmentSeconds;
+
+    private int delayedDeliveryMaxIndexesPerBucketSnapshotSegment;
 
     @Override
     public void initialize(PulsarService pulsarService) throws Exception {
@@ -58,6 +60,8 @@ public void initialize(PulsarService pulsarService) throws Exception {
         this.delayedDeliveryMaxNumBuckets = config.getDelayedDeliveryMaxNumBuckets();
         this.delayedDeliveryMaxTimeStepPerBucketSnapshotSegmentSeconds =
                 config.getDelayedDeliveryMaxTimeStepPerBucketSnapshotSegmentSeconds();
+        this.delayedDeliveryMaxIndexesPerBucketSnapshotSegment =
+                config.getDelayedDeliveryMaxIndexesPerBucketSnapshotSegment();
     }
 
     @Override
@@ -65,7 +69,7 @@ public DelayedDeliveryTracker newTracker(PersistentDispatcherMultipleConsumers d
         return new BucketDelayedDeliveryTracker(dispatcher, timer, tickTimeMillis, isDelayedDeliveryDeliverAtTimeStrict,
                 bucketSnapshotStorage, delayedDeliveryMinIndexCountPerBucket,
                 TimeUnit.SECONDS.toMillis(delayedDeliveryMaxTimeStepPerBucketSnapshotSegmentSeconds),
-                delayedDeliveryMaxNumBuckets);
+                delayedDeliveryMaxIndexesPerBucketSnapshotSegment, delayedDeliveryMaxNumBuckets);
     }
 
     @Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
index a77b272297be4..22689cd737b2b 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
@@ -70,6 +70,8 @@ public class BucketDelayedDeliveryTracker extends AbstractDelayedDeliveryTracker
 
     private final long timeStepPerBucketSnapshotSegmentInMillis;
 
+    private final int maxIndexesPerBucketSnapshotSegment;
+
     private final int maxNumBuckets;
 
     private long numberDelayedMessages;
@@ -89,9 +91,10 @@ public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispat
                                  boolean isDelayedDeliveryDeliverAtTimeStrict,
                                  BucketSnapshotStorage bucketSnapshotStorage,
                                  long minIndexCountPerBucket, long timeStepPerBucketSnapshotSegmentInMillis,
-                                 int maxNumBuckets) {
+                                 int maxIndexesPerBucketSnapshotSegment, int maxNumBuckets) {
         this(dispatcher, timer, tickTimeMillis, Clock.systemUTC(), isDelayedDeliveryDeliverAtTimeStrict,
-                bucketSnapshotStorage, minIndexCountPerBucket, timeStepPerBucketSnapshotSegmentInMillis, maxNumBuckets);
+                bucketSnapshotStorage, minIndexCountPerBucket, timeStepPerBucketSnapshotSegmentInMillis,
+                maxIndexesPerBucketSnapshotSegment, maxNumBuckets);
     }
 
     public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher,
@@ -99,10 +102,11 @@ public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispat
                                  boolean isDelayedDeliveryDeliverAtTimeStrict,
                                  BucketSnapshotStorage bucketSnapshotStorage,
                                  long minIndexCountPerBucket, long timeStepPerBucketSnapshotSegmentInMillis,
-                                 int maxNumBuckets) {
+                                 int maxIndexesPerBucketSnapshotSegment, int maxNumBuckets) {
         super(dispatcher, timer, tickTimeMillis, clock, isDelayedDeliveryDeliverAtTimeStrict);
         this.minIndexCountPerBucket = minIndexCountPerBucket;
         this.timeStepPerBucketSnapshotSegmentInMillis = timeStepPerBucketSnapshotSegmentInMillis;
+        this.maxIndexesPerBucketSnapshotSegment = maxIndexesPerBucketSnapshotSegment;
         this.maxNumBuckets = maxNumBuckets;
         this.sharedBucketPriorityQueue = new TripleLongPriorityQueue();
         this.immutableBuckets = TreeRangeMap.create();
@@ -292,7 +296,9 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver
                 && lastMutableBucket.size() >= minIndexCountPerBucket
                 && !lastMutableBucket.isEmpty()) {
             Pair immutableBucketDelayedIndexPair =
-                    lastMutableBucket.sealBucketAndAsyncPersistent(this.timeStepPerBucketSnapshotSegmentInMillis,
+                    lastMutableBucket.sealBucketAndAsyncPersistent(
+                            this.timeStepPerBucketSnapshotSegmentInMillis,
+                            this.maxIndexesPerBucketSnapshotSegment,
                             this.sharedBucketPriorityQueue);
             afterCreateImmutableBucket(immutableBucketDelayedIndexPair);
             lastMutableBucket.resetLastMutableBucketRange();
@@ -380,8 +386,9 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot(ImmutableB
                 .thenAccept(combinedDelayedIndexQueue -> {
                     Pair immutableBucketDelayedIndexPair =
                             lastMutableBucket.createImmutableBucketAndAsyncPersistent(
-                                    timeStepPerBucketSnapshotSegmentInMillis, sharedBucketPriorityQueue,
-                                    combinedDelayedIndexQueue, bucketA.startLedgerId, bucketB.endLedgerId);
+                                    timeStepPerBucketSnapshotSegmentInMillis, maxIndexesPerBucketSnapshotSegment,
+                                    sharedBucketPriorityQueue, combinedDelayedIndexQueue, bucketA.startLedgerId,
+                                    bucketB.endLedgerId);
 
                     // Merge bit map to new bucket
                     Map delayedIndexBitMapA = bucketA.getDelayedIndexBitMap();
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
index 40ba8f4c4b593..8187f8f2de158 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
@@ -50,13 +50,15 @@ class MutableBucket extends Bucket implements AutoCloseable {
 
     Pair sealBucketAndAsyncPersistent(
             long timeStepPerBucketSnapshotSegment,
+            int maxIndexesPerBucketSnapshotSegment,
             TripleLongPriorityQueue sharedQueue) {
-        return createImmutableBucketAndAsyncPersistent(timeStepPerBucketSnapshotSegment, sharedQueue,
+        return createImmutableBucketAndAsyncPersistent(timeStepPerBucketSnapshotSegment,
+                maxIndexesPerBucketSnapshotSegment, sharedQueue,
                 TripleLongPriorityDelayedIndexQueue.wrap(priorityQueue), startLedgerId, endLedgerId);
     }
 
     Pair createImmutableBucketAndAsyncPersistent(
-            final long timeStepPerBucketSnapshotSegment,
+            final long timeStepPerBucketSnapshotSegment, final int maxIndexesPerBucketSnapshotSegment,
             TripleLongPriorityQueue sharedQueue, DelayedIndexQueue delayedIndexQueue, final long startLedgerId,
             final long endLedgerId) {
         log.info("[{}] Creating bucket snapshot, startLedgerId: {}, endLedgerId: {}", dispatcherName,
@@ -98,7 +100,9 @@ Pair createImmutableBucketAndAsyncPersistent(
 
             snapshotSegmentBuilder.addIndexes(delayedIndex);
 
-            if (delayedIndexQueue.isEmpty() || delayedIndexQueue.peek().getTimestamp() > currentTimestampUpperLimit) {
+            if (delayedIndexQueue.isEmpty() || delayedIndexQueue.peek().getTimestamp() > currentTimestampUpperLimit
+                    || (maxIndexesPerBucketSnapshotSegment != -1
+                    && snapshotSegmentBuilder.getIndexesCount() >= maxIndexesPerBucketSnapshotSegment)) {
                 segmentMetadataBuilder.setMaxScheduleTimestamp(timestamp);
                 currentTimestampUpperLimit = 0;
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
index 0ba9e5f4ca2e7..378ad205bb666 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
@@ -85,7 +85,7 @@ public Object[][] provider(Method method) throws Exception {
         return switch (methodName) {
             case "test" -> new Object[][]{{
                     new BucketDelayedDeliveryTracker(dispatcher, timer, 1, clock,
-                            false, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50)
+                            false, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1, 50)
             }};
             case "testWithTimer" -> {
                 Timer timer = mock(Timer.class);
@@ -113,39 +113,43 @@ public Object[][] provider(Method method) throws Exception {
 
                 yield new Object[][]{{
                         new BucketDelayedDeliveryTracker(dispatcher, timer, 1, clock,
-                                false, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50),
+                                false, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1, 50),
                         tasks
                 }};
             }
             case "testAddWithinTickTime" -> new Object[][]{{
                     new BucketDelayedDeliveryTracker(dispatcher, timer, 100, clock,
-                            false, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50)
+                            false, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1, 50)
             }};
             case "testAddMessageWithStrictDelay" -> new Object[][]{{
                     new BucketDelayedDeliveryTracker(dispatcher, timer, 100, clock,
-                            true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50)
+                            true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1, 50)
             }};
             case "testAddMessageWithDeliverAtTimeAfterNowBeforeTickTimeFrequencyWithStrict" -> new Object[][]{{
                     new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock,
-                            true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50)
+                            true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1, 50)
             }};
             case "testAddMessageWithDeliverAtTimeAfterNowAfterTickTimeFrequencyWithStrict", "testRecoverSnapshot" ->
                     new Object[][]{{
                             new BucketDelayedDeliveryTracker(dispatcher, timer, 100000, clock,
-                                    true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50)
+                                    true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1, 50)
                     }};
             case "testAddMessageWithDeliverAtTimeAfterFullTickTimeWithStrict", "testExistDelayedMessage" ->
                     new Object[][]{{
                             new BucketDelayedDeliveryTracker(dispatcher, timer, 500, clock,
-                                    true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50)
+                                    true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1, 50)
                     }};
             case "testMergeSnapshot", "testWithBkException", "testWithCreateFailDowngrade" -> new Object[][]{{
                     new BucketDelayedDeliveryTracker(dispatcher, timer, 100000, clock,
-                            true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 10)
+                            true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1, 10)
+            }};
+            case "testMaxIndexesPerSegment" -> new Object[][]{{
+                    new BucketDelayedDeliveryTracker(dispatcher, timer, 100000, clock,
+                            true, bucketSnapshotStorage, 20, TimeUnit.HOURS.toMillis(1), 5, 100)
             }};
             default -> new Object[][]{{
                     new BucketDelayedDeliveryTracker(dispatcher, timer, 1, clock,
-                            true, bucketSnapshotStorage, 1000, TimeUnit.MILLISECONDS.toMillis(100), 50)
+                            true, bucketSnapshotStorage, 1000, TimeUnit.MILLISECONDS.toMillis(100), -1, 50)
             }};
         };
     }
@@ -196,7 +200,7 @@ public void testRecoverSnapshot(BucketDelayedDeliveryTracker tracker) {
         clockTime.set(30 * 10);
 
         tracker = new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock,
-                true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 50);
+                true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1,50);
 
         assertFalse(tracker.containsMessage(101, 101));
         assertEquals(tracker.getNumberOfDelayedMessages(), 70);
@@ -268,7 +272,7 @@ public void testMergeSnapshot(BucketDelayedDeliveryTracker tracker) {
         tracker.close();
 
         tracker = new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock,
-                true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 10);
+                true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1,10);
 
         assertEquals(tracker.getNumberOfDelayedMessages(), delayedMessagesInSnapshot.getValue());
 
@@ -318,7 +322,7 @@ public void testWithBkException(BucketDelayedDeliveryTracker tracker) {
         tracker.close();
 
         tracker = new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock,
-                true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), 10);
+                true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1,10);
 
         Long delayedMessagesInSnapshotValue = delayedMessagesInSnapshot.getValue();
         assertEquals(tracker.getNumberOfDelayedMessages(), delayedMessagesInSnapshotValue);
@@ -374,4 +378,17 @@ public void testWithCreateFailDowngrade(BucketDelayedDeliveryTracker tracker) {
             assertEquals(position, PositionImpl.get(i, i));
         }
     }
+
+    @Test(dataProvider = "delayedTracker")
+    public void testMaxIndexesPerSegment(BucketDelayedDeliveryTracker tracker) {
+        for (int i = 1; i <= 101; i++) {
+            tracker.addMessage(i, i, i * 10);
+        }
+
+        assertEquals(tracker.getImmutableBuckets().asMapOfRanges().size(), 5);
+
+        tracker.getImmutableBuckets().asMapOfRanges().forEach((k, bucket) -> {
+            assertEquals(bucket.getLastSegmentEntryId(), 4);
+        });
+    }
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java
index 5d81ba8bc0261..fa846779b08ae 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java
@@ -32,6 +32,7 @@ public void setup() throws Exception {
         conf.setDelayedDeliveryTrackerFactoryClassName(BucketDelayedDeliveryTrackerFactory.class.getName());
         conf.setDelayedDeliveryMaxNumBuckets(10);
         conf.setDelayedDeliveryMaxTimeStepPerBucketSnapshotSegmentSeconds(1);
+        conf.setDelayedDeliveryMaxIndexesPerBucketSnapshotSegment(10);
         conf.setDelayedDeliveryMinIndexCountPerBucket(50);
         conf.setManagedLedgerMaxEntriesPerLedger(50);
         conf.setManagedLedgerMinLedgerRolloverTimeMinutes(0);

From 7a11edd4fda75c474b78f1952d13ef104797738a Mon Sep 17 00:00:00 2001
From: Asaf Mesika 
Date: Wed, 15 Mar 2023 05:53:41 +0200
Subject: [PATCH 111/174] [fix][doc] Amend PIP voting process description
 (#19810)

---
 wiki/proposals/PIP.md | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/wiki/proposals/PIP.md b/wiki/proposals/PIP.md
index f76c9f0f7a235..e10452b107fd8 100644
--- a/wiki/proposals/PIP.md
+++ b/wiki/proposals/PIP.md
@@ -82,7 +82,7 @@ The process works in the following way:
    the "xxx" number should be chosen to be the next number from the existing PIP 
    issues, listed [here]([url](https://github.com/apache/pulsar/labels/PIP)).
 2. The author(s) will send a note to the dev@pulsar.apache.org mailing list
-   to start the discussion, using subject prefix `[DISCUSS] PIP-xxx: `. The discussion
+   to start the discussion, using subject prefix `[DISCUSS] PIP-xxx: {PIP TITLE}`. The discussion
    need to happen in the mailing list. Please avoid discussing it using
    GitHub comments in the PIP GitHub issue, as it creates two tracks 
    of feedback.
@@ -90,10 +90,11 @@ The process works in the following way:
    authors to the text of the proposal.
 4. Once some consensus is reached, there will be a vote to formally approve
    the proposal.
-   The vote will be held on the dev@pulsar.apache.org mailing list. Everyone
-   is welcome to vote on the proposal, though it will be considered to be binding
-   only the vote of PMC members.
-   I would be required to have a lazy majority of at least 3 binding +1s votes.
+   The vote will be held on the dev@pulsar.apache.org mailing list, by
+   sending a message using subject `[VOTE] PIP-xxx: {PIP TITLE}".
+   Everyone is welcome to vote on the proposal, though only the the vote of the PMC 
+   members will be considered binding.
+   It is required to have a lazy majority of at least 3 binding +1s votes.
    The vote should stay open for at least 48 hours.
 5. When the vote is closed, if the outcome is positive, the state of the
    proposal is updated, and the Pull Requests associated with this proposal can

From 0e96dedf66c1a0961bf5de584ebd0164fec179d9 Mon Sep 17 00:00:00 2001
From: Cong Zhao 
Date: Wed, 15 Mar 2023 12:02:04 +0800
Subject: [PATCH 112/174] [fix][broker][PIP-195] Don't clean up
 BucketDelayedDeliveryTracker when all consumer disconnect (#19801)

---
 .../pulsar/broker/delayed/bucket/Bucket.java  |  3 +-
 .../bucket/BucketDelayedDeliveryTracker.java  |  4 ++
 .../delayed/bucket/ImmutableBucket.java       | 67 ++++++++++---------
 .../broker/delayed/bucket/MutableBucket.java  |  1 +
 ...PersistentDispatcherMultipleConsumers.java |  8 ++-
 .../BucketDelayedDeliveryTrackerTest.java     | 18 +++++
 .../persistent/BucketDelayedDeliveryTest.java | 59 ++++++++++++++++
 7 files changed, 128 insertions(+), 32 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java
index 50b5cd12ead07..db864a4e264f5 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pulsar.broker.delayed.bucket;
 
+import static org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.CURSOR_INTERNAL_PROPERTY_PREFIX;
 import static org.apache.bookkeeper.mledger.util.Futures.executeWithRetry;
 import java.util.HashMap;
 import java.util.List;
@@ -38,7 +39,7 @@
 @AllArgsConstructor
 abstract class Bucket {
 
-    static final String DELAYED_BUCKET_KEY_PREFIX = "#pulsar.internal.delayed.bucket";
+    static final String DELAYED_BUCKET_KEY_PREFIX = CURSOR_INTERNAL_PROPERTY_PREFIX + "delayed.bucket";
     static final String DELIMITER = "_";
     static final int MaxRetryTimes = 3;
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
index 22689cd737b2b..31fdaa6fb7667 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
@@ -76,8 +76,12 @@ public class BucketDelayedDeliveryTracker extends AbstractDelayedDeliveryTracker
 
     private long numberDelayedMessages;
 
+    @Getter
+    @VisibleForTesting
     private final MutableBucket lastMutableBucket;
 
+    @Getter
+    @VisibleForTesting
     private final TripleLongPriorityQueue sharedBucketPriorityQueue;
 
     @Getter
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java
index c947e53843a85..c9223efa09243 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java
@@ -20,6 +20,7 @@
 
 import static org.apache.bookkeeper.mledger.util.Futures.executeWithRetry;
 import static org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker.AsyncOperationTimeoutSeconds;
+import static org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker.NULL_LONG_PROMISE;
 import com.google.protobuf.ByteString;
 import java.util.Collections;
 import java.util.List;
@@ -167,15 +168,10 @@ CompletableFuture asyncDeleteBucketSnapshot() {
         String bucketKey = bucketKey();
         long bucketId = getAndUpdateBucketId();
         return removeBucketCursorProperty(bucketKey).thenCompose(__ ->
-                executeWithRetry(() -> bucketSnapshotStorage.deleteBucketSnapshot(bucketId).whenComplete((___, ex) -> {
-                            if (ex != null) {
-                                log.warn("[{}] Failed to delete bucket snapshot. bucketKey: {}, bucketId: {}",
-                                        dispatcherName, bucketKey, bucketId, ex);
-                            }
-                        }),
+                executeWithRetry(() -> bucketSnapshotStorage.deleteBucketSnapshot(bucketId),
                         BucketSnapshotPersistenceException.class, MaxRetryTimes)).whenComplete((__, ex) -> {
                     if (ex != null) {
-                        log.warn("[{}] Failed to delete bucket snapshot, bucketId: {}, bucketKey: {}",
+                        log.error("[{}] Failed to delete bucket snapshot, bucketId: {}, bucketKey: {}",
                                 dispatcherName, bucketId, bucketKey, ex);
                     } else {
                         log.info("[{}] Delete bucket snapshot finish, bucketId: {}, bucketKey: {}",
@@ -186,35 +182,46 @@ CompletableFuture asyncDeleteBucketSnapshot() {
 
     void clear(boolean delete) {
         delayedIndexBitMap.clear();
-        getSnapshotCreateFuture().ifPresent(snapshotGenerateFuture -> {
-            if (delete) {
-                snapshotGenerateFuture.cancel(true);
-                String bucketKey = bucketKey();
-                long bucketId = getAndUpdateBucketId();
-                try {
-                    // Because bucketSnapshotStorage.deleteBucketSnapshot may be use the same thread with clear,
-                    // so we can't block deleteBucketSnapshot when clearing the bucket snapshot.
-                    removeBucketCursorProperty(bucketKey())
-                            .thenApply(__ -> bucketSnapshotStorage.deleteBucketSnapshot(bucketId).exceptionally(ex -> {
-                                log.error("Failed to delete bucket snapshot, bucketId: {}, bucketKey: {}",
-                                        bucketId, bucketKey, ex);
-                                return null;
-                            })).get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS);
-                } catch (Exception e) {
-                    log.error("Failed to delete bucket snapshot, bucketId: {}, bucketKey: {}", bucketId, bucketKey, e);
-                    if (e instanceof InterruptedException) {
-                        Thread.currentThread().interrupt();
-                    }
-                    throw new RuntimeException(e);
+        if (delete) {
+            final String bucketKey = bucketKey();
+            try {
+                getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE).exceptionally(e -> null).thenCompose(__ -> {
+                        if (getSnapshotCreateFuture().isPresent() && getBucketId().isEmpty()) {
+                            log.error("[{}] Can't found bucketId, don't execute delete operate, bucketKey: {}",
+                                    dispatcherName, bucketKey);
+                            return CompletableFuture.completedFuture(null);
+                        }
+                        long bucketId = getAndUpdateBucketId();
+                        return removeBucketCursorProperty(bucketKey()).thenAccept(___ -> {
+                            executeWithRetry(() -> bucketSnapshotStorage.deleteBucketSnapshot(bucketId),
+                            BucketSnapshotPersistenceException.class, MaxRetryTimes)
+                            .whenComplete((____, ex) -> {
+                                if (ex != null) {
+                                    log.error("[{}] Failed to delete bucket snapshot, bucketId: {}, bucketKey: {}",
+                                            dispatcherName, bucketId, bucketKey, ex);
+                                } else {
+                                    log.info("[{}] Delete bucket snapshot finish, bucketId: {}, bucketKey: {}",
+                                            dispatcherName, bucketId, bucketKey);
+                                }
+                            });
+                    });
+                }).get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS);
+            } catch (Exception e) {
+                log.error("Failed to clear bucket snapshot, bucketKey: {}", bucketKey, e);
+                if (e instanceof InterruptedException) {
+                    Thread.currentThread().interrupt();
                 }
-            } else {
+                throw new RuntimeException(e);
+            }
+        } else {
+            getSnapshotCreateFuture().ifPresent(snapshotGenerateFuture -> {
                 try {
                     snapshotGenerateFuture.get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS);
                 } catch (Exception e) {
                     log.warn("[{}] Failed wait to snapshot generate, bucketId: {}, bucketKey: {}", dispatcherName,
                             getBucketId(), bucketKey());
                 }
-            }
-        });
+            });
+        }
     }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
index 8187f8f2de158..e743f39e6920d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
@@ -175,6 +175,7 @@ void resetLastMutableBucketRange() {
     void clear() {
         this.resetLastMutableBucketRange();
         this.delayedIndexBitMap.clear();
+        this.priorityQueue.clear();
     }
 
     public void close() {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
index 475ba39c51129..059820b1b66c3 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
@@ -49,6 +49,7 @@
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.pulsar.broker.delayed.AbstractDelayedDeliveryTracker;
 import org.apache.pulsar.broker.delayed.DelayedDeliveryTracker;
+import org.apache.pulsar.broker.delayed.InMemoryDelayedDeliveryTracker;
 import org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker;
 import org.apache.pulsar.broker.service.AbstractDispatcherMultipleConsumers;
 import org.apache.pulsar.broker.service.BrokerServiceException;
@@ -165,7 +166,12 @@ public synchronized void addConsumer(Consumer consumer) throws BrokerServiceExce
                 shouldRewindBeforeReadingOrReplaying = false;
             }
             redeliveryMessages.clear();
-            delayedDeliveryTracker.ifPresent(DelayedDeliveryTracker::clear);
+            delayedDeliveryTracker.ifPresent(tracker -> {
+                // Don't clean up BucketDelayedDeliveryTracker, otherwise we will lose the bucket snapshot
+                if (tracker instanceof InMemoryDelayedDeliveryTracker) {
+                    tracker.clear();
+                }
+            });
         }
 
         if (isConsumersExceededOnSubscription()) {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
index 378ad205bb666..c834bc7a0b4c5 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
@@ -391,4 +391,22 @@ public void testMaxIndexesPerSegment(BucketDelayedDeliveryTracker tracker) {
             assertEquals(bucket.getLastSegmentEntryId(), 4);
         });
     }
+    
+    @Test(dataProvider = "delayedTracker")
+    public void testClear(BucketDelayedDeliveryTracker tracker) {
+      for (int i = 1; i <= 1001; i++) {
+          tracker.addMessage(i, i, i * 10);
+      }
+
+      assertEquals(tracker.getNumberOfDelayedMessages(), 1001);
+      assertTrue(tracker.getImmutableBuckets().asMapOfRanges().size() > 0);
+      assertEquals(tracker.getLastMutableBucket().size(), 1);
+
+      tracker.clear();
+
+      assertEquals(tracker.getNumberOfDelayedMessages(), 0);
+      assertEquals(tracker.getImmutableBuckets().asMapOfRanges().size(), 0);
+      assertEquals(tracker.getLastMutableBucket().size(), 0);
+      assertEquals(tracker.getSharedBucketPriorityQueue().size(), 0);
+    }
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java
index fa846779b08ae..292889e8c159a 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java
@@ -18,7 +18,19 @@
  */
 package org.apache.pulsar.broker.service.persistent;
 
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import lombok.Cleanup;
+import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl;
+import org.apache.pulsar.broker.BrokerTestUtil;
 import org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTrackerFactory;
+import org.apache.pulsar.broker.service.Dispatcher;
+import org.apache.pulsar.client.api.Consumer;
+import org.apache.pulsar.client.api.Producer;
+import org.apache.pulsar.client.api.Schema;
+import org.apache.pulsar.client.api.SubscriptionType;
+import org.awaitility.Awaitility;
+import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
@@ -44,4 +56,51 @@ public void setup() throws Exception {
     public void cleanup() throws Exception {
         super.internalCleanup();
     }
+
+    @Test
+    public void testBucketDelayedDeliveryWithAllConsumersDisconnecting() throws Exception {
+        String topic = BrokerTestUtil.newUniqueName("persistent://public/default/testDelaysWithAllConsumerDis");
+
+        Consumer c1 = pulsarClient.newConsumer(Schema.STRING)
+                .topic(topic)
+                .subscriptionName("sub")
+                .subscriptionType(SubscriptionType.Shared)
+                .subscribe();
+
+        @Cleanup
+        Producer producer = pulsarClient.newProducer(Schema.STRING)
+                .topic(topic)
+                .create();
+
+        for (int i = 0; i < 1000; i++) {
+            producer.newMessage()
+                    .value("msg")
+                    .deliverAfter(1, TimeUnit.HOURS)
+                    .send();
+        }
+
+        Dispatcher dispatcher = pulsar.getBrokerService().getTopicReference(topic).get().getSubscription("sub").getDispatcher();
+        Awaitility.await().untilAsserted(() -> Assert.assertEquals(dispatcher.getNumberOfDelayedMessages(), 1000));
+        List bucketKeys =
+                ((PersistentDispatcherMultipleConsumers) dispatcher).getCursor().getCursorProperties().keySet().stream()
+                        .filter(x -> x.startsWith(ManagedCursorImpl.CURSOR_INTERNAL_PROPERTY_PREFIX)).toList();
+
+        c1.close();
+
+        // Attach a new consumer. Since there are no consumers connected, this will trigger the cursor rewind
+        @Cleanup
+        Consumer c2 = pulsarClient.newConsumer(Schema.STRING)
+                .topic(topic)
+                .subscriptionName("sub")
+                .subscriptionType(SubscriptionType.Shared)
+                .subscribe();
+
+        Dispatcher dispatcher2 = pulsar.getBrokerService().getTopicReference(topic).get().getSubscription("sub").getDispatcher();
+        List bucketKeys2 =
+                ((PersistentDispatcherMultipleConsumers) dispatcher2).getCursor().getCursorProperties().keySet().stream()
+                        .filter(x -> x.startsWith(ManagedCursorImpl.CURSOR_INTERNAL_PROPERTY_PREFIX)).toList();
+
+        Awaitility.await().untilAsserted(() -> Assert.assertEquals(dispatcher2.getNumberOfDelayedMessages(), 1000));
+        Assert.assertEquals(bucketKeys, bucketKeys2);
+    }
 }

From 160a8643a1f6a153106eac9031ac2a9530ac4698 Mon Sep 17 00:00:00 2001
From: Cong Zhao 
Date: Wed, 15 Mar 2023 13:32:31 +0800
Subject: [PATCH 113/174] [improve][broker][PIP-195] Add topicName and
 cursorName for ledger metadata of bucket snapshot (#19802)

---
 .../mledger/impl/LedgerMetadataUtils.java         | 15 +++++++++++----
 .../bucket/BookkeeperBucketSnapshotStorage.java   |  9 +++++----
 .../pulsar/broker/delayed/bucket/Bucket.java      |  6 +++++-
 .../delayed/bucket/BucketSnapshotStorage.java     |  8 +++++---
 .../pulsar/broker/service/BrokerService.java      |  2 ++
 .../BookkeeperBucketSnapshotStorageTest.java      | 13 ++++++++-----
 .../broker/delayed/MockBucketSnapshotStorage.java |  3 ++-
 .../bucket/BucketDelayedDeliveryTrackerTest.java  |  2 +-
 8 files changed, 39 insertions(+), 19 deletions(-)

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/LedgerMetadataUtils.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/LedgerMetadataUtils.java
index 8571a36584e2b..4ac409a2e9bfe 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/LedgerMetadataUtils.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/LedgerMetadataUtils.java
@@ -48,7 +48,9 @@ public final class LedgerMetadataUtils {
     private static final String METADATA_PROPERTY_COMPACTEDTO = "pulsar/compactedTo";
     private static final String METADATA_PROPERTY_SCHEMAID = "pulsar/schemaId";
 
-    private static final String METADATA_PROPERTY_DELAYED_INDEX_BUCKETID = "pulsar/delayedIndexBucketId";
+    private static final String METADATA_PROPERTY_DELAYED_INDEX_BUCKET_KEY = "pulsar/delayedIndexBucketKey";
+    private static final String METADATA_PROPERTY_DELAYED_INDEX_TOPIC = "pulsar/delayedIndexTopic";
+    private static final String METADATA_PROPERTY_DELAYED_INDEX_CURSOR = "pulsar/delayedIndexCursor";
 
     /**
      * Build base metadata for every ManagedLedger.
@@ -108,14 +110,19 @@ public static Map buildMetadataForSchema(String schemaId) {
     /**
      * Build additional metadata for a delayed message index bucket.
      *
-     * @param bucketKey key of the delayed message bucket
+     * @param bucketKey  key of the delayed message bucket
+     * @param topicName  name of the topic
+     * @param cursorName name of the cursor
      * @return an immutable map which describes the schema
      */
-    public static Map buildMetadataForDelayedIndexBucket(String bucketKey) {
+    public static Map buildMetadataForDelayedIndexBucket(String bucketKey,
+                                                                         String topicName, String cursorName) {
         return Map.of(
                 METADATA_PROPERTY_APPLICATION, METADATA_PROPERTY_APPLICATION_PULSAR,
                 METADATA_PROPERTY_COMPONENT, METADATA_PROPERTY_COMPONENT_DELAYED_INDEX_BUCKET,
-                METADATA_PROPERTY_DELAYED_INDEX_BUCKETID, bucketKey.getBytes(StandardCharsets.UTF_8)
+                METADATA_PROPERTY_DELAYED_INDEX_BUCKET_KEY, bucketKey.getBytes(StandardCharsets.UTF_8),
+                METADATA_PROPERTY_DELAYED_INDEX_TOPIC, topicName.getBytes(StandardCharsets.UTF_8),
+                METADATA_PROPERTY_DELAYED_INDEX_CURSOR, cursorName.getBytes(StandardCharsets.UTF_8)
         );
     }
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java
index 7dd6266e2115c..08202bb19155d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java
@@ -56,8 +56,8 @@ public BookkeeperBucketSnapshotStorage(PulsarService pulsar) {
     @Override
     public CompletableFuture createBucketSnapshot(SnapshotMetadata snapshotMetadata,
                                                         List bucketSnapshotSegments,
-                                                        String bucketKey) {
-        return createLedger(bucketKey)
+                                                        String bucketKey, String topicName, String cursorName) {
+        return createLedger(bucketKey, topicName, cursorName)
                 .thenCompose(ledgerHandle -> addEntry(ledgerHandle, snapshotMetadata.toByteArray())
                         .thenCompose(__ -> addSnapshotSegments(ledgerHandle, bucketSnapshotSegments))
                         .thenCompose(__ -> closeLedger(ledgerHandle))
@@ -143,9 +143,10 @@ private List parseSnapshotSegmentEntries(Enumeration createLedger(String bucketKey) {
+    private CompletableFuture createLedger(String bucketKey, String topicName, String cursorName) {
         CompletableFuture future = new CompletableFuture<>();
-        Map metadata = LedgerMetadataUtils.buildMetadataForDelayedIndexBucket(bucketKey);
+        Map metadata = LedgerMetadataUtils.buildMetadataForDelayedIndexBucket(bucketKey,
+                topicName, cursorName);
         bookKeeper.asyncCreateLedger(
                 config.getManagedLedgerDefaultEnsembleSize(),
                 config.getManagedLedgerDefaultWriteQuorum(),
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java
index db864a4e264f5..7cfccff7ba328 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java
@@ -32,6 +32,7 @@
 import org.apache.bookkeeper.mledger.ManagedCursor;
 import org.apache.bookkeeper.mledger.ManagedLedgerException;
 import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat;
+import org.apache.pulsar.common.util.Codec;
 import org.roaringbitmap.RoaringBitmap;
 
 @Slf4j
@@ -130,8 +131,11 @@ CompletableFuture asyncSaveBucketSnapshot(
             ImmutableBucket bucket, DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata,
             List bucketSnapshotSegments) {
         final String bucketKey = bucket.bucketKey();
+        final String cursorName = Codec.decode(cursor.getName());
+        final String topicName = dispatcherName.substring(0, dispatcherName.lastIndexOf(" / " + cursorName));
         return executeWithRetry(
-                () -> bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata, bucketSnapshotSegments, bucketKey)
+                () -> bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata, bucketSnapshotSegments, bucketKey,
+                                topicName, cursorName)
                         .whenComplete((__, ex) -> {
                             if (ex != null) {
                                 log.warn("[{}] Failed to create bucket snapshot, bucketKey: {}",
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketSnapshotStorage.java
index c6941e289f1ac..51c89bed47af2 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketSnapshotStorage.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketSnapshotStorage.java
@@ -28,14 +28,16 @@ public interface BucketSnapshotStorage {
     /**
      * Create a delayed message index bucket snapshot with metadata and bucketSnapshotSegments.
      *
-     * @param snapshotMetadata the metadata of snapshot
+     * @param snapshotMetadata       the metadata of snapshot
      * @param bucketSnapshotSegments the list of snapshot segments
-     * @param bucketKey the key of bucket is used to generate custom storage metadata
+     * @param bucketKey              the key of bucket is used to generate custom storage metadata
+     * @param topicName              the name of topic is used to generate custom storage metadata
+     * @param cursorName             the name of cursor is used to generate custom storage metadata
      * @return the future with bucketId(ledgerId).
      */
     CompletableFuture createBucketSnapshot(SnapshotMetadata snapshotMetadata,
                                                  List bucketSnapshotSegments,
-                                                 String bucketKey);
+                                                 String bucketKey, String topicName, String cursorName);
 
     /**
      * Get delayed message index bucket snapshot metadata.
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
index 15d48e59849b6..a6345f4a56a71 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
@@ -261,6 +261,8 @@ public class BrokerService implements Closeable {
     private final ConcurrentOpenHashSet blockedDispatchers;
     private final ReadWriteLock lock = new ReentrantReadWriteLock();
 
+    @Getter
+    @VisibleForTesting
     private final DelayedDeliveryTrackerFactory delayedDeliveryTrackerFactory;
     private final ServerBootstrap defaultServerBootstrap;
     private final List protocolHandlersWorkerGroups = new ArrayList<>();
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java
index 1effe756ff2ab..72052d22b859d 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java
@@ -54,6 +54,9 @@ protected void cleanup() throws Exception {
         bucketSnapshotStorage.close();
     }
 
+    private static final String TOPIC_NAME = "topicName";
+    private static final String CURSOR_NAME = "sub";
+
     @Test
     public void testCreateSnapshot() throws ExecutionException, InterruptedException {
         DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata =
@@ -61,7 +64,7 @@ public void testCreateSnapshot() throws ExecutionException, InterruptedException
         List bucketSnapshotSegments = new ArrayList<>();
         CompletableFuture future =
                 bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata,
-                        bucketSnapshotSegments, UUID.randomUUID().toString());
+                        bucketSnapshotSegments, UUID.randomUUID().toString(), TOPIC_NAME, CURSOR_NAME);
         Long bucketId = future.get();
         Assert.assertNotNull(bucketId);
     }
@@ -90,7 +93,7 @@ public void testGetSnapshot() throws ExecutionException, InterruptedException {
 
         CompletableFuture future =
                 bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata,
-                        bucketSnapshotSegments, UUID.randomUUID().toString());
+                        bucketSnapshotSegments, UUID.randomUUID().toString(), TOPIC_NAME, CURSOR_NAME);
         Long bucketId = future.get();
         Assert.assertNotNull(bucketId);
 
@@ -129,7 +132,7 @@ public void testGetSnapshotMetadata() throws ExecutionException, InterruptedExce
 
         CompletableFuture future =
                 bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata,
-                        bucketSnapshotSegments, UUID.randomUUID().toString());
+                        bucketSnapshotSegments, UUID.randomUUID().toString(), TOPIC_NAME, CURSOR_NAME);
         Long bucketId = future.get();
         Assert.assertNotNull(bucketId);
 
@@ -151,7 +154,7 @@ public void testDeleteSnapshot() throws ExecutionException, InterruptedException
         List bucketSnapshotSegments = new ArrayList<>();
         CompletableFuture future =
                 bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata,
-                        bucketSnapshotSegments, UUID.randomUUID().toString());
+                        bucketSnapshotSegments, UUID.randomUUID().toString(), TOPIC_NAME, CURSOR_NAME);
         Long bucketId = future.get();
         Assert.assertNotNull(bucketId);
 
@@ -189,7 +192,7 @@ public void testGetBucketSnapshotLength() throws ExecutionException, Interrupted
 
         CompletableFuture future =
                 bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata,
-                        bucketSnapshotSegments, UUID.randomUUID().toString());
+                        bucketSnapshotSegments, UUID.randomUUID().toString(), TOPIC_NAME, CURSOR_NAME);
         Long bucketId = future.get();
         Assert.assertNotNull(bucketId);
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java
index b106642915587..cf7310c7067c3 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java
@@ -80,7 +80,8 @@ public void injectDeleteException(Throwable throwable) {
 
     @Override
     public CompletableFuture createBucketSnapshot(
-            SnapshotMetadata snapshotMetadata, List bucketSnapshotSegments, String bucketKey) {
+            SnapshotMetadata snapshotMetadata, List bucketSnapshotSegments, String bucketKey,
+            String topicName, String cursorName) {
         Throwable throwable = createExceptionQueue.poll();
         if (throwable != null) {
             return FutureUtil.failedFuture(throwable);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
index c834bc7a0b4c5..74101f00b960c 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
@@ -79,7 +79,7 @@ public Object[][] provider(Method method) throws Exception {
         bucketSnapshotStorage.start();
         ManagedCursor cursor = new MockManagedCursor("my_test_cursor");
         doReturn(cursor).when(dispatcher).getCursor();
-        doReturn(cursor.getName()).when(dispatcher).getName();
+        doReturn("persistent://public/default/testDelay" + " / " + cursor.getName()).when(dispatcher).getName();
 
         final String methodName = method.getName();
         return switch (methodName) {

From 11cfaea0deefc3c5d32f6504d266f84d12f89641 Mon Sep 17 00:00:00 2001
From: maheshnikam <55378196+nikam14@users.noreply.github.com>
Date: Wed, 15 Mar 2023 13:01:48 +0530
Subject: [PATCH 114/174] [improve][doc] enhancement of link (#19819)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 15419a754cfa9..fdbf7c7339b1e 100644
--- a/README.md
+++ b/README.md
@@ -254,7 +254,7 @@ Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/L
 
 ## Crypto Notice
 
-This distribution includes cryptographic software. The country in which you currently reside may have restrictions on the import, possession, use, and/or re-export to another country, of encryption software. BEFORE using any encryption software, please check your country's laws, regulations and policies concerning the import, possession, or use, and re-export of encryption software, to see if this is permitted. See  for more information.
+This distribution includes cryptographic software. The country in which you currently reside may have restrictions on the import, possession, use, and/or re-export to another country, of encryption software. BEFORE using any encryption software, please check your country's laws, regulations and policies concerning the import, possession, or use, and re-export of encryption software, to see if this is permitted. See [The Wassenaar Arrangement](http://www.wassenaar.org/) for more information.
 
 The U.S. Government Department of Commerce, Bureau of Industry and Security (BIS), has classified this software as Export Commodity Control Number (ECCN) 5D002.C.1, which includes information security software using or performing cryptographic functions with asymmetric algorithms. The form and manner of this Apache Software Foundation distribution makes it eligible for export under the License Exception ENC Technology Software Unrestricted (TSU) exception (see the BIS Export Administration Regulations, Section 740.13) for both object code and source code.
 

From 4e48cc30bfa6bf4d01659aac34d510f8b1492a10 Mon Sep 17 00:00:00 2001
From: lifepuzzlefun 
Date: Wed, 15 Mar 2023 15:57:07 +0800
Subject: [PATCH 115/174] [fix] [test] Fix flaky test
 `MetadataStoreStatsTest.testBatchMetadataStoreMetrics` (#19793)

Co-authored-by: wangjinlong 
---
 .../MultiBrokerMetadataConsistencyTest.java   | 33 ++++++++++++++++---
 1 file changed, 29 insertions(+), 4 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/zookeeper/MultiBrokerMetadataConsistencyTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/zookeeper/MultiBrokerMetadataConsistencyTest.java
index 9eeef94572ee8..c998a1ba57c35 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/zookeeper/MultiBrokerMetadataConsistencyTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/zookeeper/MultiBrokerMetadataConsistencyTest.java
@@ -19,6 +19,7 @@
 package org.apache.pulsar.broker.zookeeper;
 
 import static org.testng.Assert.assertTrue;
+import java.util.ArrayList;
 import java.util.List;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.pulsar.broker.MultiBrokerBaseTest;
@@ -43,6 +44,9 @@ protected int numberOfAdditionalBrokers() {
 
     TestZKServer testZKServer;
 
+    private final List needCloseStore =
+            new ArrayList<>();
+
     @Override
     protected void doInitConf() throws Exception {
         super.doInitConf();
@@ -59,20 +63,41 @@ protected void onCleanup() {
                 log.error("Error in stopping ZK server", e);
             }
         }
+
+        needCloseStore.forEach((storeExtended) -> {
+            try {
+                storeExtended.close();
+            } catch (Exception e) {
+                log.error("error when close storeExtended", e);
+            }
+        });
+
+        needCloseStore.clear();
     }
 
     @Override
     protected PulsarTestContext.Builder createPulsarTestContextBuilder(ServiceConfiguration conf) {
+        MetadataStoreExtended metadataStore = createMetadataStore(
+                MultiBrokerMetadataConsistencyTest.class.getName()
+                        + "metadata_store");
+
+        MetadataStoreExtended configurationStore = createMetadataStore(
+                MultiBrokerMetadataConsistencyTest.class.getName()
+                        + "configuration_store");
+
+        needCloseStore.add(metadataStore);
+        needCloseStore.add(configurationStore);
+
         return super.createPulsarTestContextBuilder(conf)
-                .localMetadataStore(createMetadataStore())
-                .configurationMetadataStore(createMetadataStore());
+                .localMetadataStore(metadataStore)
+                .configurationMetadataStore(configurationStore);
     }
 
     @NotNull
-    protected MetadataStoreExtended createMetadataStore()  {
+    protected MetadataStoreExtended createMetadataStore(String name) {
         try {
             return MetadataStoreExtended.create(testZKServer.getConnectionString(),
-                    MetadataStoreConfig.builder().build());
+                    MetadataStoreConfig.builder().metadataStoreName(name).build());
         } catch (MetadataStoreException e) {
             throw new RuntimeException(e);
         }

From 8cff1238465ca8b1315d8b5b48388d4b61ec74b6 Mon Sep 17 00:00:00 2001
From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com>
Date: Wed, 15 Mar 2023 17:22:12 -0700
Subject: [PATCH 116/174] [improve][broker] PIP-192 Made only the leader
 consume TopBundlesLoadDataStore (#19730)

Master Issue: https://github.com/apache/pulsar/issues/16691

### Motivation

Raising a PR to implement https://github.com/apache/pulsar/issues/16691.

Based on the current design, only the leader needs to consume the topk bundle load data.

### Modifications
This PR
- made only the leader consume TopBundlesLoadDataStore
- moved LeaderElectionService to ExtensibleLoadManager from ServiceUnitStateChannel.
---
 .../extensions/ExtensibleLoadManagerImpl.java | 113 +++++++++++-
 .../channel/ServiceUnitStateChannel.java      |   9 +
 .../channel/ServiceUnitStateChannelImpl.java  |  79 ++++----
 .../extensions/scheduler/UnloadScheduler.java |  11 +-
 .../extensions/store/LoadDataStore.java       |  12 ++
 .../store/TableViewLoadDataStoreImpl.java     |  46 ++++-
 .../ExtensibleLoadManagerImplTest.java        | 169 ++++++++++++++++--
 .../channel/ServiceUnitStateChannelTest.java  |  15 ++
 .../filter/BrokerFilterTestBase.java          |  11 ++
 .../scheduler/TransferShedderTest.java        |  21 +++
 .../extensions/store/LoadDataStoreTest.java   |  27 +++
 .../LeastResourceUsageWithWeightTest.java     |  12 ++
 12 files changed, 465 insertions(+), 60 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
index 7895f06fd5088..716be3718bf19 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
@@ -18,6 +18,9 @@
  */
 package org.apache.pulsar.broker.loadbalance.extensions;
 
+import static org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.Role.Follower;
+import static org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.Role.Leader;
+import com.google.common.annotations.VisibleForTesting;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -26,6 +29,7 @@
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
@@ -35,6 +39,7 @@
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.loadbalance.BrokerFilterException;
+import org.apache.pulsar.broker.loadbalance.LeaderElectionService;
 import org.apache.pulsar.broker.loadbalance.LoadManager;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl;
@@ -70,6 +75,7 @@
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.stats.Metrics;
 import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap;
+import org.apache.pulsar.metadata.api.coordination.LeaderElectionState;
 
 @Slf4j
 public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
@@ -84,6 +90,8 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
             NamespaceName.SYSTEM_NAMESPACE,
             "loadbalancer-top-bundles-load-data").toString();
 
+    private static final long MAX_ROLE_CHANGE_RETRY_DELAY_IN_MILLIS = 200;
+
     private PulsarService pulsar;
 
     private ServiceConfiguration conf;
@@ -102,6 +110,9 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
 
     private LoadManagerScheduler unloadScheduler;
 
+    @Getter
+    private LeaderElectionService leaderElectionService;
+
     @Getter
     private LoadManagerContext context;
 
@@ -142,7 +153,14 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
             lookupRequests = ConcurrentOpenHashMap.>>newBuilder()
             .build();
+    private final CountDownLatch loadStoreInitWaiter = new CountDownLatch(1);
+
+    public enum Role {
+        Leader,
+        Follower
+    }
 
+    private Role role;
 
     /**
      * Life cycle: Constructor -> initialize -> start -> close.
@@ -173,12 +191,24 @@ public void start() throws PulsarServerException {
             return;
         }
         this.brokerRegistry = new BrokerRegistryImpl(pulsar);
+        this.leaderElectionService = new LeaderElectionService(
+                pulsar.getCoordinationService(), pulsar.getSafeWebServiceAddress(),
+                state -> {
+                    pulsar.getLoadManagerExecutor().execute(() -> {
+                        if (state == LeaderElectionState.Leading) {
+                            playLeader();
+                        } else {
+                            playFollower();
+                        }
+                    });
+                });
         this.serviceUnitStateChannel = new ServiceUnitStateChannelImpl(pulsar);
         this.brokerRegistry.start();
         this.unloadManager = new UnloadManager();
         this.splitManager = new SplitManager(splitCounter);
         this.serviceUnitStateChannel.listen(unloadManager);
         this.serviceUnitStateChannel.listen(splitManager);
+        this.leaderElectionService.start();
         this.serviceUnitStateChannel.start();
         this.antiAffinityGroupPolicyHelper =
                 new AntiAffinityGroupPolicyHelper(pulsar, serviceUnitStateChannel);
@@ -189,8 +219,10 @@ public void start() throws PulsarServerException {
         try {
             this.brokerLoadDataStore = LoadDataStoreFactory
                     .create(pulsar.getClient(), BROKER_LOAD_DATA_STORE_TOPIC, BrokerLoadData.class);
+            this.brokerLoadDataStore.startTableView();
             this.topBundlesLoadDataStore = LoadDataStoreFactory
                     .create(pulsar.getClient(), TOP_BUNDLES_LOAD_DATA_STORE_TOPIC, TopBundlesLoadData.class);
+            this.loadStoreInitWaiter.countDown();
         } catch (LoadDataStoreException e) {
             throw new PulsarServerException(e);
         }
@@ -409,8 +441,15 @@ public void close() throws PulsarServerException {
                     this.serviceUnitStateChannel.close();
                 } finally {
                     this.unloadManager.close();
-                    this.started = false;
+                    try {
+                        this.leaderElectionService.close();
+                    } catch (Exception e) {
+                        throw new PulsarServerException(e);
+                    } finally {
+                        this.started = false;
+                    }
                 }
+
             }
         }
     }
@@ -421,6 +460,78 @@ private boolean isInternalTopic(String topic) {
                 || topic.startsWith(TOP_BUNDLES_LOAD_DATA_STORE_TOPIC);
     }
 
+    @VisibleForTesting
+    void playLeader() {
+        if (role != Leader) {
+            log.info("This broker:{} is changing the role from {} to {}",
+                    pulsar.getLookupServiceAddress(), role, Leader);
+            int retry = 0;
+            while (true) {
+                try {
+                    serviceUnitStateChannel.scheduleOwnershipMonitor();
+                    loadStoreInitWaiter.await();
+                    topBundlesLoadDataStore.startTableView();
+                    unloadScheduler.start();
+                    break;
+                } catch (Throwable e) {
+                    log.error("The broker:{} failed to change the role. Retrying {} th ...",
+                            pulsar.getLookupServiceAddress(), ++retry, e);
+                    try {
+                        Thread.sleep(Math.min(retry * 10, MAX_ROLE_CHANGE_RETRY_DELAY_IN_MILLIS));
+                    } catch (InterruptedException ex) {
+                        log.warn("Interrupted while sleeping.");
+                    }
+                }
+            }
+            role = Leader;
+            log.info("This broker:{} plays the leader now.", pulsar.getLookupServiceAddress());
+        }
+
+        // flush the load data when the leader is elected.
+        if (brokerLoadDataReporter != null) {
+            brokerLoadDataReporter.reportAsync(true);
+        }
+        if (topBundleLoadDataReporter != null) {
+            topBundleLoadDataReporter.reportAsync(true);
+        }
+    }
+
+    @VisibleForTesting
+    void playFollower() {
+        if (role != Follower) {
+            log.info("This broker:{} is changing the role from {} to {}",
+                    pulsar.getLookupServiceAddress(), role, Follower);
+            int retry = 0;
+            while (true) {
+                try {
+                    serviceUnitStateChannel.cancelOwnershipMonitor();
+                    loadStoreInitWaiter.await();
+                    topBundlesLoadDataStore.closeTableView();
+                    unloadScheduler.close();
+                    break;
+                } catch (Throwable e) {
+                    log.error("The broker:{} failed to change the role. Retrying {} th ...",
+                            pulsar.getLookupServiceAddress(), ++retry, e);
+                    try {
+                        Thread.sleep(Math.min(retry * 10, MAX_ROLE_CHANGE_RETRY_DELAY_IN_MILLIS));
+                    } catch (InterruptedException ex) {
+                        log.warn("Interrupted while sleeping.");
+                    }
+                }
+            }
+            role = Follower;
+            log.info("This broker:{} plays a follower now.", pulsar.getLookupServiceAddress());
+        }
+
+        // flush the load data when the leader is elected.
+        if (brokerLoadDataReporter != null) {
+            brokerLoadDataReporter.reportAsync(true);
+        }
+        if (topBundleLoadDataReporter != null) {
+            topBundleLoadDataReporter.reportAsync(true);
+        }
+    }
+
     void updateBrokerLoadMetrics(BrokerLoadData loadData) {
         this.brokerLoadMetrics.set(loadData.toMetrics(pulsar.getAdvertisedAddress()));
     }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java
index 4e92ad791ab63..719c72a67b4ea 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java
@@ -172,4 +172,13 @@ public interface ServiceUnitStateChannel extends Closeable {
      */
     Set> getOwnershipEntrySet();
 
+    /**
+     * Schedules ownership monitor to periodically check and correct invalid ownership states.
+     */
+    void scheduleOwnershipMonitor();
+
+    /**
+     * Cancels the ownership monitor.
+     */
+    void cancelOwnershipMonitor();
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
index f8686c07f05fa..791d02649baf1 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
@@ -96,7 +96,6 @@
 import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap;
 import org.apache.pulsar.metadata.api.MetadataStoreException;
 import org.apache.pulsar.metadata.api.NotificationType;
-import org.apache.pulsar.metadata.api.coordination.LeaderElectionState;
 import org.apache.pulsar.metadata.api.extended.SessionEvent;
 
 @Slf4j
@@ -119,9 +118,9 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel {
     private final String lookupServiceAddress;
     private final ConcurrentOpenHashMap> cleanupJobs;
     private final StateChangeListeners stateChangeListeners;
-    private final LeaderElectionService leaderElectionService;
     private BrokerSelectionStrategy brokerSelector;
     private BrokerRegistry brokerRegistry;
+    private LeaderElectionService leaderElectionService;
     private TableView tableview;
     private Producer producer;
     private ScheduledFuture monitorTask;
@@ -205,31 +204,7 @@ public ServiceUnitStateChannelImpl(PulsarService pulsar) {
         }
         this.maxCleanupDelayTimeInSecs = MAX_CLEAN_UP_DELAY_TIME_IN_SECS;
         this.minCleanupDelayTimeInSecs = MIN_CLEAN_UP_DELAY_TIME_IN_SECS;
-        this.leaderElectionService = new LeaderElectionService(
-                pulsar.getCoordinationService(), pulsar.getSafeWebServiceAddress(),
-                state -> {
-                    if (state == LeaderElectionState.Leading) {
-                        log.info("This broker:{} is the leader now.", lookupServiceAddress);
-                        this.monitorTask = this.pulsar.getLoadManagerExecutor()
-                                .scheduleWithFixedDelay(() -> {
-                                            try {
-                                                monitorOwnerships(brokerRegistry.getAvailableBrokersAsync()
-                                                        .get(inFlightStateWaitingTimeInMillis, MILLISECONDS));
-                                            } catch (Exception e) {
-                                                log.info("Failed to monitor the ownerships. will retry..", e);
-                                            }
-                                        },
-                                        ownershipMonitorDelayTimeInSecs, ownershipMonitorDelayTimeInSecs, SECONDS);
-                    } else {
-                        log.info("This broker:{} is a follower now.", lookupServiceAddress);
-                        if (monitorTask != null) {
-                            monitorTask.cancel(false);
-                            monitorTask = null;
-                            log.info("This previous leader broker:{} stopped the channel clean-up monitor",
-                                    lookupServiceAddress);
-                        }
-                    }
-                });
+
         Map tmpOwnerLookUpCounters = new HashMap<>();
         Map tmpHandlerCounters = new HashMap<>();
         Map tmpEventCounters = new HashMap<>();
@@ -246,6 +221,32 @@ public ServiceUnitStateChannelImpl(PulsarService pulsar) {
         this.channelState = Constructed;
     }
 
+    public void scheduleOwnershipMonitor() {
+        if (monitorTask == null) {
+            this.monitorTask = this.pulsar.getLoadManagerExecutor()
+                    .scheduleWithFixedDelay(() -> {
+                                try {
+                                    monitorOwnerships(brokerRegistry.getAvailableBrokersAsync()
+                                            .get(inFlightStateWaitingTimeInMillis, MILLISECONDS));
+                                } catch (Exception e) {
+                                    log.info("Failed to monitor the ownerships. will retry..", e);
+                                }
+                            },
+                            ownershipMonitorDelayTimeInSecs, ownershipMonitorDelayTimeInSecs, SECONDS);
+            log.info("This leader broker:{} started the ownership monitor.",
+                    lookupServiceAddress);
+        }
+    }
+
+    public void cancelOwnershipMonitor() {
+        if (monitorTask != null) {
+            monitorTask.cancel(false);
+            monitorTask = null;
+            log.info("This previous leader broker:{} stopped the ownership monitor.",
+                    lookupServiceAddress);
+        }
+    }
+
     public synchronized void start() throws PulsarServerException {
         if (!validateChannelState(LeaderElectionServiceStarted, false)) {
             throw new IllegalStateException("Invalid channel state:" + channelState.name());
@@ -255,11 +256,15 @@ public synchronized void start() throws PulsarServerException {
         try {
             this.brokerRegistry = getBrokerRegistry();
             this.brokerRegistry.addListener(this::handleBrokerRegistrationEvent);
-            leaderElectionService.start();
-            this.channelState = LeaderElectionServiceStarted;
-            if (debug) {
-                log.info("Successfully started the channel leader election service.");
+            this.leaderElectionService = getLeaderElectionService();
+            var leader = leaderElectionService.readCurrentLeader().get(
+                    MAX_CHANNEL_OWNER_ELECTION_WAITING_TIME_IN_SECS, TimeUnit.SECONDS);
+            if (leader.isPresent()) {
+                log.info("Successfully found the channel leader:{}.", leader.get());
+            } else {
+                log.warn("Failed to find the channel leader.");
             }
+            this.channelState = LeaderElectionServiceStarted;
             brokerSelector = getBrokerSelector();
 
             if (producer != null) {
@@ -327,15 +332,17 @@ protected BrokerSelectionStrategy getBrokerSelector() {
         return new LeastResourceUsageWithWeight();
     }
 
+    @VisibleForTesting
+    protected LeaderElectionService getLeaderElectionService() {
+        return ((ExtensibleLoadManagerWrapper) pulsar.getLoadManager().get())
+                .get().getLeaderElectionService();
+    }
+
     public synchronized void close() throws PulsarServerException {
         channelState = Closed;
         boolean debug = debug();
         try {
-            leaderElectionService.close();
-            if (debug) {
-                log.info("Successfully closed the channel leader election service.");
-            }
-
+            leaderElectionService = null;
             if (tableview != null) {
                 tableview.close();
                 tableview = null;
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java
index e646026978754..bc3c8eb6a94fd 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java
@@ -156,16 +156,19 @@ public synchronized void execute() {
 
     @Override
     public void start() {
-        long loadSheddingInterval = TimeUnit.MINUTES
-                .toMillis(conf.getLoadBalancerSheddingIntervalMinutes());
-        this.task = loadManagerExecutor.scheduleAtFixedRate(
-                this::execute, loadSheddingInterval, loadSheddingInterval, TimeUnit.MILLISECONDS);
+        if (this.task == null) {
+            long loadSheddingInterval = TimeUnit.MINUTES
+                    .toMillis(conf.getLoadBalancerSheddingIntervalMinutes());
+            this.task = loadManagerExecutor.scheduleAtFixedRate(
+                    this::execute, loadSheddingInterval, loadSheddingInterval, TimeUnit.MILLISECONDS);
+        }
     }
 
     @Override
     public void close() {
         if (this.task != null) {
             this.task.cancel(false);
+            this.task = null;
         }
         this.recentlyUnloadedBundles.clear();
         this.recentlyUnloadedBrokers.clear();
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStore.java
index 512811e10198c..680a36523a214 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStore.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStore.java
@@ -19,6 +19,7 @@
 package org.apache.pulsar.broker.loadbalance.extensions.store;
 
 import java.io.Closeable;
+import java.io.IOException;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
@@ -74,4 +75,15 @@ public interface LoadDataStore extends Closeable {
      */
     int size();
 
+
+    /**
+     * Closes the table view.
+     */
+    void closeTableView() throws IOException;
+
+    /**
+     * Starts the table view.
+     */
+    void startTableView() throws LoadDataStoreException;
+
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/TableViewLoadDataStoreImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/TableViewLoadDataStoreImpl.java
index 3909de2afa241..a400163ebf122 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/TableViewLoadDataStoreImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/TableViewLoadDataStoreImpl.java
@@ -26,6 +26,7 @@
 import java.util.function.BiConsumer;
 import org.apache.pulsar.client.api.Producer;
 import org.apache.pulsar.client.api.PulsarClient;
+import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.client.api.Schema;
 import org.apache.pulsar.client.api.TableView;
 
@@ -36,14 +37,22 @@
  */
 public class TableViewLoadDataStoreImpl implements LoadDataStore {
 
-    private final TableView tableView;
+    private TableView tableView;
 
     private final Producer producer;
 
+    private final PulsarClient client;
+
+    private final String topic;
+
+    private final Class clazz;
+
     public TableViewLoadDataStoreImpl(PulsarClient client, String topic, Class clazz) throws LoadDataStoreException {
         try {
-            this.tableView = client.newTableViewBuilder(Schema.JSON(clazz)).topic(topic).create();
+            this.client = client;
             this.producer = client.newProducer(Schema.JSON(clazz)).topic(topic).create();
+            this.topic = topic;
+            this.clazz = clazz;
         } catch (Exception e) {
             throw new LoadDataStoreException(e);
         }
@@ -61,30 +70,59 @@ public CompletableFuture removeAsync(String key) {
 
     @Override
     public Optional get(String key) {
+        validateTableViewStart();
         return Optional.ofNullable(tableView.get(key));
     }
 
     @Override
     public void forEach(BiConsumer action) {
+        validateTableViewStart();
         tableView.forEach(action);
     }
 
     public Set> entrySet() {
+        validateTableViewStart();
         return tableView.entrySet();
     }
 
     @Override
     public int size() {
+        validateTableViewStart();
         return tableView.size();
     }
 
+    @Override
+    public void closeTableView() throws IOException {
+        if (tableView != null) {
+            tableView.close();
+            tableView = null;
+        }
+    }
+
+    @Override
+    public void startTableView() throws LoadDataStoreException {
+        if (tableView == null) {
+            try {
+                tableView = client.newTableViewBuilder(Schema.JSON(clazz)).topic(topic).create();
+            } catch (PulsarClientException e) {
+                tableView = null;
+                throw new LoadDataStoreException(e);
+            }
+        }
+    }
+
     @Override
     public void close() throws IOException {
         if (producer != null) {
             producer.close();
         }
-        if (tableView != null) {
-            tableView.close();
+        closeTableView();
+    }
+
+    private void validateTableViewStart() {
+        if (tableView == null) {
+            throw new IllegalStateException("table view has not been started");
         }
     }
+
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
index 17fcb9952618b..aa8583c6b57b6 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
@@ -35,6 +35,7 @@
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Overloaded;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Underloaded;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Unknown;
+import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.reset;
 import static org.mockito.Mockito.spy;
@@ -42,6 +43,8 @@
 import static org.mockito.Mockito.verify;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
 import static org.testng.Assert.assertTrue;
 import static org.testng.Assert.fail;
 
@@ -49,6 +52,8 @@
 import java.util.LinkedHashMap;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
@@ -74,12 +79,14 @@
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
+import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData;
 import org.apache.pulsar.broker.loadbalance.extensions.filter.BrokerFilter;
 import org.apache.pulsar.broker.loadbalance.extensions.models.AssignCounter;
 import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter;
 import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter;
 import org.apache.pulsar.broker.loadbalance.extensions.scheduler.TransferShedder;
+import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore;
 import org.apache.pulsar.broker.lookup.LookupResult;
 import org.apache.pulsar.broker.namespace.LookupOptions;
 import org.apache.pulsar.broker.resources.NamespaceResources;
@@ -100,6 +107,7 @@
 import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
 import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage;
 import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage;
+import org.testcontainers.shaded.org.awaitility.Awaitility;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.BeforeMethod;
@@ -143,22 +151,9 @@ public void setup() throws Exception {
         additionalPulsarTestContext = createAdditionalPulsarTestContext(defaultConf);
         pulsar2 = additionalPulsarTestContext.getPulsarService();
 
-        ExtensibleLoadManagerWrapper primaryLoadManagerWrapper =
-                (ExtensibleLoadManagerWrapper) pulsar1.getLoadManager().get();
-        primaryLoadManager = spy((ExtensibleLoadManagerImpl)
-                FieldUtils.readField(primaryLoadManagerWrapper, "loadManager", true));
-        FieldUtils.writeField(primaryLoadManagerWrapper, "loadManager", primaryLoadManager, true);
-
-        ExtensibleLoadManagerWrapper secondaryLoadManagerWrapper =
-                (ExtensibleLoadManagerWrapper) pulsar2.getLoadManager().get();
-        secondaryLoadManager = spy((ExtensibleLoadManagerImpl)
-                FieldUtils.readField(secondaryLoadManagerWrapper, "loadManager", true));
-        FieldUtils.writeField(secondaryLoadManagerWrapper, "loadManager", secondaryLoadManager, true);
+        setPrimaryLoadManager();
 
-        channel1 = (ServiceUnitStateChannelImpl)
-                FieldUtils.readField(primaryLoadManager, "serviceUnitStateChannel", true);
-        channel2 = (ServiceUnitStateChannelImpl)
-                FieldUtils.readField(secondaryLoadManager, "serviceUnitStateChannel", true);
+        setSecondaryLoadManager();
 
         admin.clusters().createCluster(this.conf.getClusterName(),
                 ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build());
@@ -412,6 +407,130 @@ public Map filter(Map broker
         assertEquals(brokerLookupData.get().getWebServiceUrl(), pulsar2.getWebServiceAddress());
     }
 
+    @Test
+    public void testTopBundlesLoadDataStoreTableViewFromChannelOwner() throws Exception {
+        var topBundlesLoadDataStorePrimary =
+                (LoadDataStore) FieldUtils.readDeclaredField(primaryLoadManager, "topBundlesLoadDataStore", true);
+        var serviceUnitStateChannelPrimary =
+                (ServiceUnitStateChannelImpl) FieldUtils.readDeclaredField(primaryLoadManager,
+                        "serviceUnitStateChannel", true);
+        var tvPrimary =
+                (TableViewImpl) FieldUtils.readDeclaredField(topBundlesLoadDataStorePrimary, "tableView", true);
+
+        var topBundlesLoadDataStoreSecondary =
+                (LoadDataStore) FieldUtils.readDeclaredField(secondaryLoadManager, "topBundlesLoadDataStore", true);
+        var tvSecondary =
+                (TableViewImpl) FieldUtils.readDeclaredField(topBundlesLoadDataStoreSecondary, "tableView", true);
+
+        if (serviceUnitStateChannelPrimary.isChannelOwnerAsync().get(5, TimeUnit.SECONDS)) {
+            assertNotNull(tvPrimary);
+            assertNull(tvSecondary);
+        } else {
+            assertNull(tvPrimary);
+            assertNotNull(tvSecondary);
+        }
+
+        restartBroker();
+        pulsar1 = pulsar;
+        setPrimaryLoadManager();
+
+        var serviceUnitStateChannelPrimaryNew =
+                (ServiceUnitStateChannelImpl) FieldUtils.readDeclaredField(primaryLoadManager,
+                        "serviceUnitStateChannel", true);
+        var topBundlesLoadDataStorePrimaryNew =
+                (LoadDataStore) FieldUtils.readDeclaredField(primaryLoadManager, "topBundlesLoadDataStore"
+                        , true);
+        Awaitility.await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> {
+                    assertFalse(serviceUnitStateChannelPrimaryNew.isChannelOwnerAsync().get(5, TimeUnit.SECONDS));
+                    assertNotNull(FieldUtils.readDeclaredField(topBundlesLoadDataStoreSecondary, "tableView"
+                            , true));
+                    assertNull(FieldUtils.readDeclaredField(topBundlesLoadDataStorePrimaryNew, "tableView"
+                            , true));
+                }
+        );
+    }
+
+    @Test
+    public void testRoleChange()
+            throws Exception {
+
+
+        var topBundlesLoadDataStorePrimary = (LoadDataStore)
+                FieldUtils.readDeclaredField(primaryLoadManager, "topBundlesLoadDataStore", true);
+        var topBundlesLoadDataStorePrimarySpy = spy(topBundlesLoadDataStorePrimary);
+        AtomicInteger countPri = new AtomicInteger(3);
+        AtomicInteger countPri2 = new AtomicInteger(3);
+        doAnswer(invocationOnMock -> {
+            if (countPri.decrementAndGet() > 0) {
+                throw new RuntimeException();
+            }
+            // Call the real method
+            reset();
+            return null;
+        }).when(topBundlesLoadDataStorePrimarySpy).startTableView();
+        doAnswer(invocationOnMock -> {
+            if (countPri2.decrementAndGet() > 0) {
+                throw new RuntimeException();
+            }
+            // Call the real method
+            reset();
+            return null;
+        }).when(topBundlesLoadDataStorePrimarySpy).closeTableView();
+        FieldUtils.writeDeclaredField(primaryLoadManager, "topBundlesLoadDataStore", topBundlesLoadDataStorePrimarySpy, true);
+
+        var topBundlesLoadDataStoreSecondary = (LoadDataStore)
+                FieldUtils.readDeclaredField(secondaryLoadManager, "topBundlesLoadDataStore", true);
+        var topBundlesLoadDataStoreSecondarySpy = spy(topBundlesLoadDataStoreSecondary);
+        AtomicInteger countSec = new AtomicInteger(3);
+        AtomicInteger countSec2 = new AtomicInteger(3);
+        doAnswer(invocationOnMock -> {
+            if (countSec.decrementAndGet() > 0) {
+                throw new RuntimeException();
+            }
+            // Call the real method
+            reset();
+            return null;
+        }).when(topBundlesLoadDataStoreSecondarySpy).startTableView();
+        doAnswer(invocationOnMock -> {
+            if (countSec2.decrementAndGet() > 0) {
+                throw new RuntimeException();
+            }
+            // Call the real method
+            reset();
+            return null;
+        }).when(topBundlesLoadDataStoreSecondarySpy).closeTableView();
+        FieldUtils.writeDeclaredField(secondaryLoadManager, "topBundlesLoadDataStore", topBundlesLoadDataStoreSecondarySpy, true);
+
+        if (channel1.isChannelOwnerAsync().get(5, TimeUnit.SECONDS)) {
+            primaryLoadManager.playFollower();
+            primaryLoadManager.playFollower();
+            secondaryLoadManager.playLeader();
+            secondaryLoadManager.playLeader();
+            primaryLoadManager.playLeader();
+            primaryLoadManager.playLeader();
+            secondaryLoadManager.playFollower();
+            secondaryLoadManager.playFollower();
+        } else {
+            primaryLoadManager.playLeader();
+            primaryLoadManager.playLeader();
+            secondaryLoadManager.playFollower();
+            secondaryLoadManager.playFollower();
+            primaryLoadManager.playFollower();
+            primaryLoadManager.playFollower();
+            secondaryLoadManager.playLeader();
+            secondaryLoadManager.playLeader();
+        }
+
+
+        verify(topBundlesLoadDataStorePrimarySpy, times(3)).startTableView();
+        verify(topBundlesLoadDataStorePrimarySpy, times(3)).closeTableView();
+        verify(topBundlesLoadDataStoreSecondarySpy, times(3)).startTableView();
+        verify(topBundlesLoadDataStoreSecondarySpy, times(3)).closeTableView();
+
+        FieldUtils.writeDeclaredField(primaryLoadManager, "topBundlesLoadDataStore", topBundlesLoadDataStorePrimary, true);
+        FieldUtils.writeDeclaredField(secondaryLoadManager, "topBundlesLoadDataStore", topBundlesLoadDataStoreSecondary, true);
+    }
+
     @Test
     public void testGetMetrics() throws Exception {
         {
@@ -612,6 +731,26 @@ private static void cleanTableView(ServiceUnitStateChannel channel)
         cache.clear();
     }
 
+    private void setPrimaryLoadManager() throws IllegalAccessException {
+        ExtensibleLoadManagerWrapper wrapper =
+                (ExtensibleLoadManagerWrapper) pulsar1.getLoadManager().get();
+        primaryLoadManager = spy((ExtensibleLoadManagerImpl)
+                FieldUtils.readField(wrapper, "loadManager", true));
+        FieldUtils.writeField(wrapper, "loadManager", primaryLoadManager, true);
+        channel1 = (ServiceUnitStateChannelImpl)
+                FieldUtils.readField(primaryLoadManager, "serviceUnitStateChannel", true);
+    }
+
+    private void setSecondaryLoadManager() throws IllegalAccessException {
+        ExtensibleLoadManagerWrapper wrapper =
+                (ExtensibleLoadManagerWrapper) pulsar2.getLoadManager().get();
+        secondaryLoadManager = spy((ExtensibleLoadManagerImpl)
+                FieldUtils.readField(wrapper, "loadManager", true));
+        FieldUtils.writeField(wrapper, "loadManager", secondaryLoadManager, true);
+        channel2 = (ServiceUnitStateChannelImpl)
+                FieldUtils.readField(secondaryLoadManager, "serviceUnitStateChannel", true);
+    }
+
     private CompletableFuture getBundleAsync(PulsarService pulsar, TopicName topic) {
         return pulsar.getNamespaceService().getBundleAsync(topic);
     }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
index 64a5f63196bea..e0bd69fce64c7 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
@@ -89,6 +89,7 @@
 import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap;
 import org.apache.pulsar.metadata.api.MetadataStoreException;
 import org.apache.pulsar.metadata.api.NotificationType;
+import org.apache.pulsar.metadata.api.coordination.LeaderElectionState;
 import org.apache.pulsar.metadata.api.extended.SessionEvent;
 import org.awaitility.Awaitility;
 import org.testng.annotations.AfterClass;
@@ -1497,6 +1498,20 @@ ServiceUnitStateChannelImpl createChannel(PulsarService pulsar)
         doReturn(registry).when(channel).getBrokerRegistry();
         doReturn(brokerSelector).when(channel).getBrokerSelector();
 
+
+        var leaderElectionService = new LeaderElectionService(
+                pulsar.getCoordinationService(), pulsar.getSafeWebServiceAddress(),
+                state -> {
+                    if (state == LeaderElectionState.Leading) {
+                        channel.scheduleOwnershipMonitor();
+                    } else {
+                        channel.cancelOwnershipMonitor();
+                    }
+                });
+        leaderElectionService.start();
+
+        doReturn(leaderElectionService).when(channel).getLeaderElectionService();
+
         return channel;
     }
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilterTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilterTestBase.java
index 3de957c3b1a3a..c521861471c9a 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilterTestBase.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilterTestBase.java
@@ -34,6 +34,7 @@
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
 import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore;
+import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStoreException;
 import org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener;
 
 public class BrokerFilterTestBase {
@@ -82,6 +83,16 @@ public Set> entrySet() {
             public int size() {
                 return map.size();
             }
+
+            @Override
+            public void closeTableView() throws IOException {
+
+            }
+
+            @Override
+            public void startTableView() throws LoadDataStoreException {
+
+            }
         };
         configuration.setPreferLaterVersions(true);
         doReturn(configuration).when(mockContext).brokerConfiguration();
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
index 47bf1ad2e7ed1..3955f1ed9af2e 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
@@ -68,6 +68,7 @@
 import org.apache.pulsar.broker.loadbalance.extensions.policies.AntiAffinityGroupPolicyHelper;
 import org.apache.pulsar.broker.loadbalance.extensions.policies.IsolationPoliciesHelper;
 import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore;
+import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStoreException;
 import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies;
 import org.apache.pulsar.broker.namespace.NamespaceService;
 import org.apache.pulsar.broker.resources.LocalPoliciesResources;
@@ -267,6 +268,16 @@ public Set> entrySet() {
             public int size() {
                 return map.size();
             }
+
+            @Override
+            public void closeTableView() throws IOException {
+
+            }
+
+            @Override
+            public void startTableView() throws LoadDataStoreException {
+
+            }
         };
 
         var topBundleLoadDataStore = new LoadDataStore() {
@@ -310,6 +321,16 @@ public Set> entrySet() {
             public int size() {
                 return map.size();
             }
+
+            @Override
+            public void closeTableView() throws IOException {
+
+            }
+
+            @Override
+            public void startTableView() throws LoadDataStoreException {
+
+            }
         };
 
         BrokerRegistry brokerRegistry = mock(BrokerRegistry.class);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreTest.java
index 5e2924cd84254..184c337a47c80 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreTest.java
@@ -20,6 +20,7 @@
 
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
 import static org.testng.AssertJUnit.assertTrue;
 
 import com.google.common.collect.Sets;
@@ -74,6 +75,7 @@ public void testPushGetAndRemove() throws Exception {
         @Cleanup
         LoadDataStore loadDataStore =
                 LoadDataStoreFactory.create(pulsar.getClient(), topic, MyClass.class);
+        loadDataStore.startTableView();
         MyClass myClass1 = new MyClass("1", 1);
         loadDataStore.pushAsync("key1", myClass1).get();
 
@@ -106,6 +108,7 @@ public void testForEach() throws Exception {
         @Cleanup
         LoadDataStore loadDataStore =
                 LoadDataStoreFactory.create(pulsar.getClient(), topic, Integer.class);
+        loadDataStore.startTableView();
 
         Map map = new HashMap<>();
         for (int i = 0; i < 10; i++) {
@@ -124,4 +127,28 @@ public void testForEach() throws Exception {
         assertEquals(loadDataStore.entrySet(), map.entrySet());
     }
 
+    @Test
+    public void testTableViewRestart() throws Exception {
+        String topic = TopicDomain.persistent + "://" + NamespaceName.SYSTEM_NAMESPACE + "/" + UUID.randomUUID();
+        LoadDataStore loadDataStore =
+                LoadDataStoreFactory.create(pulsar.getClient(), topic, Integer.class);
+
+        loadDataStore.startTableView();
+        loadDataStore.pushAsync("1", 1).get();
+        Awaitility.await().untilAsserted(() -> assertEquals(loadDataStore.size(), 1));
+        assertEquals(loadDataStore.get("1").get(), 1);
+        loadDataStore.closeTableView();
+
+        loadDataStore.pushAsync("1", 2).get();
+        Exception ex = null;
+        try {
+            loadDataStore.get("1");
+        } catch (IllegalStateException e) {
+            ex = e;
+        }
+        assertNotNull(ex);
+        loadDataStore.startTableView();
+        Awaitility.await().untilAsserted(() -> assertEquals(loadDataStore.get("1").get(), 2));
+    }
+
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeightTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeightTest.java
index ebc2424cada19..0eea1d87513bf 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeightTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeightTest.java
@@ -24,6 +24,7 @@
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
 import static org.testng.Assert.assertEquals;
+import java.io.IOException;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
@@ -36,6 +37,7 @@
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData;
 import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore;
+import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStoreException;
 import org.apache.pulsar.common.naming.NamespaceName;
 import org.apache.pulsar.common.naming.ServiceUnitId;
 import org.apache.pulsar.common.naming.TopicName;
@@ -244,6 +246,16 @@ public Set> entrySet() {
             public int size() {
                 return map.size();
             }
+
+            @Override
+            public void closeTableView() throws IOException {
+
+            }
+
+            @Override
+            public void startTableView() throws LoadDataStoreException {
+
+            }
         };
 
         doReturn(conf).when(ctx).brokerConfiguration();

From 9adec1bef99fc7aedff6238f11e3d891f6e47e36 Mon Sep 17 00:00:00 2001
From: Matteo Merli 
Date: Wed, 15 Mar 2023 18:35:31 -0700
Subject: [PATCH 117/174] [fix][meta] Fixed race condition between ResourceLock
 update and invalidation (#19817)

---
 .../coordination/impl/LockManagerImpl.java    |  2 +-
 .../coordination/impl/ResourceLockImpl.java   | 56 ++++++++++++++-----
 2 files changed, 42 insertions(+), 16 deletions(-)

diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java
index 097f15af27677..ca768d38490cc 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java
@@ -124,7 +124,7 @@ private void handleSessionEvent(SessionEvent se) {
             if (se == SessionEvent.SessionReestablished) {
                 log.info("Metadata store session has been re-established. Revalidating all the existing locks.");
                 for (ResourceLockImpl lock : locks.values()) {
-                    futures.add(lock.revalidate(lock.getValue(), true));
+                    futures.add(lock.revalidate(lock.getValue(), true, true));
                 }
 
             } else if (se == SessionEvent.Reconnected) {
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java
index dea9aa1acb90f..5271a73249d80 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java
@@ -44,7 +44,7 @@ public class ResourceLockImpl implements ResourceLock {
     private long version;
     private final CompletableFuture expiredFuture;
     private boolean revalidateAfterReconnection = false;
-    private CompletableFuture revalidateFuture;
+    private CompletableFuture pendingOperationFuture;
 
     private enum State {
         Init,
@@ -61,6 +61,7 @@ public ResourceLockImpl(MetadataStoreExtended store, MetadataSerde serde, Str
         this.path = path;
         this.version = -1;
         this.expiredFuture = new CompletableFuture<>();
+        this.pendingOperationFuture = CompletableFuture.completedFuture(null);
         this.state = State.Init;
     }
 
@@ -71,7 +72,24 @@ public synchronized T getValue() {
 
     @Override
     public synchronized CompletableFuture updateValue(T newValue) {
-       return acquire(newValue);
+        // If there is an operation in progress, we're going to let it complete before attempting to
+        // update the value
+        if (pendingOperationFuture.isDone()) {
+            pendingOperationFuture = CompletableFuture.completedFuture(null);
+        }
+
+        pendingOperationFuture = pendingOperationFuture.thenCompose(v -> {
+            synchronized (ResourceLockImpl.this) {
+                if (state != State.Valid) {
+                    return CompletableFuture.failedFuture(
+                            new IllegalStateException("Lock was not in valid state: " + state));
+                }
+
+                return acquire(newValue);
+            }
+        });
+
+        return pendingOperationFuture;
     }
 
     @Override
@@ -128,7 +146,7 @@ synchronized CompletableFuture acquire(T newValue) {
                 .thenRun(() -> result.complete(null))
                 .exceptionally(ex -> {
                     if (ex.getCause() instanceof LockBusyException) {
-                        revalidate(newValue, false)
+                        revalidate(newValue, false, false)
                                 .thenAccept(__ -> result.complete(null))
                                 .exceptionally(ex1 -> {
                                    result.completeExceptionally(ex1);
@@ -185,7 +203,7 @@ synchronized void lockWasInvalidated() {
         }
 
         log.info("Lock on resource {} was invalidated", path);
-        revalidate(value, true)
+        revalidate(value, true, true)
                 .thenRun(() -> log.info("Successfully revalidated the lock on {}", path));
     }
 
@@ -193,31 +211,39 @@ synchronized CompletableFuture revalidateIfNeededAfterReconnection() {
         if (revalidateAfterReconnection) {
             revalidateAfterReconnection = false;
             log.warn("Revalidate lock at {} after reconnection", path);
-            return revalidate(value, true);
+            return revalidate(value, true, true);
         } else {
             return CompletableFuture.completedFuture(null);
         }
     }
 
-    synchronized CompletableFuture revalidate(T newValue, boolean revalidateAfterReconnection) {
-        if (revalidateFuture == null || revalidateFuture.isDone()) {
-            revalidateFuture = doRevalidate(newValue);
+    synchronized CompletableFuture revalidate(T newValue, boolean trackPendingOperation,
+                                                    boolean revalidateAfterReconnection) {
+
+        final CompletableFuture trackFuture;
+
+        if (!trackPendingOperation) {
+            trackFuture = doRevalidate(newValue);
+        } else if (pendingOperationFuture.isDone()) {
+            pendingOperationFuture = doRevalidate(newValue);
+            trackFuture = pendingOperationFuture;
         } else {
             if (log.isDebugEnabled()) {
                 log.debug("Previous revalidating is not finished while revalidate newValue={}, value={}, version={}",
                         newValue, value, version);
             }
-            CompletableFuture newFuture = new CompletableFuture<>();
-            revalidateFuture.whenComplete((unused, throwable) -> {
-                doRevalidate(newValue).thenRun(() -> newFuture.complete(null))
+            trackFuture = new CompletableFuture<>();
+            trackFuture.whenComplete((unused, throwable) -> {
+                doRevalidate(newValue).thenRun(() -> trackFuture.complete(null))
                         .exceptionally(throwable1 -> {
-                            newFuture.completeExceptionally(throwable1);
+                            trackFuture.completeExceptionally(throwable1);
                             return null;
                         });
             });
-            revalidateFuture = newFuture;
+            pendingOperationFuture = trackFuture;
         }
-        revalidateFuture.exceptionally(ex -> {
+
+        trackFuture.exceptionally(ex -> {
             synchronized (ResourceLockImpl.this) {
                 Throwable realCause = FutureUtil.unwrapCompletionException(ex);
                 if (!revalidateAfterReconnection || realCause instanceof BadVersionException
@@ -237,7 +263,7 @@ synchronized CompletableFuture revalidate(T newValue, boolean revalidateAf
             }
             return null;
         });
-        return revalidateFuture;
+        return trackFuture;
     }
 
     private synchronized CompletableFuture doRevalidate(T newValue) {

From 0ffa97efaac32f0e2b6d951966972f22f59329b5 Mon Sep 17 00:00:00 2001
From: sinan liu 
Date: Thu, 16 Mar 2023 10:01:50 +0800
Subject: [PATCH 118/174] [cleanup][broker] Remove duplicate code in the
 SchemaRegistryServiceImpl that checks for existing schema and new schema
 types (#19753)

---
 .../schema/SchemaRegistryServiceImpl.java     | 20 +++++++------------
 1 file changed, 7 insertions(+), 13 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java
index 4eb87564d0f22..ae56df248d85d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java
@@ -348,14 +348,14 @@ private SchemaRegistryFormat.SchemaInfo deleted(String schemaId, String user) {
 
     private void checkCompatible(SchemaAndMetadata existingSchema, SchemaData newSchema,
                                  SchemaCompatibilityStrategy strategy) throws IncompatibleSchemaException {
-        SchemaHash existingHash = SchemaHash.of(existingSchema.schema);
-        SchemaHash newHash = SchemaHash.of(newSchema);
         SchemaData existingSchemaData = existingSchema.schema;
         if (newSchema.getType() != existingSchemaData.getType()) {
             throw new IncompatibleSchemaException(String.format("Incompatible schema: "
                             + "exists schema type %s, new schema type %s",
                     existingSchemaData.getType(), newSchema.getType()));
         }
+        SchemaHash existingHash = SchemaHash.of(existingSchemaData);
+        SchemaHash newHash = SchemaHash.of(newSchema);
         if (!newHash.equals(existingHash)) {
             compatibilityChecks.getOrDefault(newSchema.getType(), SchemaCompatibilityCheck.DEFAULT)
                     .checkCompatible(existingSchemaData, newSchema, strategy);
@@ -465,17 +465,11 @@ private CompletableFuture checkCompatibilityWithLatest(String schemaId, Sc
                     }
                 });
 
-                if (existingSchema.schema.getType() != schema.getType()) {
-                    result.completeExceptionally(new IncompatibleSchemaException(
-                            String.format("Incompatible schema: exists schema type %s, new schema type %s",
-                                    existingSchema.schema.getType(), schema.getType())));
-                } else {
-                    try {
-                        checkCompatible(existingSchema, schema, strategy);
-                        result.complete(null);
-                    } catch (IncompatibleSchemaException e) {
-                        result.completeExceptionally(e);
-                    }
+                try {
+                    checkCompatible(existingSchema, schema, strategy);
+                    result.complete(null);
+                } catch (IncompatibleSchemaException e) {
+                    result.completeExceptionally(e);
                 }
                 return result;
             } else {

From da788794ebe211bc96d4d961de49ffeeb473ffad Mon Sep 17 00:00:00 2001
From: Yong Zhang 
Date: Thu, 16 Mar 2023 17:10:22 +0800
Subject: [PATCH 119/174] [improve] Upgrade bookkeeper to 4.15.4 (#19812)

### Motivation

Upgrade bookkeeper to 4.15.4.
---
 .../server/src/assemble/LICENSE.bin.txt       | 50 +++++++++----------
 .../shell/src/assemble/LICENSE.bin.txt        |  6 +--
 pom.xml                                       |  2 +-
 pulsar-sql/presto-distribution/LICENSE        | 24 ++++-----
 4 files changed, 41 insertions(+), 41 deletions(-)

diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt
index 4226e58e8cb0d..6b3455127b423 100644
--- a/distribution/server/src/assemble/LICENSE.bin.txt
+++ b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -347,31 +347,31 @@ The Apache Software License, Version 2.0
     - net.java.dev.jna-jna-jpms-5.12.1.jar
     - net.java.dev.jna-jna-platform-jpms-5.12.1.jar
  * BookKeeper
-    - org.apache.bookkeeper-bookkeeper-common-4.15.3.jar
-    - org.apache.bookkeeper-bookkeeper-common-allocator-4.15.3.jar
-    - org.apache.bookkeeper-bookkeeper-proto-4.15.3.jar
-    - org.apache.bookkeeper-bookkeeper-server-4.15.3.jar
-    - org.apache.bookkeeper-bookkeeper-tools-framework-4.15.3.jar
-    - org.apache.bookkeeper-circe-checksum-4.15.3.jar
-    - org.apache.bookkeeper-cpu-affinity-4.15.3.jar
-    - org.apache.bookkeeper-statelib-4.15.3.jar
-    - org.apache.bookkeeper-stream-storage-api-4.15.3.jar
-    - org.apache.bookkeeper-stream-storage-common-4.15.3.jar
-    - org.apache.bookkeeper-stream-storage-java-client-4.15.3.jar
-    - org.apache.bookkeeper-stream-storage-java-client-base-4.15.3.jar
-    - org.apache.bookkeeper-stream-storage-proto-4.15.3.jar
-    - org.apache.bookkeeper-stream-storage-server-4.15.3.jar
-    - org.apache.bookkeeper-stream-storage-service-api-4.15.3.jar
-    - org.apache.bookkeeper-stream-storage-service-impl-4.15.3.jar
-    - org.apache.bookkeeper.http-http-server-4.15.3.jar
-    - org.apache.bookkeeper.http-vertx-http-server-4.15.3.jar
-    - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.15.3.jar
-    - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.15.3.jar
-    - org.apache.distributedlog-distributedlog-common-4.15.3.jar
-    - org.apache.distributedlog-distributedlog-core-4.15.3-tests.jar
-    - org.apache.distributedlog-distributedlog-core-4.15.3.jar
-    - org.apache.distributedlog-distributedlog-protocol-4.15.3.jar
-    - org.apache.bookkeeper.stats-codahale-metrics-provider-4.15.3.jar
+    - org.apache.bookkeeper-bookkeeper-common-4.15.4.jar
+    - org.apache.bookkeeper-bookkeeper-common-allocator-4.15.4.jar
+    - org.apache.bookkeeper-bookkeeper-proto-4.15.4.jar
+    - org.apache.bookkeeper-bookkeeper-server-4.15.4.jar
+    - org.apache.bookkeeper-bookkeeper-tools-framework-4.15.4.jar
+    - org.apache.bookkeeper-circe-checksum-4.15.4.jar
+    - org.apache.bookkeeper-cpu-affinity-4.15.4.jar
+    - org.apache.bookkeeper-statelib-4.15.4.jar
+    - org.apache.bookkeeper-stream-storage-api-4.15.4.jar
+    - org.apache.bookkeeper-stream-storage-common-4.15.4.jar
+    - org.apache.bookkeeper-stream-storage-java-client-4.15.4.jar
+    - org.apache.bookkeeper-stream-storage-java-client-base-4.15.4.jar
+    - org.apache.bookkeeper-stream-storage-proto-4.15.4.jar
+    - org.apache.bookkeeper-stream-storage-server-4.15.4.jar
+    - org.apache.bookkeeper-stream-storage-service-api-4.15.4.jar
+    - org.apache.bookkeeper-stream-storage-service-impl-4.15.4.jar
+    - org.apache.bookkeeper.http-http-server-4.15.4.jar
+    - org.apache.bookkeeper.http-vertx-http-server-4.15.4.jar
+    - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.15.4.jar
+    - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.15.4.jar
+    - org.apache.distributedlog-distributedlog-common-4.15.4.jar
+    - org.apache.distributedlog-distributedlog-core-4.15.4-tests.jar
+    - org.apache.distributedlog-distributedlog-core-4.15.4.jar
+    - org.apache.distributedlog-distributedlog-protocol-4.15.4.jar
+    - org.apache.bookkeeper.stats-codahale-metrics-provider-4.15.4.jar
   * Apache HTTP Client
     - org.apache.httpcomponents-httpclient-4.5.13.jar
     - org.apache.httpcomponents-httpcore-4.4.15.jar
diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt
index 50608fddc9e3b..90896790b1fba 100644
--- a/distribution/shell/src/assemble/LICENSE.bin.txt
+++ b/distribution/shell/src/assemble/LICENSE.bin.txt
@@ -390,9 +390,9 @@ The Apache Software License, Version 2.0
     - log4j-web-2.18.0.jar
 
  * BookKeeper
-    - bookkeeper-common-allocator-4.15.3.jar
-    - cpu-affinity-4.15.3.jar
-    - circe-checksum-4.15.3.jar
+    - bookkeeper-common-allocator-4.15.4.jar
+    - cpu-affinity-4.15.4.jar
+    - circe-checksum-4.15.4.jar
   * AirCompressor
      - aircompressor-0.20.jar
  * AsyncHttpClient
diff --git a/pom.xml b/pom.xml
index 5b001f2d3f610..dc27ed54274fb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -126,7 +126,7 @@ flexible messaging model and an intuitive client API.
     
     1.21
 
-    4.15.3
+    4.15.4
     3.8.1
     1.5.0
     1.10.0
diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE
index 64c488c38bbe9..c523fae7606e1 100644
--- a/pulsar-sql/presto-distribution/LICENSE
+++ b/pulsar-sql/presto-distribution/LICENSE
@@ -424,18 +424,18 @@ The Apache Software License, Version 2.0
     - async-http-client-2.12.1.jar
     - async-http-client-netty-utils-2.12.1.jar
   * Apache Bookkeeper
-    - bookkeeper-common-4.15.3.jar
-    - bookkeeper-common-allocator-4.15.3.jar
-    - bookkeeper-proto-4.15.3.jar
-    - bookkeeper-server-4.15.3.jar
-    - bookkeeper-stats-api-4.15.3.jar
-    - bookkeeper-tools-framework-4.15.3.jar
-    - circe-checksum-4.15.3.jar
-    - codahale-metrics-provider-4.15.3.jar
-    - cpu-affinity-4.15.3.jar
-    - http-server-4.15.3.jar
-    - prometheus-metrics-provider-4.15.3.jar
-    - codahale-metrics-provider-4.15.3.jar
+    - bookkeeper-common-4.15.4.jar
+    - bookkeeper-common-allocator-4.15.4.jar
+    - bookkeeper-proto-4.15.4.jar
+    - bookkeeper-server-4.15.4.jar
+    - bookkeeper-stats-api-4.15.4.jar
+    - bookkeeper-tools-framework-4.15.4.jar
+    - circe-checksum-4.15.4.jar
+    - codahale-metrics-provider-4.15.4.jar
+    - cpu-affinity-4.15.4.jar
+    - http-server-4.15.4.jar
+    - prometheus-metrics-provider-4.15.4.jar
+    - codahale-metrics-provider-4.15.4.jar
   * Apache Commons
     - commons-cli-1.5.0.jar
     - commons-codec-1.15.jar

From 80c5791b87482bee3392308ecef45f455f8de885 Mon Sep 17 00:00:00 2001
From: yangyijun <1012293987@qq.com>
Date: Fri, 17 Mar 2023 01:34:54 +0800
Subject: [PATCH 120/174] [improve][broker]PIP-214 Add broker level metrics
 statistics and expose to prometheus (#19047)

---
 .../prometheus/AggregatedBrokerStats.java     |  67 +++++++++++
 .../prometheus/NamespaceStatsAggregator.java  |  48 +++++---
 .../broker/stats/PrometheusMetricsTest.java   | 105 ++++++++++++++++--
 .../broker/stats/TransactionMetricsTest.java  |   4 +-
 4 files changed, 197 insertions(+), 27 deletions(-)
 create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedBrokerStats.java

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedBrokerStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedBrokerStats.java
new file mode 100644
index 0000000000000..00c6cecdbfca1
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedBrokerStats.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.stats.prometheus;
+
+public class AggregatedBrokerStats {
+    public int topicsCount;
+    public int subscriptionsCount;
+    public int producersCount;
+    public int consumersCount;
+    public double rateIn;
+    public double rateOut;
+    public double throughputIn;
+    public double throughputOut;
+    public long storageSize;
+    public long storageLogicalSize;
+    public double storageWriteRate;
+    public double storageReadRate;
+    public long msgBacklog;
+
+    void updateStats(TopicStats stats) {
+        topicsCount++;
+        subscriptionsCount += stats.subscriptionsCount;
+        producersCount += stats.producersCount;
+        consumersCount += stats.consumersCount;
+        rateIn += stats.rateIn;
+        rateOut += stats.rateOut;
+        throughputIn += stats.throughputIn;
+        throughputOut += stats.throughputOut;
+        storageSize += stats.managedLedgerStats.storageSize;
+        storageLogicalSize += stats.managedLedgerStats.storageLogicalSize;
+        storageWriteRate += stats.managedLedgerStats.storageWriteRate;
+        storageReadRate += stats.managedLedgerStats.storageReadRate;
+        msgBacklog += stats.msgBacklog;
+    }
+
+    public void reset() {
+        topicsCount = 0;
+        subscriptionsCount = 0;
+        producersCount = 0;
+        consumersCount = 0;
+        rateIn = 0;
+        rateOut = 0;
+        throughputIn = 0;
+        throughputOut = 0;
+        storageSize = 0;
+        storageLogicalSize = 0;
+        storageWriteRate = 0;
+        storageReadRate = 0;
+        msgBacklog = 0;
+    }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java
index d287bf89d6c7e..918aef539cff4 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java
@@ -45,6 +45,14 @@
 @Slf4j
 public class NamespaceStatsAggregator {
 
+    private static final FastThreadLocal localBrokerStats =
+            new FastThreadLocal<>() {
+                @Override
+                protected AggregatedBrokerStats initialValue() {
+                    return new AggregatedBrokerStats();
+                }
+            };
+
     private static final FastThreadLocal localNamespaceStats =
             new FastThreadLocal<>() {
                 @Override
@@ -64,14 +72,13 @@ public static void generate(PulsarService pulsar, boolean includeTopicMetrics, b
                                 boolean includeProducerMetrics, boolean splitTopicAndPartitionIndexLabel,
                                 PrometheusMetricStreams stream) {
         String cluster = pulsar.getConfiguration().getClusterName();
+        AggregatedBrokerStats brokerStats = localBrokerStats.get();
+        brokerStats.reset();
         AggregatedNamespaceStats namespaceStats = localNamespaceStats.get();
         TopicStats topicStats = localTopicStats.get();
         Optional compactorMXBean = getCompactorMXBean(pulsar);
         LongAdder topicsCount = new LongAdder();
         Map localNamespaceTopicCount = new HashMap<>();
-
-        printDefaultBrokerStats(stream, cluster);
-
         pulsar.getBrokerService().getMultiLayerTopicMap().forEach((namespace, bundlesMap) -> {
             namespaceStats.reset();
             topicsCount.reset();
@@ -83,6 +90,8 @@ public static void generate(PulsarService pulsar, boolean includeTopicMetrics, b
                         compactorMXBean
                 );
 
+                brokerStats.updateStats(topicStats);
+
                 if (includeTopicMetrics) {
                     topicsCount.add(1);
                     TopicStats.printTopicStats(stream, topicStats, compactorMXBean, cluster, namespace, name,
@@ -104,6 +113,8 @@ public static void generate(PulsarService pulsar, boolean includeTopicMetrics, b
         if (includeTopicMetrics) {
             printTopicsCountStats(stream, localNamespaceTopicCount, cluster);
         }
+
+        printBrokerStats(stream, cluster, brokerStats);
     }
 
     private static Optional getCompactorMXBean(PulsarService pulsar) {
@@ -301,22 +312,23 @@ private static void getTopicStats(Topic topic, TopicStats stats, boolean include
                 });
     }
 
-    private static void printDefaultBrokerStats(PrometheusMetricStreams stream, String cluster) {
-        // Print metrics with 0 values. This is necessary to have the available brokers being
+    private static void printBrokerStats(PrometheusMetricStreams stream, String cluster,
+                                         AggregatedBrokerStats brokerStats) {
+        // Print metrics values. This is necessary to have the available brokers being
         // reported in the brokers dashboard even if they don't have any topic or traffic
-        writeMetric(stream, "pulsar_topics_count", 0, cluster);
-        writeMetric(stream, "pulsar_subscriptions_count", 0, cluster);
-        writeMetric(stream, "pulsar_producers_count", 0, cluster);
-        writeMetric(stream, "pulsar_consumers_count", 0, cluster);
-        writeMetric(stream, "pulsar_rate_in", 0, cluster);
-        writeMetric(stream, "pulsar_rate_out", 0, cluster);
-        writeMetric(stream, "pulsar_throughput_in", 0, cluster);
-        writeMetric(stream, "pulsar_throughput_out", 0, cluster);
-        writeMetric(stream, "pulsar_storage_size", 0, cluster);
-        writeMetric(stream, "pulsar_storage_logical_size", 0, cluster);
-        writeMetric(stream, "pulsar_storage_write_rate", 0, cluster);
-        writeMetric(stream, "pulsar_storage_read_rate", 0, cluster);
-        writeMetric(stream, "pulsar_msg_backlog", 0, cluster);
+        writeMetric(stream, "pulsar_broker_topics_count", brokerStats.topicsCount, cluster);
+        writeMetric(stream, "pulsar_broker_subscriptions_count", brokerStats.subscriptionsCount, cluster);
+        writeMetric(stream, "pulsar_broker_producers_count", brokerStats.producersCount, cluster);
+        writeMetric(stream, "pulsar_broker_consumers_count", brokerStats.consumersCount, cluster);
+        writeMetric(stream, "pulsar_broker_rate_in", brokerStats.rateIn, cluster);
+        writeMetric(stream, "pulsar_broker_rate_out", brokerStats.rateOut, cluster);
+        writeMetric(stream, "pulsar_broker_throughput_in", brokerStats.throughputIn, cluster);
+        writeMetric(stream, "pulsar_broker_throughput_out", brokerStats.throughputOut, cluster);
+        writeMetric(stream, "pulsar_broker_storage_size", brokerStats.storageSize, cluster);
+        writeMetric(stream, "pulsar_broker_storage_logical_size", brokerStats.storageLogicalSize, cluster);
+        writeMetric(stream, "pulsar_broker_storage_write_rate", brokerStats.storageWriteRate, cluster);
+        writeMetric(stream, "pulsar_broker_storage_read_rate", brokerStats.storageReadRate, cluster);
+        writeMetric(stream, "pulsar_broker_msg_backlog", brokerStats.msgBacklog, cluster);
     }
 
     private static void printTopicsCountStats(PrometheusMetricStreams stream, Map namespaceTopicsCount,
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
index 7eb6afb97cdac..13e67762ace6f 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
@@ -319,11 +319,11 @@ public void testPerTopicStats() throws Exception {
         assertEquals(cm.get(1).tags.get("namespace"), "my-property/use/my-ns");
 
         cm = (List) metrics.get("pulsar_producers_count");
-        assertEquals(cm.size(), 3);
-        assertEquals(cm.get(1).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2");
+        assertEquals(cm.size(), 2);
+        assertEquals(cm.get(1).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1");
+        assertEquals(cm.get(1).tags.get("namespace"), "my-property/use/my-ns");
+        assertEquals(cm.get(1).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1");
         assertEquals(cm.get(1).tags.get("namespace"), "my-property/use/my-ns");
-        assertEquals(cm.get(2).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1");
-        assertEquals(cm.get(2).tags.get("namespace"), "my-property/use/my-ns");
 
         cm = (List) metrics.get("topic_load_times_count");
         assertEquals(cm.size(), 1);
@@ -367,6 +367,97 @@ public void testPerTopicStats() throws Exception {
         c2.close();
     }
 
+    @Test
+    public void testPerBrokerStats() throws Exception {
+        Producer p1 = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1").create();
+        Producer p2 = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic2").create();
+
+        Consumer c1 = pulsarClient.newConsumer()
+                .topic("persistent://my-property/use/my-ns/my-topic1")
+                .subscriptionName("test")
+                .subscribe();
+
+        Consumer c2 = pulsarClient.newConsumer()
+                .topic("persistent://my-property/use/my-ns/my-topic2")
+                .subscriptionName("test")
+                .subscribe();
+
+        final int messages = 10;
+
+        for (int i = 0; i < messages; i++) {
+            String message = "my-message-" + i;
+            p1.send(message.getBytes());
+            p2.send(message.getBytes());
+        }
+
+        for (int i = 0; i < messages; i++) {
+            c1.acknowledge(c1.receive());
+            c2.acknowledge(c2.receive());
+        }
+
+        ByteArrayOutputStream statsOut = new ByteArrayOutputStream();
+        PrometheusMetricsGenerator.generate(pulsar, true, false, false, statsOut);
+        String metricsStr = statsOut.toString();
+        Multimap metrics = parseMetrics(metricsStr);
+
+        Collection brokerMetrics = metrics.get("pulsar_broker_topics_count");
+        assertEquals(brokerMetrics.size(), 1);
+        assertEquals(brokerMetrics.stream().toList().get(0).tags.get("cluster"), "test");
+
+        brokerMetrics = metrics.get("pulsar_broker_subscriptions_count");
+        assertEquals(brokerMetrics.size(), 1);
+        assertEquals(brokerMetrics.stream().toList().get(0).tags.get("cluster"), "test");
+
+        brokerMetrics = metrics.get("pulsar_broker_producers_count");
+        assertEquals(brokerMetrics.size(), 1);
+        assertEquals(brokerMetrics.stream().toList().get(0).tags.get("cluster"), "test");
+
+        brokerMetrics = metrics.get("pulsar_broker_consumers_count");
+        assertEquals(brokerMetrics.size(), 1);
+        assertEquals(brokerMetrics.stream().toList().get(0).tags.get("cluster"), "test");
+
+        brokerMetrics = metrics.get("pulsar_broker_rate_in");
+        assertEquals(brokerMetrics.size(), 1);
+        assertEquals(brokerMetrics.stream().toList().get(0).tags.get("cluster"), "test");
+
+        brokerMetrics = metrics.get("pulsar_broker_rate_out");
+        assertEquals(brokerMetrics.size(), 1);
+        assertEquals(brokerMetrics.stream().toList().get(0).tags.get("cluster"), "test");
+
+        brokerMetrics = metrics.get("pulsar_broker_throughput_in");
+        assertEquals(brokerMetrics.size(), 1);
+        assertEquals(brokerMetrics.stream().toList().get(0).tags.get("cluster"), "test");
+
+        brokerMetrics = metrics.get("pulsar_broker_throughput_out");
+        assertEquals(brokerMetrics.size(), 1);
+        assertEquals(brokerMetrics.stream().toList().get(0).tags.get("cluster"), "test");
+
+        brokerMetrics = metrics.get("pulsar_broker_storage_size");
+        assertEquals(brokerMetrics.size(), 1);
+        assertEquals(brokerMetrics.stream().toList().get(0).tags.get("cluster"), "test");
+
+        brokerMetrics = metrics.get("pulsar_broker_storage_logical_size");
+        assertEquals(brokerMetrics.size(), 1);
+        assertEquals(brokerMetrics.stream().toList().get(0).tags.get("cluster"), "test");
+
+        brokerMetrics = metrics.get("pulsar_broker_storage_write_rate");
+        assertEquals(brokerMetrics.size(), 1);
+        assertEquals(brokerMetrics.stream().toList().get(0).tags.get("cluster"), "test");
+
+        brokerMetrics = metrics.get("pulsar_broker_storage_read_rate");
+        assertEquals(brokerMetrics.size(), 1);
+        assertEquals(brokerMetrics.stream().toList().get(0).tags.get("cluster"), "test");
+
+        brokerMetrics = metrics.get("pulsar_broker_msg_backlog");
+        assertEquals(brokerMetrics.size(), 1);
+        assertEquals(brokerMetrics.stream().toList().get(0).tags.get("cluster"), "test");
+
+        p1.close();
+        p2.close();
+        c1.close();
+        c2.close();
+    }
+
     /**
      * Test that the total message and byte counts for a topic are not reset when a consumer disconnects.
      *
@@ -674,9 +765,9 @@ public void testPerNamespaceStats() throws Exception {
         assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns");
 
         cm = (List) metrics.get("pulsar_producers_count");
-        assertEquals(cm.size(), 2);
-        assertNull(cm.get(1).tags.get("topic"));
-        assertEquals(cm.get(1).tags.get("namespace"), "my-property/use/my-ns");
+        assertEquals(cm.size(), 1);
+        assertNull(cm.get(0).tags.get("topic"));
+        assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns");
 
         cm = (List) metrics.get("pulsar_in_bytes_total");
         assertEquals(cm.size(), 1);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionMetricsTest.java
index 30aedc022534c..4d38f5fad5141 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionMetricsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/TransactionMetricsTest.java
@@ -293,9 +293,9 @@ public void testManagedLedgerMetrics() throws Exception {
         metricsStr = statsOut.toString();
         metrics = parseMetrics(metricsStr);
         metric = metrics.get("pulsar_storage_size");
-        assertEquals(metric.size(), 3);
+        assertEquals(metric.size(), 2);
         metric = metrics.get("pulsar_storage_logical_size");
-        assertEquals(metric.size(), 3);
+        assertEquals(metric.size(), 2);
         metric = metrics.get("pulsar_storage_backlog_size");
         assertEquals(metric.size(), 2);
     }

From ad43c63303d9a3ec8fadffa2077dfad6ce9469fc Mon Sep 17 00:00:00 2001
From: Jiwei Guo 
Date: Sat, 18 Mar 2023 00:10:08 +0800
Subject: [PATCH 121/174] [fix][test] Move MetadataStoreStatsTest to flaky
 group to avoid blocking merge. (#19828)

---
 .../org/apache/pulsar/broker/stats/MetadataStoreStatsTest.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/MetadataStoreStatsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/MetadataStoreStatsTest.java
index 70e58c6b079f5..f4deddf0cec9e 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/MetadataStoreStatsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/MetadataStoreStatsTest.java
@@ -39,7 +39,7 @@
 import org.testng.annotations.Test;
 
 
-@Test(groups = "broker")
+@Test(groups = "flaky")
 public class MetadataStoreStatsTest extends BrokerTestBase {
 
     @BeforeMethod(alwaysRun = true)

From e0098ee0c34643371f53ee921bdc361e163f6387 Mon Sep 17 00:00:00 2001
From: jiangpengcheng 
Date: Sat, 18 Mar 2023 00:39:38 +0800
Subject: [PATCH 122/174] [improve][fn] Support configure compression type
 (#19470)

---
 .../common/functions/ProducerConfig.java      |   2 +
 pulsar-function-go/pb/Function.pb.go          | 654 ++++++++++--------
 pulsar-function-go/pf/instance.go             |  14 +-
 .../instance/JavaInstanceRunnable.java        |   4 +-
 .../pulsar/functions/sink/PulsarSink.java     |   6 +-
 .../instance/src/main/python/Function_pb2.py  | 131 +++-
 .../src/main/python/python_instance.py        |  13 +-
 .../proto/src/main/proto/Function.proto       |   9 +
 .../functions/utils/FunctionCommon.java       |  39 ++
 .../functions/utils/FunctionConfigUtils.java  |   8 +
 .../functions/utils/SourceConfigUtils.java    |   8 +
 .../utils/FunctionConfigUtilsTest.java        |   3 +
 .../utils/SourceConfigUtilsTest.java          |   2 +
 13 files changed, 578 insertions(+), 315 deletions(-)

diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/ProducerConfig.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/ProducerConfig.java
index f6af7a663fa0c..25ca2ad79c877 100644
--- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/ProducerConfig.java
+++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/functions/ProducerConfig.java
@@ -23,6 +23,7 @@
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
+import org.apache.pulsar.client.api.CompressionType;
 
 /**
  * Configuration of the producer inside the function.
@@ -38,4 +39,5 @@ public class ProducerConfig {
     private Boolean useThreadLocalProducers;
     private CryptoConfig cryptoConfig;
     private String batchBuilder;
+    private CompressionType compressionType;
 }
diff --git a/pulsar-function-go/pb/Function.pb.go b/pulsar-function-go/pb/Function.pb.go
index bee1af7cd0c22..bad1afe6658da 100644
--- a/pulsar-function-go/pb/Function.pb.go
+++ b/pulsar-function-go/pb/Function.pb.go
@@ -191,6 +191,61 @@ func (SubscriptionPosition) EnumDescriptor() ([]byte, []int) {
 	return file_Function_proto_rawDescGZIP(), []int{2}
 }
 
+type CompressionType int32
+
+const (
+	CompressionType_LZ4    CompressionType = 0
+	CompressionType_NONE   CompressionType = 1
+	CompressionType_ZLIB   CompressionType = 2
+	CompressionType_ZSTD   CompressionType = 3
+	CompressionType_SNAPPY CompressionType = 4
+)
+
+// Enum value maps for CompressionType.
+var (
+	CompressionType_name = map[int32]string{
+		0: "LZ4",
+		1: "NONE",
+		2: "ZLIB",
+		3: "ZSTD",
+		4: "SNAPPY",
+	}
+	CompressionType_value = map[string]int32{
+		"LZ4":    0,
+		"NONE":   1,
+		"ZLIB":   2,
+		"ZSTD":   3,
+		"SNAPPY": 4,
+	}
+)
+
+func (x CompressionType) Enum() *CompressionType {
+	p := new(CompressionType)
+	*p = x
+	return p
+}
+
+func (x CompressionType) String() string {
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (CompressionType) Descriptor() protoreflect.EnumDescriptor {
+	return file_Function_proto_enumTypes[3].Descriptor()
+}
+
+func (CompressionType) Type() protoreflect.EnumType {
+	return &file_Function_proto_enumTypes[3]
+}
+
+func (x CompressionType) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use CompressionType.Descriptor instead.
+func (CompressionType) EnumDescriptor() ([]byte, []int) {
+	return file_Function_proto_rawDescGZIP(), []int{3}
+}
+
 type FunctionState int32
 
 const (
@@ -221,11 +276,11 @@ func (x FunctionState) String() string {
 }
 
 func (FunctionState) Descriptor() protoreflect.EnumDescriptor {
-	return file_Function_proto_enumTypes[3].Descriptor()
+	return file_Function_proto_enumTypes[4].Descriptor()
 }
 
 func (FunctionState) Type() protoreflect.EnumType {
-	return &file_Function_proto_enumTypes[3]
+	return &file_Function_proto_enumTypes[4]
 }
 
 func (x FunctionState) Number() protoreflect.EnumNumber {
@@ -234,7 +289,7 @@ func (x FunctionState) Number() protoreflect.EnumNumber {
 
 // Deprecated: Use FunctionState.Descriptor instead.
 func (FunctionState) EnumDescriptor() ([]byte, []int) {
-	return file_Function_proto_rawDescGZIP(), []int{3}
+	return file_Function_proto_rawDescGZIP(), []int{4}
 }
 
 type FunctionDetails_Runtime int32
@@ -270,11 +325,11 @@ func (x FunctionDetails_Runtime) String() string {
 }
 
 func (FunctionDetails_Runtime) Descriptor() protoreflect.EnumDescriptor {
-	return file_Function_proto_enumTypes[4].Descriptor()
+	return file_Function_proto_enumTypes[5].Descriptor()
 }
 
 func (FunctionDetails_Runtime) Type() protoreflect.EnumType {
-	return &file_Function_proto_enumTypes[4]
+	return &file_Function_proto_enumTypes[5]
 }
 
 func (x FunctionDetails_Runtime) Number() protoreflect.EnumNumber {
@@ -322,11 +377,11 @@ func (x FunctionDetails_ComponentType) String() string {
 }
 
 func (FunctionDetails_ComponentType) Descriptor() protoreflect.EnumDescriptor {
-	return file_Function_proto_enumTypes[5].Descriptor()
+	return file_Function_proto_enumTypes[6].Descriptor()
 }
 
 func (FunctionDetails_ComponentType) Type() protoreflect.EnumType {
-	return &file_Function_proto_enumTypes[5]
+	return &file_Function_proto_enumTypes[6]
 }
 
 func (x FunctionDetails_ComponentType) Number() protoreflect.EnumNumber {
@@ -374,11 +429,11 @@ func (x CryptoSpec_FailureAction) String() string {
 }
 
 func (CryptoSpec_FailureAction) Descriptor() protoreflect.EnumDescriptor {
-	return file_Function_proto_enumTypes[6].Descriptor()
+	return file_Function_proto_enumTypes[7].Descriptor()
 }
 
 func (CryptoSpec_FailureAction) Type() protoreflect.EnumType {
-	return &file_Function_proto_enumTypes[6]
+	return &file_Function_proto_enumTypes[7]
 }
 
 func (x CryptoSpec_FailureAction) Number() protoreflect.EnumNumber {
@@ -524,6 +579,7 @@ type FunctionDetails struct {
 	Runtime              FunctionDetails_Runtime `protobuf:"varint,8,opt,name=runtime,proto3,enum=proto.FunctionDetails_Runtime" json:"runtime,omitempty"`
 	// Deprecated since, see https://github.com/apache/pulsar/issues/15560
 	//
+	// Deprecated: Do not use.
 	AutoAck              bool                          `protobuf:"varint,9,opt,name=autoAck,proto3" json:"autoAck,omitempty"`
 	Parallelism          int32                         `protobuf:"varint,10,opt,name=parallelism,proto3" json:"parallelism,omitempty"`
 	Source               *SourceSpec                   `protobuf:"bytes,11,opt,name=source,proto3" json:"source,omitempty"`
@@ -844,11 +900,12 @@ type ProducerSpec struct {
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	MaxPendingMessages                 int32       `protobuf:"varint,1,opt,name=maxPendingMessages,proto3" json:"maxPendingMessages,omitempty"`
-	MaxPendingMessagesAcrossPartitions int32       `protobuf:"varint,2,opt,name=maxPendingMessagesAcrossPartitions,proto3" json:"maxPendingMessagesAcrossPartitions,omitempty"`
-	UseThreadLocalProducers            bool        `protobuf:"varint,3,opt,name=useThreadLocalProducers,proto3" json:"useThreadLocalProducers,omitempty"`
-	CryptoSpec                         *CryptoSpec `protobuf:"bytes,4,opt,name=cryptoSpec,proto3" json:"cryptoSpec,omitempty"`
-	BatchBuilder                       string      `protobuf:"bytes,5,opt,name=batchBuilder,proto3" json:"batchBuilder,omitempty"`
+	MaxPendingMessages                 int32           `protobuf:"varint,1,opt,name=maxPendingMessages,proto3" json:"maxPendingMessages,omitempty"`
+	MaxPendingMessagesAcrossPartitions int32           `protobuf:"varint,2,opt,name=maxPendingMessagesAcrossPartitions,proto3" json:"maxPendingMessagesAcrossPartitions,omitempty"`
+	UseThreadLocalProducers            bool            `protobuf:"varint,3,opt,name=useThreadLocalProducers,proto3" json:"useThreadLocalProducers,omitempty"`
+	CryptoSpec                         *CryptoSpec     `protobuf:"bytes,4,opt,name=cryptoSpec,proto3" json:"cryptoSpec,omitempty"`
+	BatchBuilder                       string          `protobuf:"bytes,5,opt,name=batchBuilder,proto3" json:"batchBuilder,omitempty"`
+	CompressionType                    CompressionType `protobuf:"varint,6,opt,name=compressionType,proto3,enum=proto.CompressionType" json:"compressionType,omitempty"`
 }
 
 func (x *ProducerSpec) Reset() {
@@ -918,6 +975,13 @@ func (x *ProducerSpec) GetBatchBuilder() string {
 	return ""
 }
 
+func (x *ProducerSpec) GetCompressionType() CompressionType {
+	if x != nil {
+		return x.CompressionType
+	}
+	return CompressionType_LZ4
+}
+
 type CryptoSpec struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -1017,8 +1081,7 @@ type SourceSpec struct {
 	//
 	// Deprecated: Do not use.
 	TopicsToSerDeClassName map[string]string `protobuf:"bytes,4,rep,name=topicsToSerDeClassName,proto3" json:"topicsToSerDeClassName,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
-	//*
-	//
+	// *
 	InputSpecs map[string]*ConsumerSpec `protobuf:"bytes,10,rep,name=inputSpecs,proto3" json:"inputSpecs,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
 	TimeoutMs  uint64                   `protobuf:"varint,6,opt,name=timeoutMs,proto3" json:"timeoutMs,omitempty"`
 	// Deprecated: Do not use.
@@ -1030,6 +1093,7 @@ type SourceSpec struct {
 	CleanupSubscription          bool                 `protobuf:"varint,11,opt,name=cleanupSubscription,proto3" json:"cleanupSubscription,omitempty"`
 	SubscriptionPosition         SubscriptionPosition `protobuf:"varint,12,opt,name=subscriptionPosition,proto3,enum=proto.SubscriptionPosition" json:"subscriptionPosition,omitempty"`
 	NegativeAckRedeliveryDelayMs uint64               `protobuf:"varint,13,opt,name=negativeAckRedeliveryDelayMs,proto3" json:"negativeAckRedeliveryDelayMs,omitempty"`
+	SkipToLatest                 bool                 `protobuf:"varint,14,opt,name=skipToLatest,proto3" json:"skipToLatest,omitempty"`
 }
 
 func (x *SourceSpec) Reset() {
@@ -1157,6 +1221,13 @@ func (x *SourceSpec) GetNegativeAckRedeliveryDelayMs() uint64 {
 	return 0
 }
 
+func (x *SourceSpec) GetSkipToLatest() bool {
+	if x != nil {
+		return x.SkipToLatest
+	}
+	return false
+}
+
 type SinkSpec struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -1173,7 +1244,7 @@ type SinkSpec struct {
 	// If specified, this will refer to an archive that is
 	// already present in the server
 	Builtin string `protobuf:"bytes,6,opt,name=builtin,proto3" json:"builtin,omitempty"`
-	//*
+	// *
 	// Builtin schema type or custom schema class name
 	SchemaType                   string            `protobuf:"bytes,7,opt,name=schemaType,proto3" json:"schemaType,omitempty"`
 	ForwardSourceMessageProperty bool              `protobuf:"varint,8,opt,name=forwardSourceMessageProperty,proto3" json:"forwardSourceMessageProperty,omitempty"`
@@ -1350,12 +1421,13 @@ type FunctionMetaData struct {
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	FunctionDetails  *FunctionDetails            `protobuf:"bytes,1,opt,name=functionDetails,proto3" json:"functionDetails,omitempty"`
-	PackageLocation  *PackageLocationMetaData    `protobuf:"bytes,2,opt,name=packageLocation,proto3" json:"packageLocation,omitempty"`
-	Version          uint64                      `protobuf:"varint,3,opt,name=version,proto3" json:"version,omitempty"`
-	CreateTime       uint64                      `protobuf:"varint,4,opt,name=createTime,proto3" json:"createTime,omitempty"`
-	InstanceStates   map[int32]FunctionState     `protobuf:"bytes,5,rep,name=instanceStates,proto3" json:"instanceStates,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3,enum=proto.FunctionState"`
-	FunctionAuthSpec *FunctionAuthenticationSpec `protobuf:"bytes,6,opt,name=functionAuthSpec,proto3" json:"functionAuthSpec,omitempty"`
+	FunctionDetails                  *FunctionDetails            `protobuf:"bytes,1,opt,name=functionDetails,proto3" json:"functionDetails,omitempty"`
+	PackageLocation                  *PackageLocationMetaData    `protobuf:"bytes,2,opt,name=packageLocation,proto3" json:"packageLocation,omitempty"`
+	Version                          uint64                      `protobuf:"varint,3,opt,name=version,proto3" json:"version,omitempty"`
+	CreateTime                       uint64                      `protobuf:"varint,4,opt,name=createTime,proto3" json:"createTime,omitempty"`
+	InstanceStates                   map[int32]FunctionState     `protobuf:"bytes,5,rep,name=instanceStates,proto3" json:"instanceStates,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3,enum=proto.FunctionState"`
+	FunctionAuthSpec                 *FunctionAuthenticationSpec `protobuf:"bytes,6,opt,name=functionAuthSpec,proto3" json:"functionAuthSpec,omitempty"`
+	TransformFunctionPackageLocation *PackageLocationMetaData    `protobuf:"bytes,7,opt,name=transformFunctionPackageLocation,proto3" json:"transformFunctionPackageLocation,omitempty"`
 }
 
 func (x *FunctionMetaData) Reset() {
@@ -1432,18 +1504,25 @@ func (x *FunctionMetaData) GetFunctionAuthSpec() *FunctionAuthenticationSpec {
 	return nil
 }
 
+func (x *FunctionMetaData) GetTransformFunctionPackageLocation() *PackageLocationMetaData {
+	if x != nil {
+		return x.TransformFunctionPackageLocation
+	}
+	return nil
+}
+
 type FunctionAuthenticationSpec struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	//*
+	// *
 	// function authentication related data that the function authentication provider
 	// needs to cache/distribute to all workers support function authentication.
 	// Depending on the function authentication provider implementation, this can be the actual auth credentials
 	// or a pointer to the auth credentials that this function should use
 	Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"`
-	//*
+	// *
 	// classname of the function auth provicer this data is relevant to
 	Provider string `protobuf:"bytes,2,opt,name=provider,proto3" json:"provider,omitempty"`
 }
@@ -1776,7 +1855,7 @@ var file_Function_proto_rawDesc = []byte{
 	0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03,
 	0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14,
 	0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76,
-	0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x9f, 0x02, 0x0a, 0x0c, 0x50, 0x72, 0x6f,
+	0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xe1, 0x02, 0x0a, 0x0c, 0x50, 0x72, 0x6f,
 	0x64, 0x75, 0x63, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x12, 0x2e, 0x0a, 0x12, 0x6d, 0x61, 0x78,
 	0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x18,
 	0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x12, 0x6d, 0x61, 0x78, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e,
@@ -1794,202 +1873,220 @@ var file_Function_proto_rawDesc = []byte{
 	0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x53, 0x70, 0x65, 0x63, 0x52, 0x0a, 0x63, 0x72, 0x79, 0x70,
 	0x74, 0x6f, 0x53, 0x70, 0x65, 0x63, 0x12, 0x22, 0x0a, 0x0c, 0x62, 0x61, 0x74, 0x63, 0x68, 0x42,
 	0x75, 0x69, 0x6c, 0x64, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x62, 0x61,
-	0x74, 0x63, 0x68, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x65, 0x72, 0x22, 0xc1, 0x03, 0x0a, 0x0a, 0x43,
-	0x72, 0x79, 0x70, 0x74, 0x6f, 0x53, 0x70, 0x65, 0x63, 0x12, 0x3a, 0x0a, 0x18, 0x63, 0x72, 0x79,
-	0x70, 0x74, 0x6f, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x61, 0x64, 0x65, 0x72, 0x43, 0x6c, 0x61, 0x73,
-	0x73, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x18, 0x63, 0x72, 0x79,
-	0x70, 0x74, 0x6f, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x61, 0x64, 0x65, 0x72, 0x43, 0x6c, 0x61, 0x73,
-	0x73, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x34, 0x0a, 0x15, 0x63, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x4b,
-	0x65, 0x79, 0x52, 0x65, 0x61, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x15, 0x63, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x4b, 0x65, 0x79, 0x52,
-	0x65, 0x61, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x3c, 0x0a, 0x19, 0x70,
-	0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f,
-	0x6e, 0x4b, 0x65, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x19,
-	0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69,
-	0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x61, 0x0a, 0x1b, 0x70, 0x72, 0x6f,
-	0x64, 0x75, 0x63, 0x65, 0x72, 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x46, 0x61, 0x69, 0x6c, 0x75,
-	0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1f,
-	0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x53, 0x70, 0x65,
-	0x63, 0x2e, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52,
-	0x1b, 0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x46,
-	0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x61, 0x0a, 0x1b,
-	0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x46, 0x61,
-	0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28,
-	0x0e, 0x32, 0x1f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f,
-	0x53, 0x70, 0x65, 0x63, 0x2e, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69,
-	0x6f, 0x6e, 0x52, 0x1b, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x43, 0x72, 0x79, 0x70,
-	0x74, 0x6f, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22,
-	0x3d, 0x0a, 0x0d, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e,
-	0x12, 0x08, 0x0a, 0x04, 0x46, 0x41, 0x49, 0x4c, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x44, 0x49,
-	0x53, 0x43, 0x41, 0x52, 0x44, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x43, 0x4f, 0x4e, 0x53, 0x55,
-	0x4d, 0x45, 0x10, 0x02, 0x12, 0x08, 0x0a, 0x04, 0x53, 0x45, 0x4e, 0x44, 0x10, 0x0a, 0x22, 0xd1,
-	0x06, 0x0a, 0x0a, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x70, 0x65, 0x63, 0x12, 0x1c, 0x0a,
-	0x09, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
-	0x52, 0x09, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x63,
-	0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f,
-	0x6e, 0x66, 0x69, 0x67, 0x73, 0x12, 0x24, 0x0a, 0x0d, 0x74, 0x79, 0x70, 0x65, 0x43, 0x6c, 0x61,
-	0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x74, 0x79,
-	0x70, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x43, 0x0a, 0x10, 0x73,
-	0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x18,
-	0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x75,
-	0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x10,
-	0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65,
-	0x12, 0x69, 0x0a, 0x16, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x54, 0x6f, 0x53, 0x65, 0x72, 0x44,
-	0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b,
-	0x32, 0x2d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53,
-	0x70, 0x65, 0x63, 0x2e, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x54, 0x6f, 0x53, 0x65, 0x72, 0x44,
-	0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x42,
-	0x02, 0x18, 0x01, 0x52, 0x16, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x54, 0x6f, 0x53, 0x65, 0x72,
-	0x44, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x41, 0x0a, 0x0a, 0x69,
-	0x6e, 0x70, 0x75, 0x74, 0x53, 0x70, 0x65, 0x63, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x0b, 0x32,
-	0x21, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x70,
-	0x65, 0x63, 0x2e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x53, 0x70, 0x65, 0x63, 0x73, 0x45, 0x6e, 0x74,
-	0x72, 0x79, 0x52, 0x0a, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x53, 0x70, 0x65, 0x63, 0x73, 0x12, 0x1c,
-	0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28,
-	0x04, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x12, 0x28, 0x0a, 0x0d,
-	0x74, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x50, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x18, 0x07, 0x20,
-	0x01, 0x28, 0x09, 0x42, 0x02, 0x18, 0x01, 0x52, 0x0d, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x50,
-	0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x74, 0x69,
-	0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x74, 0x69, 0x6e,
-	0x12, 0x2a, 0x0a, 0x10, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e,
-	0x4e, 0x61, 0x6d, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x73, 0x75, 0x62, 0x73,
-	0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x30, 0x0a, 0x13,
-	0x63, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74,
-	0x69, 0x6f, 0x6e, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x63, 0x6c, 0x65, 0x61, 0x6e,
-	0x75, 0x70, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x4f,
-	0x0a, 0x14, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x6f,
-	0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1b, 0x2e, 0x70,
-	0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f,
-	0x6e, 0x50, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x14, 0x73, 0x75, 0x62, 0x73, 0x63,
-	0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12,
-	0x42, 0x0a, 0x1c, 0x6e, 0x65, 0x67, 0x61, 0x74, 0x69, 0x76, 0x65, 0x41, 0x63, 0x6b, 0x52, 0x65,
-	0x64, 0x65, 0x6c, 0x69, 0x76, 0x65, 0x72, 0x79, 0x44, 0x65, 0x6c, 0x61, 0x79, 0x4d, 0x73, 0x18,
-	0x0d, 0x20, 0x01, 0x28, 0x04, 0x52, 0x1c, 0x6e, 0x65, 0x67, 0x61, 0x74, 0x69, 0x76, 0x65, 0x41,
-	0x63, 0x6b, 0x52, 0x65, 0x64, 0x65, 0x6c, 0x69, 0x76, 0x65, 0x72, 0x79, 0x44, 0x65, 0x6c, 0x61,
-	0x79, 0x4d, 0x73, 0x1a, 0x49, 0x0a, 0x1b, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x54, 0x6f, 0x53,
-	0x65, 0x72, 0x44, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74,
-	0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20,
-	0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x52,
-	0x0a, 0x0f, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x53, 0x70, 0x65, 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72,
-	0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03,
-	0x6b, 0x65, 0x79, 0x12, 0x29, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01,
-	0x28, 0x0b, 0x32, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x73, 0x75,
-	0x6d, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02,
-	0x38, 0x01, 0x22, 0x95, 0x05, 0x0a, 0x08, 0x53, 0x69, 0x6e, 0x6b, 0x53, 0x70, 0x65, 0x63, 0x12,
+	0x74, 0x63, 0x68, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x65, 0x72, 0x12, 0x40, 0x0a, 0x0f, 0x63, 0x6f,
+	0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x18, 0x06, 0x20,
+	0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x6f, 0x6d, 0x70,
+	0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0f, 0x63, 0x6f, 0x6d,
+	0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x22, 0xc1, 0x03, 0x0a,
+	0x0a, 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x53, 0x70, 0x65, 0x63, 0x12, 0x3a, 0x0a, 0x18, 0x63,
+	0x72, 0x79, 0x70, 0x74, 0x6f, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x61, 0x64, 0x65, 0x72, 0x43, 0x6c,
+	0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x18, 0x63,
+	0x72, 0x79, 0x70, 0x74, 0x6f, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x61, 0x64, 0x65, 0x72, 0x43, 0x6c,
+	0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x34, 0x0a, 0x15, 0x63, 0x72, 0x79, 0x70, 0x74,
+	0x6f, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x61, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67,
+	0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x15, 0x63, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x4b, 0x65,
+	0x79, 0x52, 0x65, 0x61, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x3c, 0x0a,
+	0x19, 0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74,
+	0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09,
+	0x52, 0x19, 0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70,
+	0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x61, 0x0a, 0x1b, 0x70,
+	0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x46, 0x61, 0x69,
+	0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e,
+	0x32, 0x1f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x53,
+	0x70, 0x65, 0x63, 0x2e, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f,
+	0x6e, 0x52, 0x1b, 0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x43, 0x72, 0x79, 0x70, 0x74,
+	0x6f, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x61,
+	0x0a, 0x1b, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f,
+	0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20,
+	0x01, 0x28, 0x0e, 0x32, 0x1f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, 0x72, 0x79, 0x70,
+	0x74, 0x6f, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63,
+	0x74, 0x69, 0x6f, 0x6e, 0x52, 0x1b, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x43, 0x72,
+	0x79, 0x70, 0x74, 0x6f, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f,
+	0x6e, 0x22, 0x3d, 0x0a, 0x0d, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x41, 0x63, 0x74, 0x69,
+	0x6f, 0x6e, 0x12, 0x08, 0x0a, 0x04, 0x46, 0x41, 0x49, 0x4c, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07,
+	0x44, 0x49, 0x53, 0x43, 0x41, 0x52, 0x44, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x43, 0x4f, 0x4e,
+	0x53, 0x55, 0x4d, 0x45, 0x10, 0x02, 0x12, 0x08, 0x0a, 0x04, 0x53, 0x45, 0x4e, 0x44, 0x10, 0x0a,
+	0x22, 0xf5, 0x06, 0x0a, 0x0a, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x70, 0x65, 0x63, 0x12,
 	0x1c, 0x0a, 0x09, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01,
 	0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a,
 	0x07, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07,
 	0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x12, 0x24, 0x0a, 0x0d, 0x74, 0x79, 0x70, 0x65, 0x43,
 	0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d,
-	0x74, 0x79, 0x70, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a,
-	0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f,
-	0x70, 0x69, 0x63, 0x12, 0x37, 0x0a, 0x0c, 0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x53,
-	0x70, 0x65, 0x63, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74,
-	0x6f, 0x2e, 0x50, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x52, 0x0c,
-	0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x12, 0x26, 0x0a, 0x0e,
-	0x73, 0x65, 0x72, 0x44, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x04,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x73, 0x65, 0x72, 0x44, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73,
-	0x4e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x74, 0x69, 0x6e, 0x18,
-	0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x74, 0x69, 0x6e, 0x12, 0x1e,
-	0x0a, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x18, 0x07, 0x20, 0x01,
-	0x28, 0x09, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x42,
-	0x0a, 0x1c, 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4d,
-	0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x18, 0x08,
-	0x20, 0x01, 0x28, 0x08, 0x52, 0x1c, 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x53, 0x6f, 0x75,
-	0x72, 0x63, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72,
-	0x74, 0x79, 0x12, 0x51, 0x0a, 0x10, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x72, 0x6f, 0x70,
-	0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x70,
-	0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x53, 0x63,
-	0x68, 0x65, 0x6d, 0x61, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e,
-	0x74, 0x72, 0x79, 0x52, 0x10, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x72, 0x6f, 0x70, 0x65,
-	0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x57, 0x0a, 0x12, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65,
-	0x72, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28,
-	0x0b, 0x32, 0x27, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x53, 0x70,
-	0x65, 0x63, 0x2e, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x70, 0x65,
-	0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x12, 0x63, 0x6f, 0x6e, 0x73,
-	0x75, 0x6d, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x1a, 0x43,
-	0x0a, 0x15, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69,
-	0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c,
-	0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a,
-	0x02, 0x38, 0x01, 0x1a, 0x45, 0x0a, 0x17, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x50,
-	0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10,
-	0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79,
-	0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x67, 0x0a, 0x17, 0x50, 0x61,
-	0x63, 0x6b, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74,
-	0x61, 0x44, 0x61, 0x74, 0x61, 0x12, 0x20, 0x0a, 0x0b, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65,
-	0x50, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x70, 0x61, 0x63, 0x6b,
-	0x61, 0x67, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x2a, 0x0a, 0x10, 0x6f, 0x72, 0x69, 0x67, 0x69,
-	0x6e, 0x61, 0x6c, 0x46, 0x69, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28,
-	0x09, 0x52, 0x10, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, 0x6c, 0x46, 0x69, 0x6c, 0x65, 0x4e,
-	0x61, 0x6d, 0x65, 0x22, 0xd5, 0x03, 0x0a, 0x10, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e,
-	0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x12, 0x40, 0x0a, 0x0f, 0x66, 0x75, 0x6e, 0x63,
-	0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28,
-	0x0b, 0x32, 0x16, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69,
-	0x6f, 0x6e, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x0f, 0x66, 0x75, 0x6e, 0x63, 0x74,
-	0x69, 0x6f, 0x6e, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x48, 0x0a, 0x0f, 0x70, 0x61,
-	0x63, 0x6b, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20,
-	0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x50, 0x61, 0x63, 0x6b,
-	0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x61, 0x44,
-	0x61, 0x74, 0x61, 0x52, 0x0f, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61,
-	0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18,
-	0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1e,
-	0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01,
-	0x28, 0x04, 0x52, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x53,
-	0x0a, 0x0e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x73,
-	0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46,
-	0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x2e,
-	0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x73, 0x45, 0x6e,
-	0x74, 0x72, 0x79, 0x52, 0x0e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61,
-	0x74, 0x65, 0x73, 0x12, 0x4d, 0x0a, 0x10, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41,
-	0x75, 0x74, 0x68, 0x53, 0x70, 0x65, 0x63, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e,
-	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x75,
-	0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63,
-	0x52, 0x10, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x75, 0x74, 0x68, 0x53, 0x70,
-	0x65, 0x63, 0x1a, 0x57, 0x0a, 0x13, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74,
-	0x61, 0x74, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79,
-	0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2a, 0x0a, 0x05, 0x76,
-	0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x14, 0x2e, 0x70, 0x72, 0x6f,
-	0x74, 0x6f, 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65,
-	0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x4c, 0x0a, 0x1a, 0x46,
-	0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63,
-	0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74,
-	0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x1a, 0x0a,
-	0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x22, 0x6f, 0x0a, 0x08, 0x49, 0x6e, 0x73,
-	0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x43, 0x0a, 0x10, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f,
-	0x6e, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32,
-	0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e,
-	0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x52, 0x10, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69,
-	0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x6e,
-	0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a,
-	0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x22, 0x55, 0x0a, 0x0a, 0x41, 0x73,
-	0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2b, 0x0a, 0x08, 0x69, 0x6e, 0x73, 0x74,
-	0x61, 0x6e, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x70, 0x72, 0x6f,
-	0x74, 0x6f, 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x08, 0x69, 0x6e, 0x73,
-	0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x49,
-	0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x49,
-	0x64, 0x2a, 0x5b, 0x0a, 0x14, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x47,
-	0x75, 0x61, 0x72, 0x61, 0x6e, 0x74, 0x65, 0x65, 0x73, 0x12, 0x10, 0x0a, 0x0c, 0x41, 0x54, 0x4c,
-	0x45, 0x41, 0x53, 0x54, 0x5f, 0x4f, 0x4e, 0x43, 0x45, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x41,
-	0x54, 0x4d, 0x4f, 0x53, 0x54, 0x5f, 0x4f, 0x4e, 0x43, 0x45, 0x10, 0x01, 0x12, 0x14, 0x0a, 0x10,
-	0x45, 0x46, 0x46, 0x45, 0x43, 0x54, 0x49, 0x56, 0x45, 0x4c, 0x59, 0x5f, 0x4f, 0x4e, 0x43, 0x45,
-	0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06, 0x4d, 0x41, 0x4e, 0x55, 0x41, 0x4c, 0x10, 0x03, 0x2a, 0x3c,
-	0x0a, 0x10, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79,
-	0x70, 0x65, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x48, 0x41, 0x52, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0c,
-	0x0a, 0x08, 0x46, 0x41, 0x49, 0x4c, 0x4f, 0x56, 0x45, 0x52, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a,
-	0x4b, 0x45, 0x59, 0x5f, 0x53, 0x48, 0x41, 0x52, 0x45, 0x44, 0x10, 0x02, 0x2a, 0x30, 0x0a, 0x14,
-	0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x6f, 0x73, 0x69,
-	0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0a, 0x0a, 0x06, 0x4c, 0x41, 0x54, 0x45, 0x53, 0x54, 0x10, 0x00,
-	0x12, 0x0c, 0x0a, 0x08, 0x45, 0x41, 0x52, 0x4c, 0x49, 0x45, 0x53, 0x54, 0x10, 0x01, 0x2a, 0x29,
-	0x0a, 0x0d, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12,
-	0x0b, 0x0a, 0x07, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07,
-	0x53, 0x54, 0x4f, 0x50, 0x50, 0x45, 0x44, 0x10, 0x01, 0x42, 0x2d, 0x0a, 0x21, 0x6f, 0x72, 0x67,
-	0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x70, 0x75, 0x6c, 0x73, 0x61, 0x72, 0x2e, 0x66,
-	0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x42, 0x08,
-	0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x74, 0x79, 0x70, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x43, 0x0a,
+	0x10, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70,
+	0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e,
+	0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65,
+	0x52, 0x10, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79,
+	0x70, 0x65, 0x12, 0x69, 0x0a, 0x16, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x54, 0x6f, 0x53, 0x65,
+	0x72, 0x44, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x03,
+	0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63,
+	0x65, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x54, 0x6f, 0x53, 0x65,
+	0x72, 0x44, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72,
+	0x79, 0x42, 0x02, 0x18, 0x01, 0x52, 0x16, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x54, 0x6f, 0x53,
+	0x65, 0x72, 0x44, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x41, 0x0a,
+	0x0a, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x53, 0x70, 0x65, 0x63, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28,
+	0x0b, 0x32, 0x21, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65,
+	0x53, 0x70, 0x65, 0x63, 0x2e, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x53, 0x70, 0x65, 0x63, 0x73, 0x45,
+	0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x53, 0x70, 0x65, 0x63, 0x73,
+	0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x18, 0x06, 0x20,
+	0x01, 0x28, 0x04, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x12, 0x28,
+	0x0a, 0x0d, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x50, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x18,
+	0x07, 0x20, 0x01, 0x28, 0x09, 0x42, 0x02, 0x18, 0x01, 0x52, 0x0d, 0x74, 0x6f, 0x70, 0x69, 0x63,
+	0x73, 0x50, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x75, 0x69, 0x6c,
+	0x74, 0x69, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x74,
+	0x69, 0x6e, 0x12, 0x2a, 0x0a, 0x10, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69,
+	0x6f, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x73, 0x75,
+	0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x30,
+	0x0a, 0x13, 0x63, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69,
+	0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x63, 0x6c, 0x65,
+	0x61, 0x6e, 0x75, 0x70, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e,
+	0x12, 0x4f, 0x0a, 0x14, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e,
+	0x50, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1b,
+	0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74,
+	0x69, 0x6f, 0x6e, 0x50, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x14, 0x73, 0x75, 0x62,
+	0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f,
+	0x6e, 0x12, 0x42, 0x0a, 0x1c, 0x6e, 0x65, 0x67, 0x61, 0x74, 0x69, 0x76, 0x65, 0x41, 0x63, 0x6b,
+	0x52, 0x65, 0x64, 0x65, 0x6c, 0x69, 0x76, 0x65, 0x72, 0x79, 0x44, 0x65, 0x6c, 0x61, 0x79, 0x4d,
+	0x73, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x04, 0x52, 0x1c, 0x6e, 0x65, 0x67, 0x61, 0x74, 0x69, 0x76,
+	0x65, 0x41, 0x63, 0x6b, 0x52, 0x65, 0x64, 0x65, 0x6c, 0x69, 0x76, 0x65, 0x72, 0x79, 0x44, 0x65,
+	0x6c, 0x61, 0x79, 0x4d, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x73, 0x6b, 0x69, 0x70, 0x54, 0x6f, 0x4c,
+	0x61, 0x74, 0x65, 0x73, 0x74, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x73, 0x6b, 0x69,
+	0x70, 0x54, 0x6f, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x1a, 0x49, 0x0a, 0x1b, 0x54, 0x6f, 0x70,
+	0x69, 0x63, 0x73, 0x54, 0x6f, 0x53, 0x65, 0x72, 0x44, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e,
+	0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61,
+	0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65,
+	0x3a, 0x02, 0x38, 0x01, 0x1a, 0x52, 0x0a, 0x0f, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x53, 0x70, 0x65,
+	0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x29, 0x0a, 0x05, 0x76, 0x61, 0x6c,
+	0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x2e, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x52, 0x05, 0x76,
+	0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x95, 0x05, 0x0a, 0x08, 0x53, 0x69, 0x6e,
+	0x6b, 0x53, 0x70, 0x65, 0x63, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61,
+	0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x4e,
+	0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x18, 0x02,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x12, 0x24, 0x0a,
+	0x0d, 0x74, 0x79, 0x70, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x05,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x74, 0x79, 0x70, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e,
+	0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x18, 0x03, 0x20, 0x01,
+	0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x12, 0x37, 0x0a, 0x0c, 0x70, 0x72, 0x6f,
+	0x64, 0x75, 0x63, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32,
+	0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x50, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72,
+	0x53, 0x70, 0x65, 0x63, 0x52, 0x0c, 0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x53, 0x70,
+	0x65, 0x63, 0x12, 0x26, 0x0a, 0x0e, 0x73, 0x65, 0x72, 0x44, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73,
+	0x4e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x73, 0x65, 0x72, 0x44,
+	0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x75,
+	0x69, 0x6c, 0x74, 0x69, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x62, 0x75, 0x69,
+	0x6c, 0x74, 0x69, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x79,
+	0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61,
+	0x54, 0x79, 0x70, 0x65, 0x12, 0x42, 0x0a, 0x1c, 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x53,
+	0x6f, 0x75, 0x72, 0x63, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x50, 0x72, 0x6f, 0x70,
+	0x65, 0x72, 0x74, 0x79, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x1c, 0x66, 0x6f, 0x72, 0x77,
+	0x61, 0x72, 0x64, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65,
+	0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x12, 0x51, 0x0a, 0x10, 0x73, 0x63, 0x68, 0x65,
+	0x6d, 0x61, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x09, 0x20, 0x03,
+	0x28, 0x0b, 0x32, 0x25, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x53,
+	0x70, 0x65, 0x63, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72,
+	0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x10, 0x73, 0x63, 0x68, 0x65, 0x6d,
+	0x61, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x57, 0x0a, 0x12, 0x63,
+	0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65,
+	0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e,
+	0x53, 0x69, 0x6e, 0x6b, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65,
+	0x72, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79,
+	0x52, 0x12, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72,
+	0x74, 0x69, 0x65, 0x73, 0x1a, 0x43, 0x0a, 0x15, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x72,
+	0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a,
+	0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12,
+	0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05,
+	0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x45, 0x0a, 0x17, 0x43, 0x6f, 0x6e,
+	0x73, 0x75, 0x6d, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45,
+	0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18,
+	0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01,
+	0x22, 0x67, 0x0a, 0x17, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74,
+	0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x12, 0x20, 0x0a, 0x0b, 0x70,
+	0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x50, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x0b, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x2a, 0x0a,
+	0x10, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, 0x6c, 0x46, 0x69, 0x6c, 0x65, 0x4e, 0x61, 0x6d,
+	0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61,
+	0x6c, 0x46, 0x69, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0xc1, 0x04, 0x0a, 0x10, 0x46, 0x75,
+	0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x12, 0x40,
+	0x0a, 0x0f, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c,
+	0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e,
+	0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52,
+	0x0f, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73,
+	0x12, 0x48, 0x0a, 0x0f, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74,
+	0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x74,
+	0x6f, 0x2e, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f,
+	0x6e, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x52, 0x0f, 0x70, 0x61, 0x63, 0x6b, 0x61,
+	0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65,
+	0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x76, 0x65, 0x72,
+	0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69,
+	0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65,
+	0x54, 0x69, 0x6d, 0x65, 0x12, 0x53, 0x0a, 0x0e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65,
+	0x53, 0x74, 0x61, 0x74, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74,
+	0x61, 0x44, 0x61, 0x74, 0x61, 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74,
+	0x61, 0x74, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0e, 0x69, 0x6e, 0x73, 0x74, 0x61,
+	0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x73, 0x12, 0x4d, 0x0a, 0x10, 0x66, 0x75, 0x6e,
+	0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x75, 0x74, 0x68, 0x53, 0x70, 0x65, 0x63, 0x18, 0x06, 0x20,
+	0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x75, 0x6e, 0x63,
+	0x74, 0x69, 0x6f, 0x6e, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x69,
+	0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x52, 0x10, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e,
+	0x41, 0x75, 0x74, 0x68, 0x53, 0x70, 0x65, 0x63, 0x12, 0x6a, 0x0a, 0x20, 0x74, 0x72, 0x61, 0x6e,
+	0x73, 0x66, 0x6f, 0x72, 0x6d, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x61, 0x63,
+	0x6b, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01,
+	0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x50, 0x61, 0x63, 0x6b, 0x61,
+	0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61,
+	0x74, 0x61, 0x52, 0x20, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x46, 0x75, 0x6e,
+	0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61,
+	0x74, 0x69, 0x6f, 0x6e, 0x1a, 0x57, 0x0a, 0x13, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65,
+	0x53, 0x74, 0x61, 0x74, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b,
+	0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2a, 0x0a,
+	0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x14, 0x2e, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61,
+	0x74, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x4c, 0x0a,
+	0x1a, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74,
+	0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x12, 0x12, 0x0a, 0x04, 0x64,
+	0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12,
+	0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x22, 0x6f, 0x0a, 0x08, 0x49,
+	0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x43, 0x0a, 0x10, 0x66, 0x75, 0x6e, 0x63, 0x74,
+	0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x0b, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69,
+	0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x52, 0x10, 0x66, 0x75, 0x6e, 0x63,
+	0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x12, 0x1e, 0x0a, 0x0a,
+	0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05,
+	0x52, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x22, 0x55, 0x0a, 0x0a,
+	0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2b, 0x0a, 0x08, 0x69, 0x6e,
+	0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x08, 0x69,
+	0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x65,
+	0x72, 0x49, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x65,
+	0x72, 0x49, 0x64, 0x2a, 0x5b, 0x0a, 0x14, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e,
+	0x67, 0x47, 0x75, 0x61, 0x72, 0x61, 0x6e, 0x74, 0x65, 0x65, 0x73, 0x12, 0x10, 0x0a, 0x0c, 0x41,
+	0x54, 0x4c, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x4f, 0x4e, 0x43, 0x45, 0x10, 0x00, 0x12, 0x0f, 0x0a,
+	0x0b, 0x41, 0x54, 0x4d, 0x4f, 0x53, 0x54, 0x5f, 0x4f, 0x4e, 0x43, 0x45, 0x10, 0x01, 0x12, 0x14,
+	0x0a, 0x10, 0x45, 0x46, 0x46, 0x45, 0x43, 0x54, 0x49, 0x56, 0x45, 0x4c, 0x59, 0x5f, 0x4f, 0x4e,
+	0x43, 0x45, 0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06, 0x4d, 0x41, 0x4e, 0x55, 0x41, 0x4c, 0x10, 0x03,
+	0x2a, 0x3c, 0x0a, 0x10, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e,
+	0x54, 0x79, 0x70, 0x65, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x48, 0x41, 0x52, 0x45, 0x44, 0x10, 0x00,
+	0x12, 0x0c, 0x0a, 0x08, 0x46, 0x41, 0x49, 0x4c, 0x4f, 0x56, 0x45, 0x52, 0x10, 0x01, 0x12, 0x0e,
+	0x0a, 0x0a, 0x4b, 0x45, 0x59, 0x5f, 0x53, 0x48, 0x41, 0x52, 0x45, 0x44, 0x10, 0x02, 0x2a, 0x30,
+	0x0a, 0x14, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x6f,
+	0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0a, 0x0a, 0x06, 0x4c, 0x41, 0x54, 0x45, 0x53, 0x54,
+	0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x45, 0x41, 0x52, 0x4c, 0x49, 0x45, 0x53, 0x54, 0x10, 0x01,
+	0x2a, 0x44, 0x0a, 0x0f, 0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54,
+	0x79, 0x70, 0x65, 0x12, 0x07, 0x0a, 0x03, 0x4c, 0x5a, 0x34, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04,
+	0x4e, 0x4f, 0x4e, 0x45, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x5a, 0x4c, 0x49, 0x42, 0x10, 0x02,
+	0x12, 0x08, 0x0a, 0x04, 0x5a, 0x53, 0x54, 0x44, 0x10, 0x03, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x4e,
+	0x41, 0x50, 0x50, 0x59, 0x10, 0x04, 0x2a, 0x29, 0x0a, 0x0d, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69,
+	0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x55, 0x4e, 0x4e, 0x49,
+	0x4e, 0x47, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x54, 0x4f, 0x50, 0x50, 0x45, 0x44, 0x10,
+	0x01, 0x42, 0x2d, 0x0a, 0x21, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e,
+	0x70, 0x75, 0x6c, 0x73, 0x61, 0x72, 0x2e, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73,
+	0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x42, 0x08, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e,
+	0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
@@ -2004,74 +2101,77 @@ func file_Function_proto_rawDescGZIP() []byte {
 	return file_Function_proto_rawDescData
 }
 
-var file_Function_proto_enumTypes = make([]protoimpl.EnumInfo, 7)
+var file_Function_proto_enumTypes = make([]protoimpl.EnumInfo, 8)
 var file_Function_proto_msgTypes = make([]protoimpl.MessageInfo, 21)
 var file_Function_proto_goTypes = []interface{}{
 	(ProcessingGuarantees)(0),              // 0: proto.ProcessingGuarantees
 	(SubscriptionType)(0),                  // 1: proto.SubscriptionType
 	(SubscriptionPosition)(0),              // 2: proto.SubscriptionPosition
-	(FunctionState)(0),                     // 3: proto.FunctionState
-	(FunctionDetails_Runtime)(0),           // 4: proto.FunctionDetails.Runtime
-	(FunctionDetails_ComponentType)(0),     // 5: proto.FunctionDetails.ComponentType
-	(CryptoSpec_FailureAction)(0),          // 6: proto.CryptoSpec.FailureAction
-	(*Resources)(nil),                      // 7: proto.Resources
-	(*RetryDetails)(nil),                   // 8: proto.RetryDetails
-	(*FunctionDetails)(nil),                // 9: proto.FunctionDetails
-	(*ConsumerSpec)(nil),                   // 10: proto.ConsumerSpec
-	(*ProducerSpec)(nil),                   // 11: proto.ProducerSpec
-	(*CryptoSpec)(nil),                     // 12: proto.CryptoSpec
-	(*SourceSpec)(nil),                     // 13: proto.SourceSpec
-	(*SinkSpec)(nil),                       // 14: proto.SinkSpec
-	(*PackageLocationMetaData)(nil),        // 15: proto.PackageLocationMetaData
-	(*FunctionMetaData)(nil),               // 16: proto.FunctionMetaData
-	(*FunctionAuthenticationSpec)(nil),     // 17: proto.FunctionAuthenticationSpec
-	(*Instance)(nil),                       // 18: proto.Instance
-	(*Assignment)(nil),                     // 19: proto.Assignment
-	(*ConsumerSpec_ReceiverQueueSize)(nil), // 20: proto.ConsumerSpec.ReceiverQueueSize
-	nil,                                    // 21: proto.ConsumerSpec.SchemaPropertiesEntry
-	nil,                                    // 22: proto.ConsumerSpec.ConsumerPropertiesEntry
-	nil,                                    // 23: proto.SourceSpec.TopicsToSerDeClassNameEntry
-	nil,                                    // 24: proto.SourceSpec.InputSpecsEntry
-	nil,                                    // 25: proto.SinkSpec.SchemaPropertiesEntry
-	nil,                                    // 26: proto.SinkSpec.ConsumerPropertiesEntry
-	nil,                                    // 27: proto.FunctionMetaData.InstanceStatesEntry
+	(CompressionType)(0),                   // 3: proto.CompressionType
+	(FunctionState)(0),                     // 4: proto.FunctionState
+	(FunctionDetails_Runtime)(0),           // 5: proto.FunctionDetails.Runtime
+	(FunctionDetails_ComponentType)(0),     // 6: proto.FunctionDetails.ComponentType
+	(CryptoSpec_FailureAction)(0),          // 7: proto.CryptoSpec.FailureAction
+	(*Resources)(nil),                      // 8: proto.Resources
+	(*RetryDetails)(nil),                   // 9: proto.RetryDetails
+	(*FunctionDetails)(nil),                // 10: proto.FunctionDetails
+	(*ConsumerSpec)(nil),                   // 11: proto.ConsumerSpec
+	(*ProducerSpec)(nil),                   // 12: proto.ProducerSpec
+	(*CryptoSpec)(nil),                     // 13: proto.CryptoSpec
+	(*SourceSpec)(nil),                     // 14: proto.SourceSpec
+	(*SinkSpec)(nil),                       // 15: proto.SinkSpec
+	(*PackageLocationMetaData)(nil),        // 16: proto.PackageLocationMetaData
+	(*FunctionMetaData)(nil),               // 17: proto.FunctionMetaData
+	(*FunctionAuthenticationSpec)(nil),     // 18: proto.FunctionAuthenticationSpec
+	(*Instance)(nil),                       // 19: proto.Instance
+	(*Assignment)(nil),                     // 20: proto.Assignment
+	(*ConsumerSpec_ReceiverQueueSize)(nil), // 21: proto.ConsumerSpec.ReceiverQueueSize
+	nil,                                    // 22: proto.ConsumerSpec.SchemaPropertiesEntry
+	nil,                                    // 23: proto.ConsumerSpec.ConsumerPropertiesEntry
+	nil,                                    // 24: proto.SourceSpec.TopicsToSerDeClassNameEntry
+	nil,                                    // 25: proto.SourceSpec.InputSpecsEntry
+	nil,                                    // 26: proto.SinkSpec.SchemaPropertiesEntry
+	nil,                                    // 27: proto.SinkSpec.ConsumerPropertiesEntry
+	nil,                                    // 28: proto.FunctionMetaData.InstanceStatesEntry
 }
 var file_Function_proto_depIdxs = []int32{
 	0,  // 0: proto.FunctionDetails.processingGuarantees:type_name -> proto.ProcessingGuarantees
-	4,  // 1: proto.FunctionDetails.runtime:type_name -> proto.FunctionDetails.Runtime
-	13, // 2: proto.FunctionDetails.source:type_name -> proto.SourceSpec
-	14, // 3: proto.FunctionDetails.sink:type_name -> proto.SinkSpec
-	7,  // 4: proto.FunctionDetails.resources:type_name -> proto.Resources
-	8,  // 5: proto.FunctionDetails.retryDetails:type_name -> proto.RetryDetails
-	5,  // 6: proto.FunctionDetails.componentType:type_name -> proto.FunctionDetails.ComponentType
+	5,  // 1: proto.FunctionDetails.runtime:type_name -> proto.FunctionDetails.Runtime
+	14, // 2: proto.FunctionDetails.source:type_name -> proto.SourceSpec
+	15, // 3: proto.FunctionDetails.sink:type_name -> proto.SinkSpec
+	8,  // 4: proto.FunctionDetails.resources:type_name -> proto.Resources
+	9,  // 5: proto.FunctionDetails.retryDetails:type_name -> proto.RetryDetails
+	6,  // 6: proto.FunctionDetails.componentType:type_name -> proto.FunctionDetails.ComponentType
 	2,  // 7: proto.FunctionDetails.subscriptionPosition:type_name -> proto.SubscriptionPosition
-	20, // 8: proto.ConsumerSpec.receiverQueueSize:type_name -> proto.ConsumerSpec.ReceiverQueueSize
-	21, // 9: proto.ConsumerSpec.schemaProperties:type_name -> proto.ConsumerSpec.SchemaPropertiesEntry
-	22, // 10: proto.ConsumerSpec.consumerProperties:type_name -> proto.ConsumerSpec.ConsumerPropertiesEntry
-	12, // 11: proto.ConsumerSpec.cryptoSpec:type_name -> proto.CryptoSpec
-	12, // 12: proto.ProducerSpec.cryptoSpec:type_name -> proto.CryptoSpec
-	6,  // 13: proto.CryptoSpec.producerCryptoFailureAction:type_name -> proto.CryptoSpec.FailureAction
-	6,  // 14: proto.CryptoSpec.consumerCryptoFailureAction:type_name -> proto.CryptoSpec.FailureAction
-	1,  // 15: proto.SourceSpec.subscriptionType:type_name -> proto.SubscriptionType
-	23, // 16: proto.SourceSpec.topicsToSerDeClassName:type_name -> proto.SourceSpec.TopicsToSerDeClassNameEntry
-	24, // 17: proto.SourceSpec.inputSpecs:type_name -> proto.SourceSpec.InputSpecsEntry
-	2,  // 18: proto.SourceSpec.subscriptionPosition:type_name -> proto.SubscriptionPosition
-	11, // 19: proto.SinkSpec.producerSpec:type_name -> proto.ProducerSpec
-	25, // 20: proto.SinkSpec.schemaProperties:type_name -> proto.SinkSpec.SchemaPropertiesEntry
-	26, // 21: proto.SinkSpec.consumerProperties:type_name -> proto.SinkSpec.ConsumerPropertiesEntry
-	9,  // 22: proto.FunctionMetaData.functionDetails:type_name -> proto.FunctionDetails
-	15, // 23: proto.FunctionMetaData.packageLocation:type_name -> proto.PackageLocationMetaData
-	27, // 24: proto.FunctionMetaData.instanceStates:type_name -> proto.FunctionMetaData.InstanceStatesEntry
-	17, // 25: proto.FunctionMetaData.functionAuthSpec:type_name -> proto.FunctionAuthenticationSpec
-	16, // 26: proto.Instance.functionMetaData:type_name -> proto.FunctionMetaData
-	18, // 27: proto.Assignment.instance:type_name -> proto.Instance
-	10, // 28: proto.SourceSpec.InputSpecsEntry.value:type_name -> proto.ConsumerSpec
-	3,  // 29: proto.FunctionMetaData.InstanceStatesEntry.value:type_name -> proto.FunctionState
-	30, // [30:30] is the sub-list for method output_type
-	30, // [30:30] is the sub-list for method input_type
-	30, // [30:30] is the sub-list for extension type_name
-	30, // [30:30] is the sub-list for extension extendee
-	0,  // [0:30] is the sub-list for field type_name
+	21, // 8: proto.ConsumerSpec.receiverQueueSize:type_name -> proto.ConsumerSpec.ReceiverQueueSize
+	22, // 9: proto.ConsumerSpec.schemaProperties:type_name -> proto.ConsumerSpec.SchemaPropertiesEntry
+	23, // 10: proto.ConsumerSpec.consumerProperties:type_name -> proto.ConsumerSpec.ConsumerPropertiesEntry
+	13, // 11: proto.ConsumerSpec.cryptoSpec:type_name -> proto.CryptoSpec
+	13, // 12: proto.ProducerSpec.cryptoSpec:type_name -> proto.CryptoSpec
+	3,  // 13: proto.ProducerSpec.compressionType:type_name -> proto.CompressionType
+	7,  // 14: proto.CryptoSpec.producerCryptoFailureAction:type_name -> proto.CryptoSpec.FailureAction
+	7,  // 15: proto.CryptoSpec.consumerCryptoFailureAction:type_name -> proto.CryptoSpec.FailureAction
+	1,  // 16: proto.SourceSpec.subscriptionType:type_name -> proto.SubscriptionType
+	24, // 17: proto.SourceSpec.topicsToSerDeClassName:type_name -> proto.SourceSpec.TopicsToSerDeClassNameEntry
+	25, // 18: proto.SourceSpec.inputSpecs:type_name -> proto.SourceSpec.InputSpecsEntry
+	2,  // 19: proto.SourceSpec.subscriptionPosition:type_name -> proto.SubscriptionPosition
+	12, // 20: proto.SinkSpec.producerSpec:type_name -> proto.ProducerSpec
+	26, // 21: proto.SinkSpec.schemaProperties:type_name -> proto.SinkSpec.SchemaPropertiesEntry
+	27, // 22: proto.SinkSpec.consumerProperties:type_name -> proto.SinkSpec.ConsumerPropertiesEntry
+	10, // 23: proto.FunctionMetaData.functionDetails:type_name -> proto.FunctionDetails
+	16, // 24: proto.FunctionMetaData.packageLocation:type_name -> proto.PackageLocationMetaData
+	28, // 25: proto.FunctionMetaData.instanceStates:type_name -> proto.FunctionMetaData.InstanceStatesEntry
+	18, // 26: proto.FunctionMetaData.functionAuthSpec:type_name -> proto.FunctionAuthenticationSpec
+	16, // 27: proto.FunctionMetaData.transformFunctionPackageLocation:type_name -> proto.PackageLocationMetaData
+	17, // 28: proto.Instance.functionMetaData:type_name -> proto.FunctionMetaData
+	19, // 29: proto.Assignment.instance:type_name -> proto.Instance
+	11, // 30: proto.SourceSpec.InputSpecsEntry.value:type_name -> proto.ConsumerSpec
+	4,  // 31: proto.FunctionMetaData.InstanceStatesEntry.value:type_name -> proto.FunctionState
+	32, // [32:32] is the sub-list for method output_type
+	32, // [32:32] is the sub-list for method input_type
+	32, // [32:32] is the sub-list for extension type_name
+	32, // [32:32] is the sub-list for extension extendee
+	0,  // [0:32] is the sub-list for field type_name
 }
 
 func init() { file_Function_proto_init() }
@@ -2254,7 +2354,7 @@ func file_Function_proto_init() {
 		File: protoimpl.DescBuilder{
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: file_Function_proto_rawDesc,
-			NumEnums:      7,
+			NumEnums:      8,
 			NumMessages:   21,
 			NumExtensions: 0,
 			NumServices:   0,
diff --git a/pulsar-function-go/pf/instance.go b/pulsar-function-go/pf/instance.go
index 5e05c070e65f9..5d17cfe0c333a 100644
--- a/pulsar-function-go/pf/instance.go
+++ b/pulsar-function-go/pf/instance.go
@@ -225,7 +225,19 @@ func (gi *goInstance) getProducer(topicName string) (pulsar.Producer, error) {
 
 	batchBuilderType := pulsar.DefaultBatchBuilder
 
+	compressionType := pulsar.LZ4
 	if gi.context.instanceConf.funcDetails.Sink.ProducerSpec != nil {
+		switch gi.context.instanceConf.funcDetails.Sink.ProducerSpec.CompressionType {
+		case pb.CompressionType_NONE:
+			compressionType = pulsar.NoCompression
+		case pb.CompressionType_ZLIB:
+			compressionType = pulsar.ZLib
+		case pb.CompressionType_ZSTD:
+			compressionType = pulsar.ZSTD
+		default:
+			compressionType = pulsar.LZ4 // go doesn't support SNAPPY yet
+		}
+
 		batchBuilder := gi.context.instanceConf.funcDetails.Sink.ProducerSpec.BatchBuilder
 		if batchBuilder != "" {
 			if batchBuilder == "KEY_BASED" {
@@ -237,7 +249,7 @@ func (gi *goInstance) getProducer(topicName string) (pulsar.Producer, error) {
 	producer, err := gi.client.CreateProducer(pulsar.ProducerOptions{
 		Topic:                   topicName,
 		Properties:              properties,
-		CompressionType:         pulsar.LZ4,
+		CompressionType:         compressionType,
 		BatchingMaxPublishDelay: time.Millisecond * 10,
 		BatcherBuilderType:      batchBuilderType,
 		SendTimeout:             0,
diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java
index 0dbfa0945caa7..e2ad9e4c989d1 100644
--- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java
+++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java
@@ -904,7 +904,9 @@ private void setupOutput(ContextImpl contextImpl) throws Exception {
                             .maxPendingMessagesAcrossPartitions(conf.getMaxPendingMessagesAcrossPartitions())
                             .batchBuilder(conf.getBatchBuilder())
                             .useThreadLocalProducers(conf.getUseThreadLocalProducers())
-                            .cryptoConfig(CryptoUtils.convertFromSpec(conf.getCryptoSpec()));
+                            .cryptoConfig(CryptoUtils.convertFromSpec(conf.getCryptoSpec()))
+                            .compressionType(FunctionCommon.convertFromFunctionDetailsCompressionType(
+                                    conf.getCompressionType()));
                     pulsarSinkConfig.setProducerConfig(builder.build());
                 }
 
diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java
index 41ec4d99e71b6..8add0a78c5fff 100644
--- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java
+++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java
@@ -108,7 +108,6 @@ public Producer createProducer(PulsarClient client, String topic, String prod
                     .blockIfQueueFull(true)
                     .enableBatching(true)
                     .batchingMaxPublishDelay(10, TimeUnit.MILLISECONDS)
-                    .compressionType(CompressionType.LZ4)
                     .hashingScheme(HashingScheme.Murmur3_32Hash) //
                     .messageRoutingMode(MessageRoutingMode.CustomPartition)
                     .messageRouter(FunctionResultRouter.of())
@@ -121,6 +120,11 @@ public Producer createProducer(PulsarClient client, String topic, String prod
             }
             if (pulsarSinkConfig.getProducerConfig() != null) {
                 ProducerConfig producerConfig = pulsarSinkConfig.getProducerConfig();
+                if (producerConfig.getCompressionType() != null) {
+                    builder.compressionType(producerConfig.getCompressionType());
+                } else {
+                    builder.compressionType(CompressionType.LZ4);
+                }
                 if (producerConfig.getMaxPendingMessages() != 0) {
                     builder.maxPendingMessages(producerConfig.getMaxPendingMessages());
                 }
diff --git a/pulsar-functions/instance/src/main/python/Function_pb2.py b/pulsar-functions/instance/src/main/python/Function_pb2.py
index eebfe8589d5db..118a6a1cd8967 100644
--- a/pulsar-functions/instance/src/main/python/Function_pb2.py
+++ b/pulsar-functions/instance/src/main/python/Function_pb2.py
@@ -39,7 +39,7 @@
   syntax='proto3',
   serialized_options=b'\n!org.apache.pulsar.functions.protoB\010Function',
   create_key=_descriptor._internal_create_key,
-  serialized_pb=b'\n\x0e\x46unction.proto\x12\x05proto\"3\n\tResources\x12\x0b\n\x03\x63pu\x18\x01 \x01(\x01\x12\x0b\n\x03ram\x18\x02 \x01(\x03\x12\x0c\n\x04\x64isk\x18\x03 \x01(\x03\"B\n\x0cRetryDetails\x12\x19\n\x11maxMessageRetries\x18\x01 \x01(\x05\x12\x17\n\x0f\x64\x65\x61\x64LetterTopic\x18\x02 \x01(\t\"\xa6\x06\n\x0f\x46unctionDetails\x12\x0e\n\x06tenant\x18\x01 \x01(\t\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\x11\n\tclassName\x18\x04 \x01(\t\x12\x10\n\x08logTopic\x18\x05 \x01(\t\x12\x39\n\x14processingGuarantees\x18\x06 \x01(\x0e\x32\x1b.proto.ProcessingGuarantees\x12\x12\n\nuserConfig\x18\x07 \x01(\t\x12\x12\n\nsecretsMap\x18\x10 \x01(\t\x12/\n\x07runtime\x18\x08 \x01(\x0e\x32\x1e.proto.FunctionDetails.Runtime\x12\x13\n\x07\x61utoAck\x18\t \x01(\x08\x42\x02\x18\x01\x12\x13\n\x0bparallelism\x18\n \x01(\x05\x12!\n\x06source\x18\x0b \x01(\x0b\x32\x11.proto.SourceSpec\x12\x1d\n\x04sink\x18\x0c \x01(\x0b\x32\x0f.proto.SinkSpec\x12#\n\tresources\x18\r \x01(\x0b\x32\x10.proto.Resources\x12\x12\n\npackageUrl\x18\x0e \x01(\t\x12)\n\x0cretryDetails\x18\x0f \x01(\x0b\x32\x13.proto.RetryDetails\x12\x14\n\x0cruntimeFlags\x18\x11 \x01(\t\x12;\n\rcomponentType\x18\x12 \x01(\x0e\x32$.proto.FunctionDetails.ComponentType\x12\x1c\n\x14\x63ustomRuntimeOptions\x18\x13 \x01(\t\x12\x0f\n\x07\x62uiltin\x18\x14 \x01(\t\x12\x16\n\x0eretainOrdering\x18\x15 \x01(\x08\x12\x19\n\x11retainKeyOrdering\x18\x16 \x01(\x08\x12\x39\n\x14subscriptionPosition\x18\x17 \x01(\x0e\x32\x1b.proto.SubscriptionPosition\"\'\n\x07Runtime\x12\x08\n\x04JAVA\x10\x00\x12\n\n\x06PYTHON\x10\x01\x12\x06\n\x02GO\x10\x03\"@\n\rComponentType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x0c\n\x08\x46UNCTION\x10\x01\x12\n\n\x06SOURCE\x10\x02\x12\x08\n\x04SINK\x10\x03\"\xf7\x03\n\x0c\x43onsumerSpec\x12\x12\n\nschemaType\x18\x01 \x01(\t\x12\x16\n\x0eserdeClassName\x18\x02 \x01(\t\x12\x16\n\x0eisRegexPattern\x18\x03 \x01(\x08\x12@\n\x11receiverQueueSize\x18\x04 \x01(\x0b\x32%.proto.ConsumerSpec.ReceiverQueueSize\x12\x43\n\x10schemaProperties\x18\x05 \x03(\x0b\x32).proto.ConsumerSpec.SchemaPropertiesEntry\x12G\n\x12\x63onsumerProperties\x18\x06 \x03(\x0b\x32+.proto.ConsumerSpec.ConsumerPropertiesEntry\x12%\n\ncryptoSpec\x18\x07 \x01(\x0b\x32\x11.proto.CryptoSpec\x12\x14\n\x0cpoolMessages\x18\x08 \x01(\x08\x1a\"\n\x11ReceiverQueueSize\x12\r\n\x05value\x18\x01 \x01(\x05\x1a\x37\n\x15SchemaPropertiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x39\n\x17\x43onsumerPropertiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xb4\x01\n\x0cProducerSpec\x12\x1a\n\x12maxPendingMessages\x18\x01 \x01(\x05\x12*\n\"maxPendingMessagesAcrossPartitions\x18\x02 \x01(\x05\x12\x1f\n\x17useThreadLocalProducers\x18\x03 \x01(\x08\x12%\n\ncryptoSpec\x18\x04 \x01(\x0b\x32\x11.proto.CryptoSpec\x12\x14\n\x0c\x62\x61tchBuilder\x18\x05 \x01(\t\"\xbb\x02\n\nCryptoSpec\x12 \n\x18\x63ryptoKeyReaderClassName\x18\x01 \x01(\t\x12\x1d\n\x15\x63ryptoKeyReaderConfig\x18\x02 \x01(\t\x12!\n\x19producerEncryptionKeyName\x18\x03 \x03(\t\x12\x44\n\x1bproducerCryptoFailureAction\x18\x04 \x01(\x0e\x32\x1f.proto.CryptoSpec.FailureAction\x12\x44\n\x1b\x63onsumerCryptoFailureAction\x18\x05 \x01(\x0e\x32\x1f.proto.CryptoSpec.FailureAction\"=\n\rFailureAction\x12\x08\n\x04\x46\x41IL\x10\x00\x12\x0b\n\x07\x44ISCARD\x10\x01\x12\x0b\n\x07\x43ONSUME\x10\x02\x12\x08\n\x04SEND\x10\n\"\xe2\x04\n\nSourceSpec\x12\x11\n\tclassName\x18\x01 \x01(\t\x12\x0f\n\x07\x63onfigs\x18\x02 \x01(\t\x12\x15\n\rtypeClassName\x18\x05 \x01(\t\x12\x31\n\x10subscriptionType\x18\x03 \x01(\x0e\x32\x17.proto.SubscriptionType\x12Q\n\x16topicsToSerDeClassName\x18\x04 \x03(\x0b\x32-.proto.SourceSpec.TopicsToSerDeClassNameEntryB\x02\x18\x01\x12\x35\n\ninputSpecs\x18\n \x03(\x0b\x32!.proto.SourceSpec.InputSpecsEntry\x12\x11\n\ttimeoutMs\x18\x06 \x01(\x04\x12\x19\n\rtopicsPattern\x18\x07 \x01(\tB\x02\x18\x01\x12\x0f\n\x07\x62uiltin\x18\x08 \x01(\t\x12\x18\n\x10subscriptionName\x18\t \x01(\t\x12\x1b\n\x13\x63leanupSubscription\x18\x0b \x01(\x08\x12\x39\n\x14subscriptionPosition\x18\x0c \x01(\x0e\x32\x1b.proto.SubscriptionPosition\x12$\n\x1cnegativeAckRedeliveryDelayMs\x18\r \x01(\x04\x1a=\n\x1bTopicsToSerDeClassNameEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x46\n\x0fInputSpecsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.proto.ConsumerSpec:\x02\x38\x01\"\xdc\x03\n\x08SinkSpec\x12\x11\n\tclassName\x18\x01 \x01(\t\x12\x0f\n\x07\x63onfigs\x18\x02 \x01(\t\x12\x15\n\rtypeClassName\x18\x05 \x01(\t\x12\r\n\x05topic\x18\x03 \x01(\t\x12)\n\x0cproducerSpec\x18\x0b \x01(\x0b\x32\x13.proto.ProducerSpec\x12\x16\n\x0eserDeClassName\x18\x04 \x01(\t\x12\x0f\n\x07\x62uiltin\x18\x06 \x01(\t\x12\x12\n\nschemaType\x18\x07 \x01(\t\x12$\n\x1c\x66orwardSourceMessageProperty\x18\x08 \x01(\x08\x12?\n\x10schemaProperties\x18\t \x03(\x0b\x32%.proto.SinkSpec.SchemaPropertiesEntry\x12\x43\n\x12\x63onsumerProperties\x18\n \x03(\x0b\x32\'.proto.SinkSpec.ConsumerPropertiesEntry\x1a\x37\n\x15SchemaPropertiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x39\n\x17\x43onsumerPropertiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"H\n\x17PackageLocationMetaData\x12\x13\n\x0bpackagePath\x18\x01 \x01(\t\x12\x18\n\x10originalFileName\x18\x02 \x01(\t\"\xf0\x02\n\x10\x46unctionMetaData\x12/\n\x0f\x66unctionDetails\x18\x01 \x01(\x0b\x32\x16.proto.FunctionDetails\x12\x37\n\x0fpackageLocation\x18\x02 \x01(\x0b\x32\x1e.proto.PackageLocationMetaData\x12\x0f\n\x07version\x18\x03 \x01(\x04\x12\x12\n\ncreateTime\x18\x04 \x01(\x04\x12\x43\n\x0einstanceStates\x18\x05 \x03(\x0b\x32+.proto.FunctionMetaData.InstanceStatesEntry\x12;\n\x10\x66unctionAuthSpec\x18\x06 \x01(\x0b\x32!.proto.FunctionAuthenticationSpec\x1aK\n\x13InstanceStatesEntry\x12\x0b\n\x03key\x18\x01 \x01(\x05\x12#\n\x05value\x18\x02 \x01(\x0e\x32\x14.proto.FunctionState:\x02\x38\x01\"<\n\x1a\x46unctionAuthenticationSpec\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\x10\n\x08provider\x18\x02 \x01(\t\"Q\n\x08Instance\x12\x31\n\x10\x66unctionMetaData\x18\x01 \x01(\x0b\x32\x17.proto.FunctionMetaData\x12\x12\n\ninstanceId\x18\x02 \x01(\x05\"A\n\nAssignment\x12!\n\x08instance\x18\x01 \x01(\x0b\x32\x0f.proto.Instance\x12\x10\n\x08workerId\x18\x02 \x01(\t*[\n\x14ProcessingGuarantees\x12\x10\n\x0c\x41TLEAST_ONCE\x10\x00\x12\x0f\n\x0b\x41TMOST_ONCE\x10\x01\x12\x14\n\x10\x45\x46\x46\x45\x43TIVELY_ONCE\x10\x02\x12\n\n\x06MANUAL\x10\x03*<\n\x10SubscriptionType\x12\n\n\x06SHARED\x10\x00\x12\x0c\n\x08\x46\x41ILOVER\x10\x01\x12\x0e\n\nKEY_SHARED\x10\x02*0\n\x14SubscriptionPosition\x12\n\n\x06LATEST\x10\x00\x12\x0c\n\x08\x45\x41RLIEST\x10\x01*)\n\rFunctionState\x12\x0b\n\x07RUNNING\x10\x00\x12\x0b\n\x07STOPPED\x10\x01\x42-\n!org.apache.pulsar.functions.protoB\x08\x46unctionb\x06proto3'
+  serialized_pb=b'\n\x0e\x46unction.proto\x12\x05proto\"3\n\tResources\x12\x0b\n\x03\x63pu\x18\x01 \x01(\x01\x12\x0b\n\x03ram\x18\x02 \x01(\x03\x12\x0c\n\x04\x64isk\x18\x03 \x01(\x03\"B\n\x0cRetryDetails\x12\x19\n\x11maxMessageRetries\x18\x01 \x01(\x05\x12\x17\n\x0f\x64\x65\x61\x64LetterTopic\x18\x02 \x01(\t\"\xa6\x06\n\x0f\x46unctionDetails\x12\x0e\n\x06tenant\x18\x01 \x01(\t\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\x11\n\tclassName\x18\x04 \x01(\t\x12\x10\n\x08logTopic\x18\x05 \x01(\t\x12\x39\n\x14processingGuarantees\x18\x06 \x01(\x0e\x32\x1b.proto.ProcessingGuarantees\x12\x12\n\nuserConfig\x18\x07 \x01(\t\x12\x12\n\nsecretsMap\x18\x10 \x01(\t\x12/\n\x07runtime\x18\x08 \x01(\x0e\x32\x1e.proto.FunctionDetails.Runtime\x12\x13\n\x07\x61utoAck\x18\t \x01(\x08\x42\x02\x18\x01\x12\x13\n\x0bparallelism\x18\n \x01(\x05\x12!\n\x06source\x18\x0b \x01(\x0b\x32\x11.proto.SourceSpec\x12\x1d\n\x04sink\x18\x0c \x01(\x0b\x32\x0f.proto.SinkSpec\x12#\n\tresources\x18\r \x01(\x0b\x32\x10.proto.Resources\x12\x12\n\npackageUrl\x18\x0e \x01(\t\x12)\n\x0cretryDetails\x18\x0f \x01(\x0b\x32\x13.proto.RetryDetails\x12\x14\n\x0cruntimeFlags\x18\x11 \x01(\t\x12;\n\rcomponentType\x18\x12 \x01(\x0e\x32$.proto.FunctionDetails.ComponentType\x12\x1c\n\x14\x63ustomRuntimeOptions\x18\x13 \x01(\t\x12\x0f\n\x07\x62uiltin\x18\x14 \x01(\t\x12\x16\n\x0eretainOrdering\x18\x15 \x01(\x08\x12\x19\n\x11retainKeyOrdering\x18\x16 \x01(\x08\x12\x39\n\x14subscriptionPosition\x18\x17 \x01(\x0e\x32\x1b.proto.SubscriptionPosition\"\'\n\x07Runtime\x12\x08\n\x04JAVA\x10\x00\x12\n\n\x06PYTHON\x10\x01\x12\x06\n\x02GO\x10\x03\"@\n\rComponentType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x0c\n\x08\x46UNCTION\x10\x01\x12\n\n\x06SOURCE\x10\x02\x12\x08\n\x04SINK\x10\x03\"\xf7\x03\n\x0c\x43onsumerSpec\x12\x12\n\nschemaType\x18\x01 \x01(\t\x12\x16\n\x0eserdeClassName\x18\x02 \x01(\t\x12\x16\n\x0eisRegexPattern\x18\x03 \x01(\x08\x12@\n\x11receiverQueueSize\x18\x04 \x01(\x0b\x32%.proto.ConsumerSpec.ReceiverQueueSize\x12\x43\n\x10schemaProperties\x18\x05 \x03(\x0b\x32).proto.ConsumerSpec.SchemaPropertiesEntry\x12G\n\x12\x63onsumerProperties\x18\x06 \x03(\x0b\x32+.proto.ConsumerSpec.ConsumerPropertiesEntry\x12%\n\ncryptoSpec\x18\x07 \x01(\x0b\x32\x11.proto.CryptoSpec\x12\x14\n\x0cpoolMessages\x18\x08 \x01(\x08\x1a\"\n\x11ReceiverQueueSize\x12\r\n\x05value\x18\x01 \x01(\x05\x1a\x37\n\x15SchemaPropertiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x39\n\x17\x43onsumerPropertiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xe5\x01\n\x0cProducerSpec\x12\x1a\n\x12maxPendingMessages\x18\x01 \x01(\x05\x12*\n\"maxPendingMessagesAcrossPartitions\x18\x02 \x01(\x05\x12\x1f\n\x17useThreadLocalProducers\x18\x03 \x01(\x08\x12%\n\ncryptoSpec\x18\x04 \x01(\x0b\x32\x11.proto.CryptoSpec\x12\x14\n\x0c\x62\x61tchBuilder\x18\x05 \x01(\t\x12/\n\x0f\x63ompressionType\x18\x06 \x01(\x0e\x32\x16.proto.CompressionType\"\xbb\x02\n\nCryptoSpec\x12 \n\x18\x63ryptoKeyReaderClassName\x18\x01 \x01(\t\x12\x1d\n\x15\x63ryptoKeyReaderConfig\x18\x02 \x01(\t\x12!\n\x19producerEncryptionKeyName\x18\x03 \x03(\t\x12\x44\n\x1bproducerCryptoFailureAction\x18\x04 \x01(\x0e\x32\x1f.proto.CryptoSpec.FailureAction\x12\x44\n\x1b\x63onsumerCryptoFailureAction\x18\x05 \x01(\x0e\x32\x1f.proto.CryptoSpec.FailureAction\"=\n\rFailureAction\x12\x08\n\x04\x46\x41IL\x10\x00\x12\x0b\n\x07\x44ISCARD\x10\x01\x12\x0b\n\x07\x43ONSUME\x10\x02\x12\x08\n\x04SEND\x10\n\"\xe2\x04\n\nSourceSpec\x12\x11\n\tclassName\x18\x01 \x01(\t\x12\x0f\n\x07\x63onfigs\x18\x02 \x01(\t\x12\x15\n\rtypeClassName\x18\x05 \x01(\t\x12\x31\n\x10subscriptionType\x18\x03 \x01(\x0e\x32\x17.proto.SubscriptionType\x12Q\n\x16topicsToSerDeClassName\x18\x04 \x03(\x0b\x32-.proto.SourceSpec.TopicsToSerDeClassNameEntryB\x02\x18\x01\x12\x35\n\ninputSpecs\x18\n \x03(\x0b\x32!.proto.SourceSpec.InputSpecsEntry\x12\x11\n\ttimeoutMs\x18\x06 \x01(\x04\x12\x19\n\rtopicsPattern\x18\x07 \x01(\tB\x02\x18\x01\x12\x0f\n\x07\x62uiltin\x18\x08 \x01(\t\x12\x18\n\x10subscriptionName\x18\t \x01(\t\x12\x1b\n\x13\x63leanupSubscription\x18\x0b \x01(\x08\x12\x39\n\x14subscriptionPosition\x18\x0c \x01(\x0e\x32\x1b.proto.SubscriptionPosition\x12$\n\x1cnegativeAckRedeliveryDelayMs\x18\r \x01(\x04\x1a=\n\x1bTopicsToSerDeClassNameEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x46\n\x0fInputSpecsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.proto.ConsumerSpec:\x02\x38\x01\"\xdc\x03\n\x08SinkSpec\x12\x11\n\tclassName\x18\x01 \x01(\t\x12\x0f\n\x07\x63onfigs\x18\x02 \x01(\t\x12\x15\n\rtypeClassName\x18\x05 \x01(\t\x12\r\n\x05topic\x18\x03 \x01(\t\x12)\n\x0cproducerSpec\x18\x0b \x01(\x0b\x32\x13.proto.ProducerSpec\x12\x16\n\x0eserDeClassName\x18\x04 \x01(\t\x12\x0f\n\x07\x62uiltin\x18\x06 \x01(\t\x12\x12\n\nschemaType\x18\x07 \x01(\t\x12$\n\x1c\x66orwardSourceMessageProperty\x18\x08 \x01(\x08\x12?\n\x10schemaProperties\x18\t \x03(\x0b\x32%.proto.SinkSpec.SchemaPropertiesEntry\x12\x43\n\x12\x63onsumerProperties\x18\n \x03(\x0b\x32\'.proto.SinkSpec.ConsumerPropertiesEntry\x1a\x37\n\x15SchemaPropertiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x39\n\x17\x43onsumerPropertiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"H\n\x17PackageLocationMetaData\x12\x13\n\x0bpackagePath\x18\x01 \x01(\t\x12\x18\n\x10originalFileName\x18\x02 \x01(\t\"\xba\x03\n\x10\x46unctionMetaData\x12/\n\x0f\x66unctionDetails\x18\x01 \x01(\x0b\x32\x16.proto.FunctionDetails\x12\x37\n\x0fpackageLocation\x18\x02 \x01(\x0b\x32\x1e.proto.PackageLocationMetaData\x12\x0f\n\x07version\x18\x03 \x01(\x04\x12\x12\n\ncreateTime\x18\x04 \x01(\x04\x12\x43\n\x0einstanceStates\x18\x05 \x03(\x0b\x32+.proto.FunctionMetaData.InstanceStatesEntry\x12;\n\x10\x66unctionAuthSpec\x18\x06 \x01(\x0b\x32!.proto.FunctionAuthenticationSpec\x12H\n transformFunctionPackageLocation\x18\x07 \x01(\x0b\x32\x1e.proto.PackageLocationMetaData\x1aK\n\x13InstanceStatesEntry\x12\x0b\n\x03key\x18\x01 \x01(\x05\x12#\n\x05value\x18\x02 \x01(\x0e\x32\x14.proto.FunctionState:\x02\x38\x01\"<\n\x1a\x46unctionAuthenticationSpec\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\x10\n\x08provider\x18\x02 \x01(\t\"Q\n\x08Instance\x12\x31\n\x10\x66unctionMetaData\x18\x01 \x01(\x0b\x32\x17.proto.FunctionMetaData\x12\x12\n\ninstanceId\x18\x02 \x01(\x05\"A\n\nAssignment\x12!\n\x08instance\x18\x01 \x01(\x0b\x32\x0f.proto.Instance\x12\x10\n\x08workerId\x18\x02 \x01(\t*[\n\x14ProcessingGuarantees\x12\x10\n\x0c\x41TLEAST_ONCE\x10\x00\x12\x0f\n\x0b\x41TMOST_ONCE\x10\x01\x12\x14\n\x10\x45\x46\x46\x45\x43TIVELY_ONCE\x10\x02\x12\n\n\x06MANUAL\x10\x03*<\n\x10SubscriptionType\x12\n\n\x06SHARED\x10\x00\x12\x0c\n\x08\x46\x41ILOVER\x10\x01\x12\x0e\n\nKEY_SHARED\x10\x02*0\n\x14SubscriptionPosition\x12\n\n\x06LATEST\x10\x00\x12\x0c\n\x08\x45\x41RLIEST\x10\x01*D\n\x0f\x43ompressionType\x12\x07\n\x03LZ4\x10\x00\x12\x08\n\x04NONE\x10\x01\x12\x08\n\x04ZLIB\x10\x02\x12\x08\n\x04ZSTD\x10\x03\x12\n\n\x06SNAPPY\x10\x04*)\n\rFunctionState\x12\x0b\n\x07RUNNING\x10\x00\x12\x0b\n\x07STOPPED\x10\x01\x42-\n!org.apache.pulsar.functions.protoB\x08\x46unctionb\x06proto3'
 )
 
 _PROCESSINGGUARANTEES = _descriptor.EnumDescriptor(
@@ -72,8 +72,8 @@
   ],
   containing_type=None,
   serialized_options=None,
-  serialized_start=3711,
-  serialized_end=3802,
+  serialized_start=3834,
+  serialized_end=3925,
 )
 _sym_db.RegisterEnumDescriptor(_PROCESSINGGUARANTEES)
 
@@ -103,8 +103,8 @@
   ],
   containing_type=None,
   serialized_options=None,
-  serialized_start=3804,
-  serialized_end=3864,
+  serialized_start=3927,
+  serialized_end=3987,
 )
 _sym_db.RegisterEnumDescriptor(_SUBSCRIPTIONTYPE)
 
@@ -129,12 +129,53 @@
   ],
   containing_type=None,
   serialized_options=None,
-  serialized_start=3866,
-  serialized_end=3914,
+  serialized_start=3989,
+  serialized_end=4037,
 )
 _sym_db.RegisterEnumDescriptor(_SUBSCRIPTIONPOSITION)
 
 SubscriptionPosition = enum_type_wrapper.EnumTypeWrapper(_SUBSCRIPTIONPOSITION)
+_COMPRESSIONTYPE = _descriptor.EnumDescriptor(
+  name='CompressionType',
+  full_name='proto.CompressionType',
+  filename=None,
+  file=DESCRIPTOR,
+  create_key=_descriptor._internal_create_key,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='LZ4', index=0, number=0,
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
+    _descriptor.EnumValueDescriptor(
+      name='NONE', index=1, number=1,
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
+    _descriptor.EnumValueDescriptor(
+      name='ZLIB', index=2, number=2,
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
+    _descriptor.EnumValueDescriptor(
+      name='ZSTD', index=3, number=3,
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
+    _descriptor.EnumValueDescriptor(
+      name='SNAPPY', index=4, number=4,
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
+  ],
+  containing_type=None,
+  serialized_options=None,
+  serialized_start=4039,
+  serialized_end=4107,
+)
+_sym_db.RegisterEnumDescriptor(_COMPRESSIONTYPE)
+
+CompressionType = enum_type_wrapper.EnumTypeWrapper(_COMPRESSIONTYPE)
 _FUNCTIONSTATE = _descriptor.EnumDescriptor(
   name='FunctionState',
   full_name='proto.FunctionState',
@@ -155,8 +196,8 @@
   ],
   containing_type=None,
   serialized_options=None,
-  serialized_start=3916,
-  serialized_end=3957,
+  serialized_start=4109,
+  serialized_end=4150,
 )
 _sym_db.RegisterEnumDescriptor(_FUNCTIONSTATE)
 
@@ -170,6 +211,11 @@
 KEY_SHARED = 2
 LATEST = 0
 EARLIEST = 1
+LZ4 = 0
+NONE = 1
+ZLIB = 2
+ZSTD = 3
+SNAPPY = 4
 RUNNING = 0
 STOPPED = 1
 
@@ -269,8 +315,8 @@
   ],
   containing_type=None,
   serialized_options=None,
-  serialized_start=1899,
-  serialized_end=1960,
+  serialized_start=1948,
+  serialized_end=2009,
 )
 _sym_db.RegisterEnumDescriptor(_CRYPTOSPEC_FAILUREACTION)
 
@@ -779,6 +825,13 @@
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='compressionType', full_name='proto.ProducerSpec.compressionType', index=5,
+      number=6, type=14, cpp_type=8, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
   ],
   extensions=[
   ],
@@ -792,7 +845,7 @@
   oneofs=[
   ],
   serialized_start=1462,
-  serialized_end=1642,
+  serialized_end=1691,
 )
 
 
@@ -852,8 +905,8 @@
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=1645,
-  serialized_end=1960,
+  serialized_start=1694,
+  serialized_end=2009,
 )
 
 
@@ -891,8 +944,8 @@
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=2440,
-  serialized_end=2501,
+  serialized_start=2489,
+  serialized_end=2550,
 )
 
 _SOURCESPEC_INPUTSPECSENTRY = _descriptor.Descriptor(
@@ -929,8 +982,8 @@
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=2503,
-  serialized_end=2573,
+  serialized_start=2552,
+  serialized_end=2622,
 )
 
 _SOURCESPEC = _descriptor.Descriptor(
@@ -1044,8 +1097,8 @@
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=1963,
-  serialized_end=2573,
+  serialized_start=2012,
+  serialized_end=2622,
 )
 
 
@@ -1222,8 +1275,8 @@
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=2576,
-  serialized_end=3052,
+  serialized_start=2625,
+  serialized_end=3101,
 )
 
 
@@ -1261,8 +1314,8 @@
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=3054,
-  serialized_end=3126,
+  serialized_start=3103,
+  serialized_end=3175,
 )
 
 
@@ -1300,8 +1353,8 @@
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=3422,
-  serialized_end=3497,
+  serialized_start=3545,
+  serialized_end=3620,
 )
 
 _FUNCTIONMETADATA = _descriptor.Descriptor(
@@ -1354,6 +1407,13 @@
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='transformFunctionPackageLocation', full_name='proto.FunctionMetaData.transformFunctionPackageLocation', index=6,
+      number=7, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
   ],
   extensions=[
   ],
@@ -1366,8 +1426,8 @@
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=3129,
-  serialized_end=3497,
+  serialized_start=3178,
+  serialized_end=3620,
 )
 
 
@@ -1405,8 +1465,8 @@
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=3499,
-  serialized_end=3559,
+  serialized_start=3622,
+  serialized_end=3682,
 )
 
 
@@ -1444,8 +1504,8 @@
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=3561,
-  serialized_end=3642,
+  serialized_start=3684,
+  serialized_end=3765,
 )
 
 
@@ -1483,8 +1543,8 @@
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=3644,
-  serialized_end=3709,
+  serialized_start=3767,
+  serialized_end=3832,
 )
 
 _FUNCTIONDETAILS.fields_by_name['processingGuarantees'].enum_type = _PROCESSINGGUARANTEES
@@ -1505,6 +1565,7 @@
 _CONSUMERSPEC.fields_by_name['consumerProperties'].message_type = _CONSUMERSPEC_CONSUMERPROPERTIESENTRY
 _CONSUMERSPEC.fields_by_name['cryptoSpec'].message_type = _CRYPTOSPEC
 _PRODUCERSPEC.fields_by_name['cryptoSpec'].message_type = _CRYPTOSPEC
+_PRODUCERSPEC.fields_by_name['compressionType'].enum_type = _COMPRESSIONTYPE
 _CRYPTOSPEC.fields_by_name['producerCryptoFailureAction'].enum_type = _CRYPTOSPEC_FAILUREACTION
 _CRYPTOSPEC.fields_by_name['consumerCryptoFailureAction'].enum_type = _CRYPTOSPEC_FAILUREACTION
 _CRYPTOSPEC_FAILUREACTION.containing_type = _CRYPTOSPEC
@@ -1526,6 +1587,7 @@
 _FUNCTIONMETADATA.fields_by_name['packageLocation'].message_type = _PACKAGELOCATIONMETADATA
 _FUNCTIONMETADATA.fields_by_name['instanceStates'].message_type = _FUNCTIONMETADATA_INSTANCESTATESENTRY
 _FUNCTIONMETADATA.fields_by_name['functionAuthSpec'].message_type = _FUNCTIONAUTHENTICATIONSPEC
+_FUNCTIONMETADATA.fields_by_name['transformFunctionPackageLocation'].message_type = _PACKAGELOCATIONMETADATA
 _INSTANCE.fields_by_name['functionMetaData'].message_type = _FUNCTIONMETADATA
 _ASSIGNMENT.fields_by_name['instance'].message_type = _INSTANCE
 DESCRIPTOR.message_types_by_name['Resources'] = _RESOURCES
@@ -1544,6 +1606,7 @@
 DESCRIPTOR.enum_types_by_name['ProcessingGuarantees'] = _PROCESSINGGUARANTEES
 DESCRIPTOR.enum_types_by_name['SubscriptionType'] = _SUBSCRIPTIONTYPE
 DESCRIPTOR.enum_types_by_name['SubscriptionPosition'] = _SUBSCRIPTIONPOSITION
+DESCRIPTOR.enum_types_by_name['CompressionType'] = _COMPRESSIONTYPE
 DESCRIPTOR.enum_types_by_name['FunctionState'] = _FUNCTIONSTATE
 _sym_db.RegisterFileDescriptor(DESCRIPTOR)
 
diff --git a/pulsar-functions/instance/src/main/python/python_instance.py b/pulsar-functions/instance/src/main/python/python_instance.py
index ac723232f4839..c679288789536 100755
--- a/pulsar-functions/instance/src/main/python/python_instance.py
+++ b/pulsar-functions/instance/src/main/python/python_instance.py
@@ -352,6 +352,17 @@ def setup_producer(self):
       if crypto_key_reader is not None:
         encryption_key = self.instance_config.function_details.sink.producerSpec.cryptoSpec.producerEncryptionKeyName[0]
 
+      compression_type = pulsar.CompressionType.LZ4
+      if self.instance_config.function_details.sink.producerSpec.compressionType is not None:
+        if self.instance_config.function_details.sink.producerSpec.compressionType == Function_pb2.CompressionType.Value("NONE"):
+          compression_type = pulsar.CompressionType.NONE
+        elif self.instance_config.function_details.sink.producerSpec.compressionType == Function_pb2.CompressionType.Value("ZLIB"):
+          compression_type = pulsar.CompressionType.ZLib
+        elif self.instance_config.function_details.sink.producerSpec.compressionType == Function_pb2.CompressionType.Value("ZSTD"):
+          compression_type = pulsar.CompressionType.ZSTD
+        elif self.instance_config.function_details.sink.producerSpec.compressionType == Function_pb2.CompressionType.Value("SNAPPY"):
+          compression_type = pulsar.CompressionType.SNAPPY
+
       self.producer = self.pulsar_client.create_producer(
         str(self.instance_config.function_details.sink.topic),
         schema=self.output_schema,
@@ -359,7 +370,7 @@ def setup_producer(self):
         batching_enabled=True,
         batching_type=batch_type,
         batching_max_publish_delay_ms=10,
-        compression_type=pulsar.CompressionType.LZ4,
+        compression_type=compression_type,
         # set send timeout to be infinity to prevent potential deadlock with consumer
         # that might happen when consumer is blocked due to unacked messages
         send_timeout_millis=0,
diff --git a/pulsar-functions/proto/src/main/proto/Function.proto b/pulsar-functions/proto/src/main/proto/Function.proto
index 101d45bc59cd7..de3f03a39008c 100644
--- a/pulsar-functions/proto/src/main/proto/Function.proto
+++ b/pulsar-functions/proto/src/main/proto/Function.proto
@@ -41,6 +41,14 @@ enum SubscriptionPosition {
     EARLIEST = 1;
 }
 
+enum CompressionType {
+    LZ4 = 0;
+    NONE = 1;
+    ZLIB = 2;
+    ZSTD = 3;
+    SNAPPY = 4;
+}
+
 message Resources {
     double cpu = 1;
     int64 ram = 2;
@@ -112,6 +120,7 @@ message ProducerSpec {
     bool useThreadLocalProducers = 3;
     CryptoSpec cryptoSpec = 4;
     string batchBuilder = 5;
+    CompressionType compressionType = 6;
 }
 
 message CryptoSpec {
diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java
index bda99a39478a3..d3ce9d93a2d36 100644
--- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java
+++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java
@@ -44,6 +44,7 @@
 import lombok.extern.slf4j.Slf4j;
 import net.jodah.typetools.TypeResolver;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.pulsar.client.api.CompressionType;
 import org.apache.pulsar.client.api.MessageId;
 import org.apache.pulsar.client.api.SubscriptionInitialPosition;
 import org.apache.pulsar.client.impl.MessageIdImpl;
@@ -568,4 +569,42 @@ public static SubscriptionInitialPosition convertFromFunctionDetailsSubscription
             return SubscriptionInitialPosition.Latest;
         }
     }
+
+    public static CompressionType convertFromFunctionDetailsCompressionType(
+            org.apache.pulsar.functions.proto.Function.CompressionType compressionType) {
+        if (compressionType == null) {
+            return CompressionType.LZ4;
+        }
+        switch (compressionType) {
+            case NONE:
+                return CompressionType.NONE;
+            case ZLIB:
+                return CompressionType.ZLIB;
+            case ZSTD:
+                return CompressionType.ZSTD;
+            case SNAPPY:
+                return CompressionType.SNAPPY;
+            default:
+                return CompressionType.LZ4;
+        }
+    }
+
+    public static org.apache.pulsar.functions.proto.Function.CompressionType convertFromCompressionType(
+       CompressionType compressionType) {
+        if (compressionType == null) {
+            return org.apache.pulsar.functions.proto.Function.CompressionType.LZ4;
+        }
+        switch (compressionType) {
+            case NONE:
+                return org.apache.pulsar.functions.proto.Function.CompressionType.NONE;
+            case ZLIB:
+                return org.apache.pulsar.functions.proto.Function.CompressionType.ZLIB;
+            case ZSTD:
+                return org.apache.pulsar.functions.proto.Function.CompressionType.ZSTD;
+            case SNAPPY:
+                return org.apache.pulsar.functions.proto.Function.CompressionType.SNAPPY;
+            default:
+                return org.apache.pulsar.functions.proto.Function.CompressionType.LZ4;
+        }
+    }
 }
diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java
index d02fe5f788b5a..d20d50df1bdd1 100644
--- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java
+++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java
@@ -24,6 +24,8 @@
 import static org.apache.commons.lang3.StringUtils.isEmpty;
 import static org.apache.pulsar.common.functions.Utils.BUILTIN;
 import static org.apache.pulsar.common.util.ClassLoaderUtils.loadJar;
+import static org.apache.pulsar.functions.utils.FunctionCommon.convertFromCompressionType;
+import static org.apache.pulsar.functions.utils.FunctionCommon.convertFromFunctionDetailsCompressionType;
 import static org.apache.pulsar.functions.utils.FunctionCommon.convertFromFunctionDetailsSubscriptionPosition;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -271,6 +273,11 @@ public static FunctionDetails convert(FunctionConfig functionConfig, ExtractedFu
             if (producerConf.getBatchBuilder() != null) {
                 pbldr.setBatchBuilder(producerConf.getBatchBuilder());
             }
+            if (producerConf.getCompressionType() != null) {
+                pbldr.setCompressionType(convertFromCompressionType(producerConf.getCompressionType()));
+            } else {
+                pbldr.setCompressionType(Function.CompressionType.LZ4);
+            }
             sinkSpecBuilder.setProducerSpec(pbldr.build());
         }
         functionDetailsBuilder.setSink(sinkSpecBuilder);
@@ -471,6 +478,7 @@ public static FunctionConfig convertFromDetails(FunctionDetails functionDetails)
                 producerConfig.setBatchBuilder(spec.getBatchBuilder());
             }
             producerConfig.setUseThreadLocalProducers(spec.getUseThreadLocalProducers());
+            producerConfig.setCompressionType(convertFromFunctionDetailsCompressionType(spec.getCompressionType()));
             functionConfig.setProducerConfig(producerConfig);
         }
         if (!isEmpty(functionDetails.getLogTopic())) {
diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SourceConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SourceConfigUtils.java
index 24d4259a74a29..ec0c5c444ccba 100644
--- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SourceConfigUtils.java
+++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SourceConfigUtils.java
@@ -19,6 +19,8 @@
 package org.apache.pulsar.functions.utils;
 
 import static org.apache.commons.lang3.StringUtils.isEmpty;
+import static org.apache.pulsar.functions.utils.FunctionCommon.convertFromCompressionType;
+import static org.apache.pulsar.functions.utils.FunctionCommon.convertFromFunctionDetailsCompressionType;
 import static org.apache.pulsar.functions.utils.FunctionCommon.convertProcessingGuarantee;
 import static org.apache.pulsar.functions.utils.FunctionCommon.getSourceType;
 import com.fasterxml.jackson.core.type.TypeReference;
@@ -164,6 +166,11 @@ public static FunctionDetails convert(SourceConfig sourceConfig, ExtractedSource
             if (conf.getBatchBuilder() != null) {
                 pbldr.setBatchBuilder(conf.getBatchBuilder());
             }
+            if (conf.getCompressionType() != null) {
+                pbldr.setCompressionType(convertFromCompressionType(conf.getCompressionType()));
+            } else {
+                pbldr.setCompressionType(Function.CompressionType.LZ4);
+            }
             sinkSpecBuilder.setProducerSpec(pbldr.build());
         }
 
@@ -264,6 +271,7 @@ public static SourceConfig convertFromDetails(FunctionDetails functionDetails) {
                 producerConfig.setBatchBuilder(spec.getBatchBuilder());
             }
             producerConfig.setUseThreadLocalProducers(spec.getUseThreadLocalProducers());
+            producerConfig.setCompressionType(convertFromFunctionDetailsCompressionType(spec.getCompressionType()));
             sourceConfig.setProducerConfig(producerConfig);
         }
         if (functionDetails.hasResources()) {
diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java
index 4ba5138f7c6b9..8b4470d8c76c7 100644
--- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java
+++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java
@@ -20,6 +20,7 @@
 
 import com.google.gson.Gson;
 
+import org.apache.pulsar.client.api.CompressionType;
 import org.apache.pulsar.client.api.SubscriptionInitialPosition;
 import com.google.protobuf.InvalidProtocolBufferException;
 import com.google.protobuf.util.JsonFormat;
@@ -96,6 +97,7 @@ public void testConvertBackFidelity() {
         producerConfig.setMaxPendingMessagesAcrossPartitions(1000);
         producerConfig.setUseThreadLocalProducers(true);
         producerConfig.setBatchBuilder("DEFAULT");
+        producerConfig.setCompressionType(CompressionType.ZLIB);
         functionConfig.setProducerConfig(producerConfig);
         Function.FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig, (ClassLoader) null);
         FunctionConfig convertedConfig = FunctionConfigUtils.convertFromDetails(functionDetails);
@@ -137,6 +139,7 @@ public void testConvertWindow() {
         producerConfig.setMaxPendingMessagesAcrossPartitions(1000);
         producerConfig.setUseThreadLocalProducers(true);
         producerConfig.setBatchBuilder("KEY_BASED");
+        producerConfig.setCompressionType(CompressionType.SNAPPY);
         functionConfig.setProducerConfig(producerConfig);
         Function.FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig, (ClassLoader) null);
         FunctionConfig convertedConfig = FunctionConfigUtils.convertFromDetails(functionDetails);
diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SourceConfigUtilsTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SourceConfigUtilsTest.java
index 63485d7993fad..49313dbf02c62 100644
--- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SourceConfigUtilsTest.java
+++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SourceConfigUtilsTest.java
@@ -22,6 +22,7 @@
 import lombok.Data;
 import lombok.NoArgsConstructor;
 import lombok.experimental.Accessors;
+import org.apache.pulsar.client.api.CompressionType;
 import org.apache.pulsar.common.functions.FunctionConfig;
 import org.apache.pulsar.common.functions.ProducerConfig;
 import org.apache.pulsar.common.functions.Resources;
@@ -370,6 +371,7 @@ private SourceConfig createSourceConfig() {
         producerConfig.setMaxPendingMessagesAcrossPartitions(1000);
         producerConfig.setUseThreadLocalProducers(true);
         producerConfig.setBatchBuilder("DEFAULT");
+        producerConfig.setCompressionType(CompressionType.ZSTD);
         sourceConfig.setProducerConfig(producerConfig);
 
         sourceConfig.setConfigs(configs);

From 4ab4463fee5f773cc29104598c707ed25f827f37 Mon Sep 17 00:00:00 2001
From: Kai Wang 
Date: Sat, 18 Mar 2023 17:44:45 +0800
Subject: [PATCH 123/174] [improve][broker] PIP-192: Add metrics for unload
 operation (#19749)

PIP: https://github.com/apache/pulsar/issues/16691

### Motivation
Raising a PR to implement https://github.com/apache/pulsar/issues/16691.

We need to support metrics for unload/transfer operations in Load Manager Extension.

### Modifications
In this PR:
* Change the `findBundlesForUnloading` method return type from `UnloadDecision` to `Set`.
* The `UnloadDecision` no longer contains all unload objects. Each unload object has its own reason.
* Add units test to verify the unload counter.
---
 .../extensions/ExtensibleLoadManagerImpl.java |  18 +-
 .../extensions/manager/UnloadManager.java     |  25 +-
 .../extensions/models/UnloadCounter.java      |  72 +++--
 .../extensions/models/UnloadDecision.java     |  62 +----
 .../AntiAffinityGroupPolicyHelper.java        |   3 -
 .../scheduler/NamespaceUnloadStrategy.java    |   7 +-
 .../extensions/scheduler/TransferShedder.java |  93 ++++---
 .../extensions/scheduler/UnloadScheduler.java | 129 ++++++---
 .../ExtensibleLoadManagerImplTest.java        |  38 +--
 .../extensions/manager/UnloadManagerTest.java |  51 +++-
 .../scheduler/TransferShedderTest.java        | 260 ++++++++----------
 .../scheduler/UnloadSchedulerTest.java        |  51 +++-
 12 files changed, 440 insertions(+), 369 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
index 716be3718bf19..486c32153589f 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
@@ -20,6 +20,8 @@
 
 import static org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.Role.Follower;
 import static org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.Role.Leader;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Success;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Admin;
 import com.google.common.annotations.VisibleForTesting;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -204,8 +206,8 @@ public void start() throws PulsarServerException {
                 });
         this.serviceUnitStateChannel = new ServiceUnitStateChannelImpl(pulsar);
         this.brokerRegistry.start();
-        this.unloadManager = new UnloadManager();
         this.splitManager = new SplitManager(splitCounter);
+        this.unloadManager = new UnloadManager(unloadCounter);
         this.serviceUnitStateChannel.listen(unloadManager);
         this.serviceUnitStateChannel.listen(splitManager);
         this.leaderElectionService.start();
@@ -265,7 +267,8 @@ public void start() throws PulsarServerException {
                         interval, TimeUnit.MILLISECONDS);
 
         this.unloadScheduler = new UnloadScheduler(
-                pulsar.getLoadManagerExecutor(), unloadManager, context, serviceUnitStateChannel);
+                pulsar, pulsar.getLoadManagerExecutor(), unloadManager,
+                context, serviceUnitStateChannel, antiAffinityGroupPolicyHelper, unloadCounter, unloadMetrics);
         this.unloadScheduler.start();
         this.splitScheduler = new SplitScheduler(
                 pulsar, serviceUnitStateChannel, splitManager, splitCounter, splitMetrics, context);
@@ -401,16 +404,21 @@ public CompletableFuture unloadNamespaceBundleAsync(ServiceUnitId bundle,
                         log.warn(msg);
                         throw new IllegalArgumentException(msg);
                     }
-                    return unloadAsync(new Unload(sourceBroker, bundle.toString(), destinationBroker),
+                    Unload unload = new Unload(sourceBroker, bundle.toString(), destinationBroker);
+                    UnloadDecision unloadDecision =
+                            new UnloadDecision(unload, Success, Admin);
+                    return unloadAsync(unloadDecision,
                             conf.getNamespaceBundleUnloadingTimeoutMs(), TimeUnit.MILLISECONDS);
                 });
     }
 
-    private CompletableFuture unloadAsync(Unload unload,
+    private CompletableFuture unloadAsync(UnloadDecision unloadDecision,
                                                long timeout,
                                                TimeUnit timeoutUnit) {
+        Unload unload = unloadDecision.getUnload();
         CompletableFuture future = serviceUnitStateChannel.publishUnloadEventAsync(unload);
-        return unloadManager.waitAsync(future, unload.serviceUnit(), timeout, timeoutUnit);
+        return unloadManager.waitAsync(future, unload.serviceUnit(), unloadDecision, timeout, timeoutUnit)
+                .thenRun(() -> unloadCounter.updateUnloadBrokerCount(1));
     }
 
     @Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManager.java
index ead6384daba8d..2dde0c4708e41 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManager.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManager.java
@@ -18,6 +18,8 @@
  */
 package org.apache.pulsar.broker.loadbalance.extensions.manager;
 
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Failure;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Unknown;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
@@ -25,6 +27,8 @@
 import lombok.extern.slf4j.Slf4j;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData;
+import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter;
+import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision;
 
 /**
  * Unload manager.
@@ -32,9 +36,11 @@
 @Slf4j
 public class UnloadManager implements StateChangeListener {
 
+    private final UnloadCounter counter;
     private final Map> inFlightUnloadRequest;
 
-    public UnloadManager() {
+    public UnloadManager(UnloadCounter counter) {
+        this.counter = counter;
         this.inFlightUnloadRequest = new ConcurrentHashMap<>();
     }
 
@@ -43,14 +49,8 @@ private void complete(String serviceUnit, Throwable ex) {
             if (!future.isDone()) {
                 if (ex != null) {
                     future.completeExceptionally(ex);
-                    if (log.isDebugEnabled()) {
-                        log.debug("Complete exceptionally unload bundle: {}", serviceUnit, ex);
-                    }
                 } else {
                     future.complete(null);
-                    if (log.isDebugEnabled()) {
-                        log.debug("Complete unload bundle: {}", serviceUnit);
-                    }
                 }
             }
             return null;
@@ -59,6 +59,7 @@ private void complete(String serviceUnit, Throwable ex) {
 
     public CompletableFuture waitAsync(CompletableFuture eventPubFuture,
                                              String bundle,
+                                             UnloadDecision decision,
                                              long timeout,
                                              TimeUnit timeoutUnit) {
 
@@ -74,7 +75,15 @@ public CompletableFuture waitAsync(CompletableFuture eventPubFuture,
                 }
             });
             return future;
-        }));
+        })).whenComplete((__, ex) -> {
+            if (ex != null) {
+                counter.update(Failure, Unknown);
+                log.warn("Failed to unload bundle: {}", bundle, ex);
+                return;
+            }
+            log.info("Complete unload bundle: {}", bundle);
+            counter.update(decision);
+        });
     }
 
     @Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadCounter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadCounter.java
index e2a51b1248967..37483b58b53e1 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadCounter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadCounter.java
@@ -21,6 +21,7 @@
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Failure;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Skip;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Success;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Admin;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Balanced;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.CoolDown;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBrokers;
@@ -30,11 +31,13 @@
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Overloaded;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Underloaded;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Unknown;
+import com.google.common.annotations.VisibleForTesting;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import org.apache.commons.lang3.mutable.MutableLong;
+import java.util.concurrent.atomic.AtomicLong;
+import lombok.Getter;
 import org.apache.pulsar.common.stats.Metrics;
 
 /**
@@ -45,36 +48,63 @@ public class UnloadCounter {
     long unloadBrokerCount = 0;
     long unloadBundleCount = 0;
 
-    final Map> breakdownCounters;
+    @Getter
+    @VisibleForTesting
+    final Map> breakdownCounters;
 
+    @Getter
+    @VisibleForTesting
     double loadAvg;
+    @Getter
+    @VisibleForTesting
     double loadStd;
 
+    private volatile long updatedAt = 0;
+
     public UnloadCounter() {
         breakdownCounters = Map.of(
                 Success, Map.of(
-                        Overloaded, new MutableLong(),
-                        Underloaded, new MutableLong()),
+                        Overloaded, new AtomicLong(),
+                        Underloaded, new AtomicLong(),
+                        Admin, new AtomicLong()),
                 Skip, Map.of(
-                        Balanced, new MutableLong(),
-                        NoBundles, new MutableLong(),
-                        CoolDown, new MutableLong(),
-                        OutDatedData, new MutableLong(),
-                        NoLoadData, new MutableLong(),
-                        NoBrokers, new MutableLong(),
-                        Unknown, new MutableLong()),
+                        Balanced, new AtomicLong(),
+                        NoBundles, new AtomicLong(),
+                        CoolDown, new AtomicLong(),
+                        OutDatedData, new AtomicLong(),
+                        NoLoadData, new AtomicLong(),
+                        NoBrokers, new AtomicLong(),
+                        Unknown, new AtomicLong()),
                 Failure, Map.of(
-                        Unknown, new MutableLong())
+                        Unknown, new AtomicLong())
         );
     }
 
     public void update(UnloadDecision decision) {
-        var unloads = decision.getUnloads();
-        unloadBrokerCount += unloads.keySet().size();
-        unloadBundleCount += unloads.values().size();
-        breakdownCounters.get(decision.getLabel()).get(decision.getReason()).increment();
-        loadAvg = decision.loadAvg;
-        loadStd = decision.loadStd;
+        if (decision.getLabel() == Success) {
+            unloadBundleCount++;
+        }
+        breakdownCounters.get(decision.getLabel()).get(decision.getReason()).incrementAndGet();
+        updatedAt = System.currentTimeMillis();
+    }
+
+    public void update(UnloadDecision.Label label, UnloadDecision.Reason reason) {
+        if (label == Success) {
+            unloadBundleCount++;
+        }
+        breakdownCounters.get(label).get(reason).incrementAndGet();
+        updatedAt = System.currentTimeMillis();
+    }
+
+    public void updateLoadData(double loadAvg, double loadStd) {
+        this.loadAvg = loadAvg;
+        this.loadStd = loadStd;
+        updatedAt = System.currentTimeMillis();
+    }
+
+    public void updateUnloadBrokerCount(int unloadBrokerCount) {
+        this.unloadBrokerCount += unloadBrokerCount;
+        updatedAt = System.currentTimeMillis();
     }
 
     public List toMetrics(String advertisedBrokerAddress) {
@@ -125,4 +155,8 @@ public List toMetrics(String advertisedBrokerAddress) {
 
         return metrics;
     }
-}
\ No newline at end of file
+
+    public long updatedAt() {
+        return updatedAt;
+    }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadDecision.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadDecision.java
index 67503db34eee7..e1087ab6e53ba 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadDecision.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadDecision.java
@@ -18,29 +18,21 @@
  */
 package org.apache.pulsar.broker.loadbalance.extensions.models;
 
-import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Failure;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Skip;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Success;
-import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Balanced;
-import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBundles;
-import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoLoadData;
-import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Overloaded;
-import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Underloaded;
-import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Unknown;
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.Multimap;
+import lombok.AllArgsConstructor;
 import lombok.Data;
 
 /**
  * Defines the information required to unload or transfer a service unit(e.g. bundle).
  */
 @Data
+@AllArgsConstructor
 public class UnloadDecision {
-    Multimap unloads;
+    Unload unload;
     Label label;
     Reason reason;
-    Double loadAvg;
-    Double loadStd;
+
     public enum Label {
         Success,
         Skip,
@@ -55,39 +47,20 @@ public enum Reason {
         OutDatedData,
         NoLoadData,
         NoBrokers,
+        Admin,
         Unknown
     }
 
     public UnloadDecision() {
-        unloads = ArrayListMultimap.create();
+        unload = null;
         label = null;
         reason = null;
-        loadAvg = null;
-        loadStd = null;
     }
 
     public void clear() {
-        unloads.clear();
+        unload = null;
         label = null;
         reason = null;
-        loadAvg = null;
-        loadStd = null;
-    }
-
-    public void skip(int numOfOverloadedBrokers,
-                     int numOfUnderloadedBrokers,
-                     int numOfBrokersWithEmptyLoadData,
-                     int numOfBrokersWithFewBundles) {
-        label = Skip;
-        if (numOfOverloadedBrokers == 0 && numOfUnderloadedBrokers == 0) {
-            reason = Balanced;
-        } else if (numOfBrokersWithEmptyLoadData > 0) {
-            reason = NoLoadData;
-        } else if (numOfBrokersWithFewBundles > 0) {
-            reason = NoBundles;
-        } else {
-            reason = Unknown;
-        }
     }
 
     public void skip(Reason reason) {
@@ -95,22 +68,9 @@ public void skip(Reason reason) {
         this.reason = reason;
     }
 
-    public void succeed(
-                        int numOfOverloadedBrokers,
-                        int numOfUnderloadedBrokers) {
-
-        label = Success;
-        if (numOfOverloadedBrokers > numOfUnderloadedBrokers) {
-            reason = Overloaded;
-        } else {
-            reason = Underloaded;
-        }
-    }
-
-
-    public void fail() {
-        label = Failure;
-        reason = Unknown;
+    public void succeed(Reason reason) {
+        this.label = Success;
+        this.reason = reason;
     }
 
-}
\ No newline at end of file
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java
index 28acf5fba0ea1..c8332a1d7b5e6 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java
@@ -54,9 +54,6 @@ public boolean canUnload(
             String bundle,
             String srcBroker,
             Optional dstBroker) {
-
-
-
         try {
             var antiAffinityGroupOptional = LoadManagerShared.getNamespaceAntiAffinityGroup(
                     pulsar, LoadManagerShared.getNamespaceNameFromBundleName(bundle));
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/NamespaceUnloadStrategy.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/NamespaceUnloadStrategy.java
index b4dc92d92187d..42af396abcad4 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/NamespaceUnloadStrategy.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/NamespaceUnloadStrategy.java
@@ -19,6 +19,7 @@
 package org.apache.pulsar.broker.loadbalance.extensions.scheduler;
 
 import java.util.Map;
+import java.util.Set;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision;
 
@@ -38,8 +39,8 @@ public interface NamespaceUnloadStrategy {
      * @param recentlyUnloadedBrokers The recently unloaded brokers.
      * @return unloadDecision containing a list of the bundles that should be unloaded.
      */
-    UnloadDecision findBundlesForUnloading(LoadManagerContext context,
-                                           Map recentlyUnloadedBundles,
-                                           Map recentlyUnloadedBrokers);
+    Set findBundlesForUnloading(LoadManagerContext context,
+                                                Map recentlyUnloadedBundles,
+                                                Map recentlyUnloadedBrokers);
 
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java
index 9f9582df2cc28..3c67479bcc7cb 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java
@@ -18,12 +18,20 @@
  */
 package org.apache.pulsar.broker.loadbalance.extensions.scheduler;
 
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Failure;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Skip;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Balanced;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.CoolDown;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBrokers;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBundles;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoLoadData;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.OutDatedData;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Overloaded;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Underloaded;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Unknown;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.MinMaxPriorityQueue;
+import java.util.HashSet;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
@@ -39,6 +47,7 @@
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData;
 import org.apache.pulsar.broker.loadbalance.extensions.models.Unload;
+import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision;
 import org.apache.pulsar.broker.loadbalance.extensions.policies.AntiAffinityGroupPolicyHelper;
 import org.apache.pulsar.broker.loadbalance.extensions.policies.IsolationPoliciesHelper;
@@ -80,19 +89,26 @@ public class TransferShedder implements NamespaceUnloadStrategy {
     private final IsolationPoliciesHelper isolationPoliciesHelper;
     private final AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper;
 
-    private final UnloadDecision decision = new UnloadDecision();
+    private final Set decisionCache;
+    private final UnloadCounter counter;
 
     @VisibleForTesting
-    public TransferShedder(){
+    public TransferShedder(UnloadCounter counter){
         this.pulsar = null;
+        this.decisionCache = new HashSet<>();
         this.allocationPolicies = null;
+        this.counter = counter;
         this.isolationPoliciesHelper = null;
         this.antiAffinityGroupPolicyHelper = null;
     }
 
-    public TransferShedder(PulsarService pulsar, AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper) {
+    public TransferShedder(PulsarService pulsar,
+                           UnloadCounter counter,
+                           AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper){
         this.pulsar = pulsar;
+        this.decisionCache = new HashSet<>();
         this.allocationPolicies = new SimpleResourceAllocationPolicies(pulsar);
+        this.counter = counter;
         this.isolationPoliciesHelper = new IsolationPoliciesHelper(allocationPolicies);
         this.antiAffinityGroupPolicyHelper = antiAffinityGroupPolicyHelper;
     }
@@ -241,13 +257,12 @@ public String toString() {
 
 
     @Override
-    public UnloadDecision findBundlesForUnloading(LoadManagerContext context,
+    public Set findBundlesForUnloading(LoadManagerContext context,
                                                   Map recentlyUnloadedBundles,
                                                   Map recentlyUnloadedBrokers) {
         final var conf = context.brokerConfiguration();
-        decision.clear();
+        decisionCache.clear();
         stats.clear();
-        var selectedBundlesCache = decision.getUnloads();
 
         try {
             final var loadStore = context.brokerLoadDataStore();
@@ -255,22 +270,17 @@ public UnloadDecision findBundlesForUnloading(LoadManagerContext context,
             boolean debugMode = conf.isLoadBalancerDebugModeEnabled() || log.isDebugEnabled();
 
             var skipReason = stats.update(context.brokerLoadDataStore(), recentlyUnloadedBrokers, conf);
-            if (!skipReason.isEmpty()) {
-                decision.skip(skipReason.get());
-                log.warn("Failed to update load stat. Reason:{}. Stop unloading.", decision.getReason());
-                return decision;
+            if (skipReason.isPresent()) {
+                log.warn("Failed to update load stat. Reason:{}. Stop unloading.", skipReason.get());
+                counter.update(Skip, skipReason.get());
+                return decisionCache;
             }
-            decision.setLoadAvg(stats.avg);
-            decision.setLoadStd(stats.std);
+            counter.updateLoadData(stats.avg, stats.std);
 
             if (debugMode) {
                 log.info("brokers' load stats:{}", stats);
             }
 
-            // success metrics
-            int numOfOverloadedBrokers = 0;
-            int numOfUnderloadedBrokers = 0;
-
             // skip metrics
             int numOfBrokersWithEmptyLoadData = 0;
             int numOfBrokersWithFewBundles = 0;
@@ -283,9 +293,9 @@ public UnloadDecision findBundlesForUnloading(LoadManagerContext context,
                 availableBrokers = context.brokerRegistry().getAvailableBrokerLookupDataAsync()
                         .get(context.brokerConfiguration().getMetadataStoreOperationTimeoutSeconds(), TimeUnit.SECONDS);
             } catch (ExecutionException | InterruptedException | TimeoutException e) {
-                decision.skip(Unknown);
-                log.warn("Failed to fetch available brokers. Reason:{}. Stop unloading.", decision.getReason(), e);
-                return decision;
+                counter.update(Skip, Unknown);
+                log.warn("Failed to fetch available brokers. Reason: Unknown. Stop unloading.", e);
+                return decisionCache;
             }
 
             while (true) {
@@ -295,6 +305,7 @@ public UnloadDecision findBundlesForUnloading(LoadManagerContext context,
                     }
                     break;
                 }
+                UnloadDecision.Reason reason;
                 if (stats.std() <= targetStd) {
                     if (hasMsgThroughput(context, stats.minBrokers.peekLast())) {
                         if (debugMode) {
@@ -303,10 +314,10 @@ public UnloadDecision findBundlesForUnloading(LoadManagerContext context,
                         }
                         break;
                     } else {
-                        numOfUnderloadedBrokers++;
+                        reason = Underloaded;
                     }
                 } else {
-                    numOfOverloadedBrokers++;
+                    reason = Overloaded;
                 }
 
                 String maxBroker = stats.maxBrokers().pollLast();
@@ -365,13 +376,16 @@ && isTransferable(context, availableBrokers,
                             if (remainingTopBundles > 1
                                     && (trafficMarkedToOffload < offloadThroughput
                                     || !atLeastOneBundleSelected)) {
+                                Unload unload;
                                 if (transfer) {
-                                    selectedBundlesCache.put(maxBroker,
-                                            new Unload(maxBroker, bundle, Optional.of(minBroker)));
+                                    unload = new Unload(maxBroker, bundle, Optional.of(minBroker));
                                 } else {
-                                    selectedBundlesCache.put(maxBroker,
-                                            new Unload(maxBroker, bundle));
+                                    unload = new Unload(maxBroker, bundle);
                                 }
+                                var decision = new UnloadDecision();
+                                decision.setUnload(unload);
+                                decision.succeed(reason);
+                                decisionCache.add(decision);
                                 trafficMarkedToOffload += throughput;
                                 atLeastOneBundleSelected = true;
                                 remainingTopBundles--;
@@ -403,26 +417,24 @@ && isTransferable(context, availableBrokers,
             }
 
             if (debugMode) {
-                log.info("selectedBundlesCache:{}", selectedBundlesCache);
+                log.info("decisionCache:{}", decisionCache);
             }
-
-            if (decision.getUnloads().isEmpty()) {
-                decision.skip(
-                        numOfOverloadedBrokers,
-                        numOfUnderloadedBrokers,
-                        numOfBrokersWithEmptyLoadData,
-                        numOfBrokersWithFewBundles);
-            } else {
-                decision.succeed(
-                        numOfOverloadedBrokers,
-                        numOfUnderloadedBrokers);
+            if (decisionCache.isEmpty()) {
+                UnloadDecision.Reason reason;
+                if (numOfBrokersWithEmptyLoadData > 0) {
+                    reason = NoLoadData;
+                } else if (numOfBrokersWithFewBundles > 0) {
+                    reason = NoBundles;
+                } else {
+                    reason = Balanced;
+                }
+                counter.update(Skip, reason);
             }
         } catch (Throwable e) {
             log.error("Failed to process unloading. ", e);
-            decision.fail();
+            this.counter.update(Failure, Unknown);
         }
-
-        return decision;
+        return decisionCache;
     }
 
 
@@ -457,7 +469,6 @@ private boolean isTransferable(LoadManagerContext context,
         if (!antiAffinityGroupPolicyHelper.canUnload(availableBrokers, bundle, srcBroker, dstBroker)) {
             return false;
         }
-
         return true;
     }
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java
index bc3c8eb6a94fd..31310c5c9cc80 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java
@@ -18,21 +18,30 @@
  */
 package org.apache.pulsar.broker.loadbalance.extensions.scheduler;
 
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Success;
 import com.google.common.annotations.VisibleForTesting;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel;
 import org.apache.pulsar.broker.loadbalance.extensions.manager.UnloadManager;
+import org.apache.pulsar.broker.loadbalance.extensions.models.Unload;
+import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision;
+import org.apache.pulsar.broker.loadbalance.extensions.policies.AntiAffinityGroupPolicyHelper;
+import org.apache.pulsar.common.stats.Metrics;
 import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.common.util.Reflections;
 
@@ -43,6 +52,8 @@ public class UnloadScheduler implements LoadManagerScheduler {
 
     private final ScheduledExecutorService loadManagerExecutor;
 
+    private final PulsarService pulsar;
+
     private final UnloadManager unloadManager;
 
     private final LoadManagerContext context;
@@ -51,32 +62,49 @@ public class UnloadScheduler implements LoadManagerScheduler {
 
     private final ServiceConfiguration conf;
 
+    private final UnloadCounter counter;
+
+    private final AtomicReference> unloadMetrics;
+
+    private long counterLastUpdatedAt = 0;
+
     private volatile ScheduledFuture task;
 
+    private final Set unloadBrokers;
+
     private final Map recentlyUnloadedBundles;
 
     private final Map recentlyUnloadedBrokers;
 
-    private volatile CompletableFuture currentRunningFuture = null;
-
-    public UnloadScheduler(ScheduledExecutorService loadManagerExecutor,
+    public UnloadScheduler(PulsarService pulsar,
+                           ScheduledExecutorService loadManagerExecutor,
                            UnloadManager unloadManager,
                            LoadManagerContext context,
-                           ServiceUnitStateChannel channel) {
-        this(loadManagerExecutor, unloadManager, context,
-                channel, createNamespaceUnloadStrategy(context.brokerConfiguration()));
+                           ServiceUnitStateChannel channel,
+                           AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper,
+                           UnloadCounter counter,
+                           AtomicReference> unloadMetrics) {
+        this(pulsar, loadManagerExecutor, unloadManager, context, channel,
+                createNamespaceUnloadStrategy(pulsar, antiAffinityGroupPolicyHelper, counter), counter, unloadMetrics);
     }
 
     @VisibleForTesting
-    protected UnloadScheduler(ScheduledExecutorService loadManagerExecutor,
+    protected UnloadScheduler(PulsarService pulsar,
+                              ScheduledExecutorService loadManagerExecutor,
                               UnloadManager unloadManager,
                               LoadManagerContext context,
                               ServiceUnitStateChannel channel,
-                              NamespaceUnloadStrategy strategy) {
+                              NamespaceUnloadStrategy strategy,
+                              UnloadCounter counter,
+                              AtomicReference> unloadMetrics) {
+        this.pulsar = pulsar;
         this.namespaceUnloadStrategy = strategy;
         this.recentlyUnloadedBundles = new HashMap<>();
         this.recentlyUnloadedBrokers = new HashMap<>();
+        this.unloadBrokers = new HashSet<>();
         this.loadManagerExecutor = loadManagerExecutor;
+        this.counter = counter;
+        this.unloadMetrics = unloadMetrics;
         this.unloadManager = unloadManager;
         this.context = context;
         this.conf = context.brokerConfiguration();
@@ -96,62 +124,72 @@ public synchronized void execute() {
             }
             return;
         }
-        if (currentRunningFuture != null && !currentRunningFuture.isDone()) {
-            if (debugMode) {
-                log.info("Auto namespace unload is running. Skipping.");
-            }
-            return;
-        }
         // Remove bundles who have been unloaded for longer than the grace period from the recently unloaded map.
         final long timeout = System.currentTimeMillis()
                 - TimeUnit.MINUTES.toMillis(conf.getLoadBalancerSheddingGracePeriodMinutes());
         recentlyUnloadedBundles.keySet().removeIf(e -> recentlyUnloadedBundles.get(e) < timeout);
 
-        this.currentRunningFuture = channel.isChannelOwnerAsync().thenCompose(isChannelOwner -> {
-            if (!isChannelOwner) {
-                if (debugMode) {
-                    log.info("Current broker is not channel owner. Skipping.");
+        long asyncOpTimeoutMs = conf.getNamespaceBundleUnloadingTimeoutMs();
+        synchronized (namespaceUnloadStrategy) {
+            try {
+                Boolean isChannelOwner = channel.isChannelOwnerAsync().get(asyncOpTimeoutMs, TimeUnit.MILLISECONDS);
+                if (!isChannelOwner) {
+                    if (debugMode) {
+                        log.info("Current broker is not channel owner. Skipping.");
+                    }
+                    return;
                 }
-                return CompletableFuture.completedFuture(null);
-            }
-            return context.brokerRegistry().getAvailableBrokersAsync().thenCompose(availableBrokers -> {
+                List availableBrokers = context.brokerRegistry().getAvailableBrokersAsync()
+                        .get(asyncOpTimeoutMs, TimeUnit.MILLISECONDS);
                 if (debugMode) {
-                   log.info("Available brokers: {}", availableBrokers);
+                    log.info("Available brokers: {}", availableBrokers);
                 }
                 if (availableBrokers.size() <= 1) {
                     log.info("Only 1 broker available: no load shedding will be performed. Skipping.");
-                    return CompletableFuture.completedFuture(null);
+                    return;
                 }
-                final UnloadDecision unloadDecision = namespaceUnloadStrategy
+                final Set decisions = namespaceUnloadStrategy
                         .findBundlesForUnloading(context, recentlyUnloadedBundles, recentlyUnloadedBrokers);
                 if (debugMode) {
                     log.info("[{}] Unload decision result: {}",
-                            namespaceUnloadStrategy.getClass().getSimpleName(), unloadDecision.toString());
+                            namespaceUnloadStrategy.getClass().getSimpleName(), decisions);
                 }
-                if (unloadDecision.getUnloads().isEmpty()) {
+                if (decisions.isEmpty()) {
                     if (debugMode) {
                         log.info("[{}] Unload decision unloads is empty. Skipping.",
                                 namespaceUnloadStrategy.getClass().getSimpleName());
                     }
-                    return CompletableFuture.completedFuture(null);
+                    return;
                 }
                 List> futures = new ArrayList<>();
-                unloadDecision.getUnloads().forEach((broker, unload) -> {
-                    log.info("[{}] Unloading bundle: {}", namespaceUnloadStrategy.getClass().getSimpleName(), unload);
-                    futures.add(unloadManager.waitAsync(channel.publishUnloadEventAsync(unload), unload.serviceUnit(),
-                                    conf.getNamespaceBundleUnloadingTimeoutMs(), TimeUnit.MILLISECONDS)
-                            .thenAccept(__ -> {
-                                recentlyUnloadedBundles.put(unload.serviceUnit(), System.currentTimeMillis());
-                                recentlyUnloadedBrokers.put(unload.sourceBroker(), System.currentTimeMillis());
-                    }));
-                });
-                return FutureUtil.waitForAll(futures).exceptionally(ex -> {
-                    log.error("[{}] Namespace unload has exception.",
-                            namespaceUnloadStrategy.getClass().getSimpleName(), ex);
-                    return null;
+                unloadBrokers.clear();
+                decisions.forEach(decision -> {
+                    if (decision.getLabel() == Success) {
+                        Unload unload = decision.getUnload();
+                        log.info("[{}] Unloading bundle: {}",
+                                namespaceUnloadStrategy.getClass().getSimpleName(), unload);
+                        futures.add(unloadManager.waitAsync(channel.publishUnloadEventAsync(unload),
+                                        unload.serviceUnit(), decision, asyncOpTimeoutMs, TimeUnit.MILLISECONDS)
+                                .thenAccept(__ -> {
+                                    unloadBrokers.add(unload.sourceBroker());
+                                    recentlyUnloadedBundles.put(unload.serviceUnit(), System.currentTimeMillis());
+                                    recentlyUnloadedBrokers.put(unload.sourceBroker(), System.currentTimeMillis());
+                                }));
+                    }
                 });
-            });
-        });
+                FutureUtil.waitForAll(futures)
+                        .whenComplete((__, ex) -> counter.updateUnloadBrokerCount(unloadBrokers.size()))
+                        .get(asyncOpTimeoutMs, TimeUnit.MILLISECONDS);
+            } catch (Exception ex) {
+                log.error("[{}] Namespace unload has exception.",
+                        namespaceUnloadStrategy.getClass().getSimpleName(), ex);
+            } finally {
+                if (counter.updatedAt() > counterLastUpdatedAt) {
+                    unloadMetrics.set(counter.toMetrics(pulsar.getAdvertisedAddress()));
+                    counterLastUpdatedAt = counter.updatedAt();
+                }
+            }
+        }
     }
 
     @Override
@@ -174,7 +212,10 @@ public void close() {
         this.recentlyUnloadedBrokers.clear();
     }
 
-    private static NamespaceUnloadStrategy createNamespaceUnloadStrategy(ServiceConfiguration conf) {
+    private static NamespaceUnloadStrategy createNamespaceUnloadStrategy(PulsarService pulsar,
+                                                                         AntiAffinityGroupPolicyHelper helper,
+                                                                         UnloadCounter counter) {
+        ServiceConfiguration conf = pulsar.getConfiguration();
         try {
             return Reflections.createInstance(conf.getLoadBalancerLoadSheddingStrategy(), NamespaceUnloadStrategy.class,
                     Thread.currentThread().getContextClassLoader());
@@ -183,7 +224,7 @@ private static NamespaceUnloadStrategy createNamespaceUnloadStrategy(ServiceConf
                     conf.getLoadBalancerLoadPlacementStrategy(), e);
         }
         log.error("create namespace unload strategy failed. using TransferShedder instead.");
-        return new TransferShedder();
+        return new TransferShedder(pulsar, counter, helper);
     }
 
     private boolean isLoadBalancerSheddingEnabled() {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
index aa8583c6b57b6..354a27a63c02a 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
@@ -64,8 +64,6 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ConcurrentMap;
-import org.apache.commons.lang3.mutable.MutableLong;
 import org.apache.commons.lang3.reflect.FieldUtils;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
@@ -74,9 +72,7 @@
 import org.apache.pulsar.broker.loadbalance.LeaderBroker;
 import org.apache.pulsar.broker.loadbalance.LeaderElectionService;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState;
-import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl;
-import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData;
@@ -206,10 +202,9 @@ protected void cleanup() throws Exception {
     }
 
     @BeforeMethod
-    protected void initializeState() throws IllegalAccessException {
+    protected void initializeState() throws PulsarAdminException {
+        admin.namespaces().unload("public/default");
         reset(primaryLoadManager, secondaryLoadManager);
-        cleanTableView(channel1);
-        cleanTableView(channel2);
     }
 
     @Test
@@ -561,20 +556,20 @@ public void testGetMetrics() throws Exception {
             FieldUtils.writeDeclaredField(unloadCounter, "loadStd", 0.3, true);
             FieldUtils.writeDeclaredField(unloadCounter, "breakdownCounters", Map.of(
                     Success, new LinkedHashMap<>() {{
-                        put(Overloaded, new MutableLong(1));
-                        put(Underloaded, new MutableLong(2));
+                        put(Overloaded, new AtomicLong(1));
+                        put(Underloaded, new AtomicLong(2));
                     }},
                     Skip, new LinkedHashMap<>() {{
-                        put(Balanced, new MutableLong(3));
-                        put(NoBundles, new MutableLong(4));
-                        put(CoolDown, new MutableLong(5));
-                        put(OutDatedData, new MutableLong(6));
-                        put(NoLoadData, new MutableLong(7));
-                        put(NoBrokers, new MutableLong(8));
-                        put(Unknown, new MutableLong(9));
+                        put(Balanced, new AtomicLong(3));
+                        put(NoBundles, new AtomicLong(4));
+                        put(CoolDown, new AtomicLong(5));
+                        put(OutDatedData, new AtomicLong(6));
+                        put(NoLoadData, new AtomicLong(7));
+                        put(NoBrokers, new AtomicLong(8));
+                        put(Unknown, new AtomicLong(9));
                     }},
                     Failure, Map.of(
-                            Unknown, new MutableLong(10))
+                            Unknown, new AtomicLong(10))
             ), true);
             unloadMetrics.set(unloadCounter.toMetrics(pulsar.getAdvertisedAddress()));
         }
@@ -722,15 +717,6 @@ public void initialize(PulsarService pulsar) {
 
     }
 
-    private static void cleanTableView(ServiceUnitStateChannel channel)
-            throws IllegalAccessException {
-        var tv = (TableViewImpl)
-                FieldUtils.readField(channel, "tableview", true);
-        var cache = (ConcurrentMap)
-                FieldUtils.readField(tv, "data", true);
-        cache.clear();
-    }
-
     private void setPrimaryLoadManager() throws IllegalAccessException {
         ExtensibleLoadManagerWrapper wrapper =
                 (ExtensibleLoadManagerWrapper) pulsar1.getLoadManager().get();
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManagerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManagerTest.java
index 75ef913b8a851..6a2ae1cc562cc 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManagerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManagerTest.java
@@ -19,6 +19,10 @@
 package org.apache.pulsar.broker.loadbalance.extensions.manager;
 
 import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.VERSION_ID_INIT;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Failure;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Success;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Admin;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Unknown;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertTrue;
 import static org.testng.Assert.fail;
@@ -32,6 +36,9 @@
 import org.apache.commons.lang3.reflect.FieldUtils;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData;
+import org.apache.pulsar.broker.loadbalance.extensions.models.Unload;
+import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter;
+import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision;
 import org.apache.pulsar.common.util.FutureUtil;
 import org.testng.annotations.Test;
 
@@ -41,10 +48,13 @@ public class UnloadManagerTest {
 
     @Test
     public void testEventPubFutureHasException() {
-        UnloadManager manager = new UnloadManager();
+        UnloadCounter counter = new UnloadCounter();
+        UnloadManager manager = new UnloadManager(counter);
+        var unloadDecision =
+                new UnloadDecision(new Unload("broker-1", "bundle-1"), Success, Admin);
         CompletableFuture future =
                 manager.waitAsync(FutureUtil.failedFuture(new Exception("test")),
-                        "bundle-1", 10, TimeUnit.SECONDS);
+                        "bundle-1", unloadDecision, 10, TimeUnit.SECONDS);
 
         assertTrue(future.isCompletedExceptionally());
         try {
@@ -53,14 +63,18 @@ public void testEventPubFutureHasException() {
         } catch (Exception ex) {
             assertEquals(ex.getCause().getMessage(), "test");
         }
+        assertEquals(counter.getBreakdownCounters().get(Failure).get(Unknown).get(), 1);
     }
 
     @Test
     public void testTimeout() throws IllegalAccessException {
-        UnloadManager manager = new UnloadManager();
+        UnloadCounter counter = new UnloadCounter();
+        UnloadManager manager = new UnloadManager(counter);
+        var unloadDecision =
+                new UnloadDecision(new Unload("broker-1", "bundle-1"), Success, Admin);
         CompletableFuture future =
                 manager.waitAsync(CompletableFuture.completedFuture(null),
-                        "bundle-1", 3, TimeUnit.SECONDS);
+                        "bundle-1", unloadDecision, 3, TimeUnit.SECONDS);
         Map> inFlightUnloadRequestMap = getInFlightUnloadRequestMap(manager);
 
         assertEquals(inFlightUnloadRequestMap.size(), 1);
@@ -73,14 +87,18 @@ public void testTimeout() throws IllegalAccessException {
         }
 
         assertEquals(inFlightUnloadRequestMap.size(), 0);
+        assertEquals(counter.getBreakdownCounters().get(Failure).get(Unknown).get(), 1);
     }
 
     @Test
     public void testSuccess() throws IllegalAccessException, ExecutionException, InterruptedException {
-        UnloadManager manager = new UnloadManager();
+        UnloadCounter counter = new UnloadCounter();
+        UnloadManager manager = new UnloadManager(counter);
+        var unloadDecision =
+                new UnloadDecision(new Unload("broker-1", "bundle-1"), Success, Admin);
         CompletableFuture future =
                 manager.waitAsync(CompletableFuture.completedFuture(null),
-                        "bundle-1", 5, TimeUnit.SECONDS);
+                        "bundle-1", unloadDecision, 5, TimeUnit.SECONDS);
         Map> inFlightUnloadRequestMap = getInFlightUnloadRequestMap(manager);
 
         assertEquals(inFlightUnloadRequestMap.size(), 1);
@@ -109,10 +127,11 @@ public void testSuccess() throws IllegalAccessException, ExecutionException, Int
                 new ServiceUnitStateData(ServiceUnitState.Free, "broker-1", VERSION_ID_INIT), null);
         assertEquals(inFlightUnloadRequestMap.size(), 0);
         future.get();
+        assertEquals(counter.getBreakdownCounters().get(Success).get(Admin).get(), 1);
 
         // Success with Owned state.
         future = manager.waitAsync(CompletableFuture.completedFuture(null),
-                "bundle-1", 5, TimeUnit.SECONDS);
+                "bundle-1", unloadDecision, 5, TimeUnit.SECONDS);
         inFlightUnloadRequestMap = getInFlightUnloadRequestMap(manager);
 
         assertEquals(inFlightUnloadRequestMap.size(), 1);
@@ -121,14 +140,19 @@ public void testSuccess() throws IllegalAccessException, ExecutionException, Int
                 new ServiceUnitStateData(ServiceUnitState.Owned, "broker-1", VERSION_ID_INIT), null);
         assertEquals(inFlightUnloadRequestMap.size(), 0);
         future.get();
+
+        assertEquals(counter.getBreakdownCounters().get(Success).get(Admin).get(), 2);
     }
 
     @Test
     public void testFailedStage() throws IllegalAccessException {
-        UnloadManager manager = new UnloadManager();
+        UnloadCounter counter = new UnloadCounter();
+        UnloadManager manager = new UnloadManager(counter);
+        var unloadDecision =
+                new UnloadDecision(new Unload("broker-1", "bundle-1"), Success, Admin);
         CompletableFuture future =
                 manager.waitAsync(CompletableFuture.completedFuture(null),
-                        "bundle-1", 5, TimeUnit.SECONDS);
+                        "bundle-1", unloadDecision, 5, TimeUnit.SECONDS);
         Map> inFlightUnloadRequestMap = getInFlightUnloadRequestMap(manager);
 
         assertEquals(inFlightUnloadRequestMap.size(), 1);
@@ -146,14 +170,18 @@ public void testFailedStage() throws IllegalAccessException {
         }
 
         assertEquals(inFlightUnloadRequestMap.size(), 0);
+        assertEquals(counter.getBreakdownCounters().get(Failure).get(Unknown).get(), 1);
     }
 
     @Test
     public void testClose() throws IllegalAccessException {
-        UnloadManager manager = new UnloadManager();
+        UnloadCounter counter = new UnloadCounter();
+        UnloadManager manager = new UnloadManager(counter);
+        var unloadDecision =
+                new UnloadDecision(new Unload("broker-1", "bundle-1"), Success, Admin);
         CompletableFuture future =
                 manager.waitAsync(CompletableFuture.completedFuture(null),
-                        "bundle-1", 5, TimeUnit.SECONDS);
+                        "bundle-1", unloadDecision,5, TimeUnit.SECONDS);
         Map> inFlightUnloadRequestMap = getInFlightUnloadRequestMap(manager);
         assertEquals(inFlightUnloadRequestMap.size(), 1);
         manager.close();
@@ -165,6 +193,7 @@ public void testClose() throws IllegalAccessException {
         } catch (Exception ex) {
             assertTrue(ex.getCause() instanceof IllegalStateException);
         }
+        assertEquals(counter.getBreakdownCounters().get(Failure).get(Unknown).get(), 1);
     }
 
     private Map> getInFlightUnloadRequestMap(UnloadManager manager)
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
index 3955f1ed9af2e..a89f33bb98737 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
@@ -45,6 +45,7 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Random;
@@ -64,6 +65,7 @@
 import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData;
 import org.apache.pulsar.broker.loadbalance.extensions.models.TopKBundles;
 import org.apache.pulsar.broker.loadbalance.extensions.models.Unload;
+import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision;
 import org.apache.pulsar.broker.loadbalance.extensions.policies.AntiAffinityGroupPolicyHelper;
 import org.apache.pulsar.broker.loadbalance.extensions.policies.IsolationPoliciesHelper;
@@ -351,19 +353,19 @@ public void startTableView() throws LoadDataStoreException {
 
     @Test
     public void testEmptyBrokerLoadData() {
-        TransferShedder transferShedder = new TransferShedder();
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = getContext();
         ctx.brokerConfiguration().setLoadBalancerDebugModeEnabled(true);
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
-        var expected = new UnloadDecision();
-        expected.setLabel(Skip);
-        expected.setReason(NoBrokers);
-        assertEquals(res, expected);
+        assertTrue(res.isEmpty());
+        assertEquals(counter.getBreakdownCounters().get(Skip).get(NoBrokers).get(), 1);
     }
 
     @Test
     public void testEmptyTopBundlesLoadData() {
-        TransferShedder transferShedder = new TransferShedder();
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = getContext();
         ctx.brokerConfiguration().setLoadBalancerDebugModeEnabled(true);
         var brokerLoadDataStore = ctx.brokerLoadDataStore();
@@ -373,21 +375,20 @@ public void testEmptyTopBundlesLoadData() {
         brokerLoadDataStore.pushAsync("broker3", getCpuLoad(ctx,  20));
 
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
-        var expected = new UnloadDecision();
-        expected.setLabel(Skip);
-        expected.setReason(NoLoadData);
-        expected.setLoadAvg(0.39999999999999997);
-        expected.setLoadStd(0.35590260840104376);
-        assertEquals(res, expected);
+        assertTrue(res.isEmpty());
+        assertEquals(counter.getBreakdownCounters().get(Skip).get(NoLoadData).get(), 1);
+        assertEquals(counter.getLoadAvg(), 0.39999999999999997);
+        assertEquals(counter.getLoadStd(), 0.35590260840104376);
     }
 
     @Test
     public void testOutDatedLoadData() throws IllegalAccessException {
-        TransferShedder transferShedder = new TransferShedder();
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = setupContext();
         var brokerLoadDataStore = ctx.brokerLoadDataStore();
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
-        assertEquals(res.getUnloads().size(), 2);
+        assertEquals(res.size(), 2);
 
 
         FieldUtils.writeDeclaredField(brokerLoadDataStore.get("broker1").get(), "updatedAt", 0, true);
@@ -398,16 +399,14 @@ public void testOutDatedLoadData() throws IllegalAccessException {
 
 
         res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
-
-        var expected = new UnloadDecision();
-        expected.setLabel(Skip);
-        expected.setReason(OutDatedData);
-        assertEquals(res, expected);
+        assertTrue(res.isEmpty());
+        assertEquals(counter.getBreakdownCounters().get(Skip).get(OutDatedData).get(), 1);
     }
 
     @Test
     public void testRecentlyUnloadedBrokers() {
-        TransferShedder transferShedder = new TransferShedder();
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = setupContext();
 
         Map recentlyUnloadedBrokers = new HashMap<>();
@@ -416,32 +415,27 @@ public void testRecentlyUnloadedBrokers() {
         recentlyUnloadedBrokers.put("broker1", oldTS);
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), recentlyUnloadedBrokers);
 
-        var expected = new UnloadDecision();
-        var unloads = expected.getUnloads();
-        unloads.put("broker5",
-                new Unload("broker5", bundleE1, Optional.of("broker1")));
-        unloads.put("broker4",
-                new Unload("broker4", bundleD1, Optional.of("broker2")));
-
-        expected.setLabel(Success);
-        expected.setReason(Overloaded);
-        expected.setLoadAvg(setupLoadAvg);
-        expected.setLoadStd(setupLoadStd);
+        var expected = new HashSet();
+        expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")),
+                Success, Overloaded));
+        expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")),
+                Success, Overloaded));
         assertEquals(res, expected);
+        assertEquals(counter.getLoadAvg(), setupLoadAvg);
+        assertEquals(counter.getLoadStd(), setupLoadStd);
 
         var now = System.currentTimeMillis();
         recentlyUnloadedBrokers.put("broker1", now);
         res = transferShedder.findBundlesForUnloading(ctx, Map.of(), recentlyUnloadedBrokers);
 
-        expected = new UnloadDecision();
-        expected.setLabel(Skip);
-        expected.setReason(CoolDown);
-        assertEquals(res, expected);
+        assertTrue(res.isEmpty());
+        assertEquals(counter.getBreakdownCounters().get(Skip).get(CoolDown).get(), 1);
     }
 
     @Test
     public void testRecentlyUnloadedBundles() {
-        TransferShedder transferShedder = new TransferShedder();
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = setupContext();
         Map recentlyUnloadedBundles = new HashMap<>();
         var now = System.currentTimeMillis();
@@ -451,35 +445,34 @@ public void testRecentlyUnloadedBundles() {
         recentlyUnloadedBundles.put(bundleD2, now);
         var res = transferShedder.findBundlesForUnloading(ctx, recentlyUnloadedBundles, Map.of());
 
-        var expected = new UnloadDecision();
-        expected.setLabel(Skip);
-        expected.skip(NoBundles);
-        expected.setLoadAvg(setupLoadAvg);
-        expected.setLoadStd(setupLoadStd);
-        assertEquals(res, expected);
+        assertTrue(res.isEmpty());
+        assertEquals(counter.getBreakdownCounters().get(Skip).get(NoBundles).get(), 1);
+        assertEquals(counter.getLoadAvg(), setupLoadAvg);
+        assertEquals(counter.getLoadStd(), setupLoadStd);
     }
 
     @Test
     public void testGetAvailableBrokersFailed() {
-        TransferShedder transferShedder = new TransferShedder();
+        var pulsar = getMockPulsar();
+        UnloadCounter counter = new UnloadCounter();
+        AntiAffinityGroupPolicyHelper affinityGroupPolicyHelper = mock(AntiAffinityGroupPolicyHelper.class);
+        TransferShedder transferShedder = new TransferShedder(pulsar, counter, affinityGroupPolicyHelper);
         var ctx = setupContext();
         BrokerRegistry registry = ctx.brokerRegistry();
         doReturn(FutureUtil.failedFuture(new TimeoutException())).when(registry).getAvailableBrokerLookupDataAsync();
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
 
-        var expected = new UnloadDecision();
-        expected.setLabel(Skip);
-        expected.skip(Unknown);
-        expected.setLoadAvg(setupLoadAvg);
-        expected.setLoadStd(setupLoadStd);
-        assertEquals(res, expected);
+        assertTrue(res.isEmpty());
+        assertEquals(counter.getBreakdownCounters().get(Skip).get(Unknown).get(), 1);
+        assertEquals(counter.getLoadAvg(), setupLoadAvg);
+        assertEquals(counter.getLoadStd(), setupLoadStd);
     }
 
     @Test(timeOut = 30 * 1000)
     public void testBundlesWithIsolationPolicies() throws IllegalAccessException {
-
-
-        TransferShedder transferShedder = new TransferShedder(pulsar, antiAffinityGroupPolicyHelper);
+        var pulsar = getMockPulsar();
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = spy(new TransferShedder(pulsar, counter, antiAffinityGroupPolicyHelper));
 
         var allocationPoliciesSpy = (SimpleResourceAllocationPolicies)
                 spy(FieldUtils.readDeclaredField(transferShedder, "allocationPolicies", true));
@@ -487,33 +480,27 @@ public void testBundlesWithIsolationPolicies() throws IllegalAccessException {
         IsolationPoliciesHelper isolationPoliciesHelper = new IsolationPoliciesHelper(allocationPoliciesSpy);
         FieldUtils.writeDeclaredField(transferShedder, "isolationPoliciesHelper", isolationPoliciesHelper, true);
 
-        // Test transfer to a has isolation policies broker.
         setIsolationPolicies(allocationPoliciesSpy, "my-tenant/my-namespaceE",
                 Set.of("broker5"), Set.of(), Set.of(), 1);
         var ctx = setupContext();
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
-        var expected = new UnloadDecision();
-        var unloads = expected.getUnloads();
-        unloads.put("broker4",
-                new Unload("broker4", bundleD1, Optional.of("broker2")));
-        expected.setLabel(Success);
-        expected.setReason(Overloaded);
-        expected.setLoadAvg(setupLoadAvg);
-        expected.setLoadStd(setupLoadStd);
+        var expected = new HashSet();
+        expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")),
+                Success, Overloaded));
         assertEquals(res, expected);
+        assertEquals(counter.getLoadAvg(), setupLoadAvg);
+        assertEquals(counter.getLoadStd(), setupLoadStd);
 
         // Test unload a has isolation policies broker.
         ctx.brokerConfiguration().setLoadBalancerTransferEnabled(false);
         res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
-        expected = new UnloadDecision();
-        unloads = expected.getUnloads();
-        unloads.put("broker4",
-                new Unload("broker4", bundleD1, Optional.empty()));
-        expected.setLabel(Success);
-        expected.setReason(Overloaded);
-        expected.setLoadAvg(setupLoadAvg);
-        expected.setLoadStd(setupLoadStd);
+
+        expected = new HashSet<>();
+        expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.empty()),
+                Success, Overloaded));
         assertEquals(res, expected);
+        assertEquals(counter.getLoadAvg(), setupLoadAvg);
+        assertEquals(counter.getLoadStd(), setupLoadStd);
     }
 
     public BrokerLookupData getLookupData() {
@@ -590,7 +577,8 @@ private PulsarService getMockPulsar() {
     @Test
     public void testBundlesWithAntiAffinityGroup() throws IllegalAccessException, MetadataStoreException {
         var pulsar = getMockPulsar();
-        TransferShedder transferShedder = new TransferShedder(pulsar, antiAffinityGroupPolicyHelper);
+        var counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(pulsar, counter, antiAffinityGroupPolicyHelper);
         var allocationPoliciesSpy = (SimpleResourceAllocationPolicies)
                 spy(FieldUtils.readDeclaredField(transferShedder, "allocationPolicies", true));
         doReturn(false).when(allocationPoliciesSpy).areIsolationPoliciesPresent(any());
@@ -603,29 +591,26 @@ public void testBundlesWithAntiAffinityGroup() throws IllegalAccessException, Me
         doReturn(false).when(antiAffinityGroupPolicyHelper).canUnload(any(), any(), any(), any());
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
 
-        var expected = new UnloadDecision();
-        expected.setLabel(Skip);
-        expected.skip(NoBundles);
-        expected.setLoadAvg(setupLoadAvg);
-        expected.setLoadStd(setupLoadStd);
-        assertEquals(res, expected);
+        assertTrue(res.isEmpty());
+        assertEquals(counter.getBreakdownCounters().get(Skip).get(NoBundles).get(), 1);
+        assertEquals(counter.getLoadAvg(), setupLoadAvg);
+        assertEquals(counter.getLoadStd(), setupLoadStd);
+
 
         doReturn(true).when(antiAffinityGroupPolicyHelper).canUnload(any(), eq(bundleE1), any(), any());
         var res2 = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
-        var expected2 = new UnloadDecision();
-        var unloads = expected2.getUnloads();
-        unloads.put("broker5",
-                new Unload("broker5", bundleE1, Optional.of("broker1")));
-        expected2.setLabel(Success);
-        expected2.setReason(Overloaded);
-        expected2.setLoadAvg(setupLoadAvg);
-        expected2.setLoadStd(setupLoadStd);
-        assertEquals(res2, expected2);
+        var expected2 = new HashSet<>();
+        expected2.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")),
+                Success, Overloaded));
+        assertEquals(res, expected2);
+        assertEquals(counter.getLoadAvg(), setupLoadAvg);
+        assertEquals(counter.getLoadStd(), setupLoadStd);
     }
 
     @Test
     public void testTargetStd() {
-        TransferShedder transferShedder = new TransferShedder();
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = getContext();
         ctx.brokerConfiguration().setLoadBalancerDebugModeEnabled(true);
         var brokerLoadDataStore = ctx.brokerLoadDataStore();
@@ -641,17 +626,16 @@ public void testTargetStd() {
 
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
 
-        var expected = new UnloadDecision();
-        expected.setLabel(Skip);
-        expected.skip(Balanced);
-        expected.setLoadAvg(0.2000000063578288);
-        expected.setLoadStd(0.08164966587949089);
-        assertEquals(res, expected);
+        assertTrue(res.isEmpty());
+        assertEquals(counter.getBreakdownCounters().get(Skip).get(Balanced).get(), 1);
+        assertEquals(counter.getLoadAvg(), 0.2000000063578288);
+        assertEquals(counter.getLoadStd(), 0.08164966587949089);
     }
 
     @Test
     public void testSingleTopBundlesLoadData() {
-        TransferShedder transferShedder = new TransferShedder();
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = setupContext();
         var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
         topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 1));
@@ -661,38 +645,35 @@ public void testSingleTopBundlesLoadData() {
         topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 70));
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
 
-        var expected = new UnloadDecision();
-        expected.setLabel(Skip);
-        expected.skip(NoBundles);
-        expected.setLoadAvg(setupLoadAvg);
-        expected.setLoadStd(setupLoadStd);
-        assertEquals(res, expected);
+        assertTrue(res.isEmpty());
+        assertEquals(counter.getBreakdownCounters().get(Skip).get(NoBundles).get(), 1);
+        assertEquals(counter.getLoadAvg(), setupLoadAvg);
+        assertEquals(counter.getLoadStd(), setupLoadStd);
     }
 
 
     @Test
     public void testTargetStdAfterTransfer() {
-        TransferShedder transferShedder = new TransferShedder();
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = setupContext();
         var brokerLoadDataStore = ctx.brokerLoadDataStore();
         brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx,  55));
         brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx,  65));
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
 
-        var expected = new UnloadDecision();
-        var unloads = expected.getUnloads();
-        unloads.put("broker5",
-                new Unload("broker5", bundleE1, Optional.of("broker1")));
-        expected.setLabel(Success);
-        expected.setReason(Overloaded);
-        expected.setLoadAvg(0.26400000000000007);
-        expected.setLoadStd(0.27644891028904417);
+        var expected = new HashSet();
+        expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")),
+                Success, Overloaded));
         assertEquals(res, expected);
+        assertEquals(counter.getLoadAvg(), 0.26400000000000007);
+        assertEquals(counter.getLoadStd(), 0.27644891028904417);
     }
 
     @Test
     public void testMinBrokerWithZeroTraffic() throws IllegalAccessException {
-        TransferShedder transferShedder = new TransferShedder();
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = setupContext();
         var brokerLoadDataStore = ctx.brokerLoadDataStore();
         brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx,  55));
@@ -705,64 +686,57 @@ public void testMinBrokerWithZeroTraffic() throws IllegalAccessException {
 
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
 
-        var expected = new UnloadDecision();
-        var unloads = expected.getUnloads();
-        unloads.put("broker5",
-                new Unload("broker5", bundleE1, Optional.of("broker1")));
-        unloads.put("broker4",
-                new Unload("broker4", bundleD1, Optional.of("broker2")));
-        expected.setLabel(Success);
-        expected.setReason(Underloaded);
-        expected.setLoadAvg(0.26400000000000007);
-        expected.setLoadStd(0.27644891028904417);
+        var expected = new HashSet();
+        expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")),
+                Success, Overloaded));
+        expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")),
+                Success, Underloaded));
         assertEquals(res, expected);
+        assertEquals(counter.getLoadAvg(), 0.26400000000000007);
+        assertEquals(counter.getLoadStd(), 0.27644891028904417);
     }
 
     @Test
     public void testMaxNumberOfTransfersPerShedderCycle() {
-        TransferShedder transferShedder = new TransferShedder();
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = setupContext();
         ctx.brokerConfiguration()
                 .setLoadBalancerMaxNumberOfBrokerTransfersPerCycle(10);
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
-
-        var expected = new UnloadDecision();
-        var unloads = expected.getUnloads();
-        unloads.put("broker5",
-                new Unload("broker5", bundleE1, Optional.of("broker1")));
-        unloads.put("broker4",
-                new Unload("broker4", bundleD1, Optional.of("broker2")));
-        expected.setLabel(Success);
-        expected.setReason(Overloaded);
-        expected.setLoadAvg(setupLoadAvg);
-        expected.setLoadStd(setupLoadStd);
+        var expected = new HashSet();
+        expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")),
+                Success, Overloaded));
+        expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")),
+                Success, Overloaded));
         assertEquals(res, expected);
+        assertEquals(counter.getLoadAvg(), setupLoadAvg);
+        assertEquals(counter.getLoadStd(), setupLoadStd);
     }
 
     @Test
     public void testRemainingTopBundles() {
-        TransferShedder transferShedder = new TransferShedder();
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = setupContext();
         var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
         topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 3000000, 2000000));
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
 
-        var expected = new UnloadDecision();
-        var unloads = expected.getUnloads();
-        unloads.put("broker5",
-                new Unload("broker5", bundleE1, Optional.of("broker1")));
-        unloads.put("broker4",
-                new Unload("broker4", bundleD1, Optional.of("broker2")));
-        expected.setLabel(Success);
-        expected.setReason(Overloaded);
-        expected.setLoadAvg(setupLoadAvg);
-        expected.setLoadStd(setupLoadStd);
+        var expected = new HashSet();
+        expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")),
+                Success, Overloaded));
+        expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")),
+                Success, Overloaded));
         assertEquals(res, expected);
+        assertEquals(counter.getLoadAvg(), setupLoadAvg);
+        assertEquals(counter.getLoadStd(), setupLoadStd);
     }
 
     @Test
     public void testRandomLoad() throws IllegalAccessException {
-        TransferShedder transferShedder = new TransferShedder();
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
         for (int i = 0; i < 5; i++) {
             var ctx = setupContext(10);
             var conf = ctx.brokerConfiguration();
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java
index 73d4eb1f18bfb..7d9cf556360e4 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java
@@ -28,23 +28,29 @@
 import static org.mockito.Mockito.verify;
 
 import com.google.common.collect.Lists;
+import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.loadbalance.extensions.BrokerRegistry;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel;
 import org.apache.pulsar.broker.loadbalance.extensions.manager.UnloadManager;
 import org.apache.pulsar.broker.loadbalance.extensions.models.Unload;
+import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision;
 import org.apache.pulsar.client.util.ExecutorProvider;
+import org.apache.pulsar.common.stats.Metrics;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
+import java.util.List;
+import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
 
 @Test(groups = "broker")
 public class UnloadSchedulerTest {
@@ -70,23 +76,28 @@ public void tearDown() {
 
     @Test(timeOut = 30 * 1000)
     public void testExecuteSuccess() {
+        AtomicReference> reference = new AtomicReference<>();
+        UnloadCounter counter = new UnloadCounter();
         LoadManagerContext context = setupContext();
         BrokerRegistry registry = context.brokerRegistry();
         ServiceUnitStateChannel channel = mock(ServiceUnitStateChannel.class);
         UnloadManager unloadManager = mock(UnloadManager.class);
+        PulsarService pulsar = mock(PulsarService.class);
         NamespaceUnloadStrategy unloadStrategy = mock(NamespaceUnloadStrategy.class);
         doReturn(CompletableFuture.completedFuture(true)).when(channel).isChannelOwnerAsync();
         doReturn(CompletableFuture.completedFuture(Lists.newArrayList("broker-1", "broker-2")))
                 .when(registry).getAvailableBrokersAsync();
         doReturn(CompletableFuture.completedFuture(null)).when(channel).publishUnloadEventAsync(any());
         doReturn(CompletableFuture.completedFuture(null)).when(unloadManager)
-                .waitAsync(any(), any(), anyLong(), any());
+                .waitAsync(any(), any(), any(), anyLong(), any());
         UnloadDecision decision = new UnloadDecision();
         Unload unload = new Unload("broker-1", "bundle-1");
-        decision.getUnloads().put("broker-1", unload);
-        doReturn(decision).when(unloadStrategy).findBundlesForUnloading(any(), any(), any());
+        decision.setUnload(unload);
+        decision.setLabel(UnloadDecision.Label.Success);
+        doReturn(Set.of(decision)).when(unloadStrategy).findBundlesForUnloading(any(), any(), any());
 
-        UnloadScheduler scheduler = new UnloadScheduler(loadManagerExecutor, unloadManager, context, channel, unloadStrategy);
+        UnloadScheduler scheduler = new UnloadScheduler(pulsar, loadManagerExecutor, unloadManager, context,
+                channel, unloadStrategy, counter, reference);
 
         scheduler.execute();
 
@@ -94,7 +105,7 @@ public void testExecuteSuccess() {
 
         // Test empty unload.
         UnloadDecision emptyUnload = new UnloadDecision();
-        doReturn(emptyUnload).when(unloadStrategy).findBundlesForUnloading(any(), any(), any());
+        doReturn(Set.of(emptyUnload)).when(unloadStrategy).findBundlesForUnloading(any(), any(), any());
 
         scheduler.execute();
 
@@ -103,26 +114,29 @@ public void testExecuteSuccess() {
 
     @Test(timeOut = 30 * 1000)
     public void testExecuteMoreThenOnceWhenFirstNotDone() throws InterruptedException {
+        AtomicReference> reference = new AtomicReference<>();
+        UnloadCounter counter = new UnloadCounter();
         LoadManagerContext context = setupContext();
         BrokerRegistry registry = context.brokerRegistry();
         ServiceUnitStateChannel channel = mock(ServiceUnitStateChannel.class);
         UnloadManager unloadManager = mock(UnloadManager.class);
+        PulsarService pulsar = mock(PulsarService.class);
         NamespaceUnloadStrategy unloadStrategy = mock(NamespaceUnloadStrategy.class);
         doReturn(CompletableFuture.completedFuture(true)).when(channel).isChannelOwnerAsync();
         doAnswer(__ -> CompletableFuture.supplyAsync(() -> {
                 try {
                     // Delay 5 seconds to finish.
-                    TimeUnit.SECONDS.sleep(5);
+                    TimeUnit.SECONDS.sleep(1);
                 } catch (InterruptedException e) {
                     throw new RuntimeException(e);
                 }
                 return Lists.newArrayList("broker-1", "broker-2");
             }, Executors.newFixedThreadPool(1))).when(registry).getAvailableBrokersAsync();
-        UnloadScheduler scheduler = new UnloadScheduler(loadManagerExecutor, unloadManager, context, channel, unloadStrategy);
-
-        ExecutorService executorService = Executors.newFixedThreadPool(10);
-        CountDownLatch latch = new CountDownLatch(10);
-        for (int i = 0; i < 10; i++) {
+        UnloadScheduler scheduler = new UnloadScheduler(pulsar, loadManagerExecutor, unloadManager, context,
+                channel, unloadStrategy, counter, reference);
+        ExecutorService executorService = Executors.newFixedThreadPool(5);
+        CountDownLatch latch = new CountDownLatch(5);
+        for (int i = 0; i < 5; i++) {
             executorService.execute(() -> {
                 scheduler.execute();
                 latch.countDown();
@@ -130,18 +144,21 @@ public void testExecuteMoreThenOnceWhenFirstNotDone() throws InterruptedExceptio
         }
         latch.await();
 
-        verify(registry, times(1)).getAvailableBrokersAsync();
+        verify(registry, times(5)).getAvailableBrokersAsync();
     }
 
     @Test(timeOut = 30 * 1000)
     public void testDisableLoadBalancer() {
+        AtomicReference> reference = new AtomicReference<>();
+        UnloadCounter counter = new UnloadCounter();
         LoadManagerContext context = setupContext();
         context.brokerConfiguration().setLoadBalancerEnabled(false);
         ServiceUnitStateChannel channel = mock(ServiceUnitStateChannel.class);
         NamespaceUnloadStrategy unloadStrategy = mock(NamespaceUnloadStrategy.class);
         UnloadManager unloadManager = mock(UnloadManager.class);
-        UnloadScheduler scheduler = new UnloadScheduler(loadManagerExecutor, unloadManager, context, channel, unloadStrategy);
-
+        PulsarService pulsar = mock(PulsarService.class);
+        UnloadScheduler scheduler = new UnloadScheduler(pulsar, loadManagerExecutor, unloadManager, context,
+                channel, unloadStrategy, counter, reference);
         scheduler.execute();
 
         verify(channel, times(0)).isChannelOwnerAsync();
@@ -155,12 +172,16 @@ public void testDisableLoadBalancer() {
 
     @Test(timeOut = 30 * 1000)
     public void testNotChannelOwner() {
+        AtomicReference> reference = new AtomicReference<>();
+        UnloadCounter counter = new UnloadCounter();
         LoadManagerContext context = setupContext();
         context.brokerConfiguration().setLoadBalancerEnabled(false);
         ServiceUnitStateChannel channel = mock(ServiceUnitStateChannel.class);
         NamespaceUnloadStrategy unloadStrategy = mock(NamespaceUnloadStrategy.class);
         UnloadManager unloadManager = mock(UnloadManager.class);
-        UnloadScheduler scheduler = new UnloadScheduler(loadManagerExecutor, unloadManager, context, channel, unloadStrategy);
+        PulsarService pulsar = mock(PulsarService.class);
+        UnloadScheduler scheduler = new UnloadScheduler(pulsar, loadManagerExecutor, unloadManager, context,
+                channel, unloadStrategy, counter, reference);
         doReturn(CompletableFuture.completedFuture(false)).when(channel).isChannelOwnerAsync();
 
         scheduler.execute();

From 13d78dcbdb59907d2541e7ee8fe96101fd441b12 Mon Sep 17 00:00:00 2001
From: Kai Wang 
Date: Sun, 19 Mar 2023 11:16:34 +0800
Subject: [PATCH 124/174] [improve][broker] PIP-192: Make split admin API
 functional (#19773)

Master Issue: https://github.com/apache/pulsar/issues/16691

### Motivation

Raising a PR to implement https://github.com/apache/pulsar/issues/16691, this PR makes the split admin API functional.

### Modifications

* Make the split admin API functional when enabling the `ExtensibleLoadManager`.
* Support more split algorithm.
---
 .../extensions/ExtensibleLoadManagerImpl.java | 60 +++++++++++++++++-
 .../channel/ServiceUnitStateChannelImpl.java  | 34 +++++++++--
 .../channel/ServiceUnitStateData.java         | 17 ++++--
 .../loadbalance/extensions/models/Split.java  |  7 +++
 ...faultNamespaceBundleSplitStrategyImpl.java |  2 +-
 .../broker/namespace/NamespaceService.java    |  9 +--
 .../pulsar/broker/web/PulsarWebResource.java  |  7 ++-
 .../pulsar/common/naming/NamespaceBundle.java |  2 +-
 ...pecifiedPositionsBundleSplitAlgorithm.java |  2 +-
 .../ExtensibleLoadManagerImplTest.java        | 61 +++++++++++++++++++
 .../channel/ServiceUnitStateChannelTest.java  | 11 ++--
 .../extensions/channel/models/SplitTest.java  | 22 ++++++-
 .../scheduler/SplitSchedulerTest.java         |  5 +-
 ...faultNamespaceBundleSplitStrategyTest.java | 17 +++---
 14 files changed, 216 insertions(+), 40 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
index 486c32153589f..fedcad4d009b2 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
@@ -18,10 +18,11 @@
  */
 package org.apache.pulsar.broker.loadbalance.extensions;
 
+import static java.lang.String.format;
 import static org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.Role.Follower;
 import static org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.Role.Leader;
-import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Success;
-import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Admin;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Label.Success;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Admin;
 import com.google.common.annotations.VisibleForTesting;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -56,7 +57,9 @@
 import org.apache.pulsar.broker.loadbalance.extensions.manager.SplitManager;
 import org.apache.pulsar.broker.loadbalance.extensions.manager.UnloadManager;
 import org.apache.pulsar.broker.loadbalance.extensions.models.AssignCounter;
+import org.apache.pulsar.broker.loadbalance.extensions.models.Split;
 import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter;
+import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision;
 import org.apache.pulsar.broker.loadbalance.extensions.models.Unload;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision;
@@ -71,11 +74,15 @@
 import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStoreFactory;
 import org.apache.pulsar.broker.loadbalance.extensions.strategy.BrokerSelectionStrategy;
 import org.apache.pulsar.broker.loadbalance.extensions.strategy.LeastResourceUsageWithWeight;
+import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared;
+import org.apache.pulsar.common.naming.NamespaceBundle;
+import org.apache.pulsar.common.naming.NamespaceBundleSplitAlgorithm;
 import org.apache.pulsar.common.naming.NamespaceName;
 import org.apache.pulsar.common.naming.ServiceUnitId;
 import org.apache.pulsar.common.naming.TopicDomain;
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.stats.Metrics;
+import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap;
 import org.apache.pulsar.metadata.api.coordination.LeaderElectionState;
 
@@ -406,7 +413,7 @@ public CompletableFuture unloadNamespaceBundleAsync(ServiceUnitId bundle,
                     }
                     Unload unload = new Unload(sourceBroker, bundle.toString(), destinationBroker);
                     UnloadDecision unloadDecision =
-                            new UnloadDecision(unload, Success, Admin);
+                            new UnloadDecision(unload, UnloadDecision.Label.Success, UnloadDecision.Reason.Admin);
                     return unloadAsync(unloadDecision,
                             conf.getNamespaceBundleUnloadingTimeoutMs(), TimeUnit.MILLISECONDS);
                 });
@@ -421,6 +428,53 @@ private CompletableFuture unloadAsync(UnloadDecision unloadDecision,
                 .thenRun(() -> unloadCounter.updateUnloadBrokerCount(1));
     }
 
+    public CompletableFuture splitNamespaceBundleAsync(ServiceUnitId bundle,
+                                                             NamespaceBundleSplitAlgorithm splitAlgorithm,
+                                                             List boundaries) {
+        final String namespaceName = LoadManagerShared.getNamespaceNameFromBundleName(bundle.toString());
+        final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle.toString());
+        NamespaceBundle namespaceBundle =
+                pulsar.getNamespaceService().getNamespaceBundleFactory().getBundle(namespaceName, bundleRange);
+        return pulsar.getNamespaceService().getSplitBoundary(namespaceBundle, splitAlgorithm, boundaries)
+                .thenCompose(splitBundlesPair -> {
+                    if (splitBundlesPair == null) {
+                        String msg = format("Bundle %s not found under namespace", namespaceBundle);
+                        log.error(msg);
+                        return FutureUtil.failedFuture(new IllegalStateException(msg));
+                    }
+
+                    return getOwnershipAsync(Optional.empty(), bundle)
+                            .thenCompose(brokerOpt -> {
+                                if (brokerOpt.isEmpty()) {
+                                    String msg = String.format("Namespace bundle: %s is not owned by any broker.",
+                                            bundle);
+                                    log.warn(msg);
+                                    throw new IllegalStateException(msg);
+                                }
+                                String sourceBroker = brokerOpt.get();
+                                SplitDecision splitDecision = new SplitDecision();
+                                List splitBundles = splitBundlesPair.getRight();
+                                Map> splitServiceUnitToDestBroker = new HashMap<>();
+                                splitBundles.forEach(splitBundle -> splitServiceUnitToDestBroker
+                                        .put(splitBundle.getBundleRange(), Optional.empty()));
+                                splitDecision.setSplit(
+                                        new Split(bundle.toString(), sourceBroker, splitServiceUnitToDestBroker));
+                                splitDecision.setLabel(Success);
+                                splitDecision.setReason(Admin);
+                                return splitAsync(splitDecision,
+                                        conf.getNamespaceBundleUnloadingTimeoutMs(), TimeUnit.MILLISECONDS);
+                            });
+                });
+    }
+
+    private CompletableFuture splitAsync(SplitDecision decision,
+                                               long timeout,
+                                               TimeUnit timeoutUnit) {
+        Split split = decision.getSplit();
+        CompletableFuture future = serviceUnitStateChannel.publishSplitEventAsync(split);
+        return splitManager.waitAsync(future, decision.getSplit().serviceUnit(), decision, timeout, timeoutUnit);
+    }
+
     @Override
     public void close() throws PulsarServerException {
         if (!this.started) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
index 791d02649baf1..ca9e55923e799 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
@@ -87,6 +87,7 @@
 import org.apache.pulsar.client.api.TableView;
 import org.apache.pulsar.common.naming.NamespaceBundle;
 import org.apache.pulsar.common.naming.NamespaceBundleFactory;
+import org.apache.pulsar.common.naming.NamespaceBundleSplitAlgorithm;
 import org.apache.pulsar.common.naming.NamespaceBundles;
 import org.apache.pulsar.common.naming.NamespaceName;
 import org.apache.pulsar.common.naming.TopicDomain;
@@ -533,7 +534,8 @@ public CompletableFuture publishSplitEventAsync(Split split) {
         eventCounters.get(eventType).getTotal().incrementAndGet();
         String serviceUnit = split.serviceUnit();
         ServiceUnitStateData next =
-                new ServiceUnitStateData(Splitting, null, split.sourceBroker(), getNextVersionId(serviceUnit));
+                new ServiceUnitStateData(Splitting, null, split.sourceBroker(),
+                        split.splitServiceUnitToDestBroker(), getNextVersionId(serviceUnit));
         return pubAsync(serviceUnit, next).whenComplete((__, ex) -> {
             if (ex != null) {
                 eventCounters.get(eventType).getFailure().incrementAndGet();
@@ -784,22 +786,40 @@ private CompletableFuture closeServiceUnit(String serviceUnit) {
     }
 
     private CompletableFuture splitServiceUnit(String serviceUnit, ServiceUnitStateData data) {
-        // Write the child ownerships to BSC.
+        // Write the child ownerships to channel.
         long startTime = System.nanoTime();
         NamespaceService namespaceService = pulsar.getNamespaceService();
         NamespaceBundleFactory bundleFactory = namespaceService.getNamespaceBundleFactory();
         NamespaceBundle bundle = getNamespaceBundle(serviceUnit);
         CompletableFuture completionFuture = new CompletableFuture<>();
+        Map> bundleToDestBroker = data.splitServiceUnitToDestBroker();
+        List boundaries = null;
+        NamespaceBundleSplitAlgorithm nsBundleSplitAlgorithm =
+                namespaceService.getNamespaceBundleSplitAlgorithmByName(
+                        config.getDefaultNamespaceBundleSplitAlgorithm());
+        if (bundleToDestBroker != null && bundleToDestBroker.size() == 2) {
+            Set boundariesSet = new HashSet<>();
+            String namespace = bundle.getNamespaceObject().toString();
+            bundleToDestBroker.forEach((bundleRange, destBroker) -> {
+                NamespaceBundle subBundle = bundleFactory.getBundle(namespace, bundleRange);
+                boundariesSet.add(subBundle.getKeyRange().lowerEndpoint());
+                boundariesSet.add(subBundle.getKeyRange().upperEndpoint());
+            });
+            boundaries = new ArrayList<>(boundariesSet);
+            nsBundleSplitAlgorithm = NamespaceBundleSplitAlgorithm.SPECIFIED_POSITIONS_DIVIDE_ALGO;
+        }
         final AtomicInteger counter = new AtomicInteger(0);
-        this.splitServiceUnitOnceAndRetry(namespaceService, bundleFactory, bundle, serviceUnit, data,
-                counter, startTime, completionFuture);
+        this.splitServiceUnitOnceAndRetry(namespaceService, bundleFactory, nsBundleSplitAlgorithm,
+                bundle, boundaries, serviceUnit, data, counter, startTime, completionFuture);
         return completionFuture;
     }
 
     @VisibleForTesting
     protected void splitServiceUnitOnceAndRetry(NamespaceService namespaceService,
                                                 NamespaceBundleFactory bundleFactory,
+                                                NamespaceBundleSplitAlgorithm algorithm,
                                                 NamespaceBundle bundle,
+                                                List boundaries,
                                                 String serviceUnit,
                                                 ServiceUnitStateData data,
                                                 AtomicInteger counter,
@@ -807,7 +827,8 @@ protected void splitServiceUnitOnceAndRetry(NamespaceService namespaceService,
                                                 CompletableFuture completionFuture) {
         CompletableFuture> updateFuture = new CompletableFuture<>();
 
-        pulsar.getNamespaceService().getSplitBoundary(bundle, null).thenAccept(splitBundlesPair -> {
+        namespaceService.getSplitBoundary(bundle, algorithm, boundaries)
+                .thenAccept(splitBundlesPair -> {
             // Split and updateNamespaceBundles. Update may fail because of concurrent write to Zookeeper.
             if (splitBundlesPair == null) {
                 String msg = format("Bundle %s not found under namespace", serviceUnit);
@@ -877,7 +898,8 @@ protected void splitServiceUnitOnceAndRetry(NamespaceService namespaceService,
                 log.warn("Failed to update bundle range in metadata store. Retrying {} th / {} limit",
                         counter.get(), NamespaceService.BUNDLE_SPLIT_RETRY_LIMIT, ex);
                 pulsar.getExecutor().schedule(() -> splitServiceUnitOnceAndRetry(namespaceService, bundleFactory,
-                                bundle, serviceUnit, data, counter, startTime, completionFuture), 100, MILLISECONDS);
+                        algorithm, bundle, boundaries, serviceUnit, data, counter, startTime, completionFuture),
+                        100, MILLISECONDS);
             } else if (throwable instanceof IllegalArgumentException) {
                 completionFuture.completeExceptionally(throwable);
             } else {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java
index c26dce83a4434..52c7e27d65033 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java
@@ -18,7 +18,9 @@
  */
 package org.apache.pulsar.broker.loadbalance.extensions.channel;
 
+import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import org.apache.commons.lang3.StringUtils;
 
 /**
@@ -27,7 +29,8 @@
  */
 
 public record ServiceUnitStateData(
-        ServiceUnitState state, String dstBroker, String sourceBroker, boolean force, long timestamp, long versionId) {
+        ServiceUnitState state, String dstBroker, String sourceBroker,
+        Map> splitServiceUnitToDestBroker, boolean force, long timestamp, long versionId) {
 
     public ServiceUnitStateData {
         Objects.requireNonNull(state);
@@ -36,16 +39,22 @@ public record ServiceUnitStateData(
         }
     }
 
+    public ServiceUnitStateData(ServiceUnitState state, String dstBroker, String sourceBroker,
+                                Map> splitServiceUnitToDestBroker, long versionId) {
+        this(state, dstBroker, sourceBroker, splitServiceUnitToDestBroker, false,
+                System.currentTimeMillis(), versionId);
+    }
+
     public ServiceUnitStateData(ServiceUnitState state, String dstBroker, String sourceBroker, long versionId) {
-        this(state, dstBroker, sourceBroker, false, System.currentTimeMillis(), versionId);
+        this(state, dstBroker, sourceBroker, null, false, System.currentTimeMillis(), versionId);
     }
 
     public ServiceUnitStateData(ServiceUnitState state, String dstBroker, long versionId) {
-        this(state, dstBroker, null, false, System.currentTimeMillis(), versionId);
+        this(state, dstBroker, null, null, false, System.currentTimeMillis(), versionId);
     }
 
     public ServiceUnitStateData(ServiceUnitState state, String dstBroker, boolean force, long versionId) {
-        this(state, dstBroker, null, force, System.currentTimeMillis(), versionId);
+        this(state, dstBroker, null, null, force, System.currentTimeMillis(), versionId);
     }
 
     public static ServiceUnitState state(ServiceUnitStateData data) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/Split.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/Split.java
index fcec8b0ae5541..ac9a36e6dbfd3 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/Split.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/Split.java
@@ -30,5 +30,12 @@ public record Split(
 
     public Split {
         Objects.requireNonNull(serviceUnit);
+        if (splitServiceUnitToDestBroker != null && splitServiceUnitToDestBroker.size() != 2) {
+            throw new IllegalArgumentException("Split service unit should be split into 2 service units.");
+        }
+    }
+
+    public Split(String serviceUnit, String sourceBroker) {
+        this(serviceUnit, sourceBroker, null);
     }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java
index e572fd4161bdb..da04a287f1ac5 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java
@@ -138,7 +138,7 @@ public Set findBundlesToSplit(LoadManagerContext context, PulsarS
                                     maxBundleBandwidth / LoadManagerShared.MIBI);
                         }
                         var decision = new SplitDecision();
-                        decision.setSplit(new Split(bundle, context.brokerRegistry().getBrokerId(), new HashMap<>()));
+                        decision.setSplit(new Split(bundle, context.brokerRegistry().getBrokerId()));
                         decision.succeed(reason);
                         decisionCache.add(decision);
                         int bundleNum = namespaceBundleCount.getOrDefault(namespace, 0);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
index 899539e1db6a7..de18d50f3e144 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
@@ -822,7 +822,10 @@ public boolean isNamespaceBundleDisabled(NamespaceBundle bundle) throws Exceptio
     public CompletableFuture splitAndOwnBundle(NamespaceBundle bundle, boolean unload,
                                                      NamespaceBundleSplitAlgorithm splitAlgorithm,
                                                      List boundaries) {
-
+        if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) {
+            return ExtensibleLoadManagerImpl.get(loadManager.get())
+                    .splitNamespaceBundleAsync(bundle, splitAlgorithm, boundaries);
+        }
         final CompletableFuture unloadFuture = new CompletableFuture<>();
         final AtomicInteger counter = new AtomicInteger(BUNDLE_SPLIT_RETRY_LIMIT);
         splitAndOwnBundleOnceAndRetry(bundle, unload, counter, unloadFuture, splitAlgorithm, boundaries);
@@ -963,10 +966,8 @@ void splitAndOwnBundleOnceAndRetry(NamespaceBundle bundle,
      * @return A pair, left is target namespace bundle, right is split bundles.
      */
     public CompletableFuture>> getSplitBoundary(
-            NamespaceBundle bundle, List boundaries) {
+            NamespaceBundle bundle, NamespaceBundleSplitAlgorithm nsBundleSplitAlgorithm, List boundaries) {
         BundleSplitOption bundleSplitOption = getBundleSplitOption(bundle, boundaries, config);
-        NamespaceBundleSplitAlgorithm nsBundleSplitAlgorithm =
-                getNamespaceBundleSplitAlgorithmByName(config.getDefaultNamespaceBundleSplitAlgorithm());
         CompletableFuture> splitBoundary =
                 nsBundleSplitAlgorithm.getSplitBoundary(bundleSplitOption);
         return splitBoundary.thenCompose(splitBoundaries -> {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java
index 321a127ad97bd..c5713ebddaa2f 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java
@@ -636,9 +636,6 @@ protected CompletableFuture validateNamespaceBundleOwnershipAsy
         } catch (WebApplicationException wae) {
             return CompletableFuture.failedFuture(wae);
         }
-        if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config())) {
-            return CompletableFuture.completedFuture(nsBundle);
-        }
         return validateBundleOwnershipAsync(nsBundle, authoritative, readOnly)
                 .thenApply(__ -> nsBundle);
     }
@@ -718,6 +715,10 @@ public CompletableFuture validateBundleOwnershipAsync(NamespaceBundle bund
                         throw new RestException(Status.PRECONDITION_FAILED,
                                 "Failed to find ownership for ServiceUnit:" + bundle.toString());
                     }
+                    // If the load manager is extensible load manager, we don't need check the authoritative.
+                    if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config())) {
+                        return CompletableFuture.completedFuture(null);
+                    }
                     return nsService.isServiceUnitOwnedAsync(bundle)
                             .thenAccept(owned -> {
                                 if (!owned) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundle.java b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundle.java
index 0b7e09fee2cec..78a80e077e42a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundle.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundle.java
@@ -127,7 +127,7 @@ private static String getKey(NamespaceName nsname, Range keyRange) {
         return String.format("%s/0x%08x_0x%08x", nsname, keyRange.lowerEndpoint(), keyRange.upperEndpoint());
     }
 
-    Range getKeyRange() {
+    public Range getKeyRange() {
         return this.keyRange;
     }
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/SpecifiedPositionsBundleSplitAlgorithm.java b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/SpecifiedPositionsBundleSplitAlgorithm.java
index 6c341f185ca60..1fd9a85cfa782 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/SpecifiedPositionsBundleSplitAlgorithm.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/SpecifiedPositionsBundleSplitAlgorithm.java
@@ -54,4 +54,4 @@ public CompletableFuture> getSplitBoundary(BundleSplitOption bundleSp
             return CompletableFuture.completedFuture(splitBoundaries);
         });
     }
-}
\ No newline at end of file
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
index 354a27a63c02a..d977e63330586 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
@@ -95,6 +95,7 @@
 import org.apache.pulsar.common.naming.NamespaceName;
 import org.apache.pulsar.common.naming.ServiceUnitId;
 import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.policies.data.BundlesData;
 import org.apache.pulsar.common.policies.data.ClusterData;
 import org.apache.pulsar.common.policies.data.Policies;
 import org.apache.pulsar.common.policies.data.TenantInfo;
@@ -372,6 +373,66 @@ private void checkOwnershipState(String broker, NamespaceBundle bundle)
         assertFalse(otherLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get());
     }
 
+    @Test(timeOut = 30 * 1000)
+    public void testSplitBundleAdminAPI() throws Exception {
+        String namespace = "public/default";
+        String topic = "persistent://" + namespace + "/test-split";
+        admin.topics().createPartitionedTopic(topic, 10);
+        BundlesData bundles = admin.namespaces().getBundles(namespace);
+        int numBundles = bundles.getNumBundles();
+        var bundleRanges = bundles.getBoundaries().stream().map(Long::decode).sorted().toList();
+
+        String firstBundle = bundleRanges.get(0) + "_" + bundleRanges.get(1);
+
+        long mid = bundleRanges.get(0) + (bundleRanges.get(1) - bundleRanges.get(0)) / 2;
+
+        admin.namespaces().splitNamespaceBundle(namespace, firstBundle, true, null);
+
+        BundlesData bundlesData = admin.namespaces().getBundles(namespace);
+        assertEquals(bundlesData.getNumBundles(), numBundles + 1);
+        String lowBundle = String.format("0x%08x", bundleRanges.get(0));
+        String midBundle = String.format("0x%08x", mid);
+        String highBundle = String.format("0x%08x", bundleRanges.get(1));
+        assertTrue(bundlesData.getBoundaries().contains(lowBundle));
+        assertTrue(bundlesData.getBoundaries().contains(midBundle));
+        assertTrue(bundlesData.getBoundaries().contains(highBundle));
+
+        // Test split bundle with invalid bundle range.
+        try {
+            admin.namespaces().splitNamespaceBundle(namespace, "invalid", true, null);
+            fail();
+        } catch (PulsarAdminException ex) {
+            assertTrue(ex.getMessage().contains("Invalid bundle range"));
+        }
+    }
+
+    @Test(timeOut = 30 * 1000)
+    public void testSplitBundleWithSpecificPositionAdminAPI() throws Exception {
+        String namespace = "public/default";
+        String topic = "persistent://" + namespace + "/test-split-with-specific-position";
+        admin.topics().createPartitionedTopic(topic, 10);
+        BundlesData bundles = admin.namespaces().getBundles(namespace);
+        int numBundles = bundles.getNumBundles();
+
+        var bundleRanges = bundles.getBoundaries().stream().map(Long::decode).sorted().toList();
+
+        String firstBundle = bundleRanges.get(0) + "_" + bundleRanges.get(1);
+
+        long mid = bundleRanges.get(0) + (bundleRanges.get(1) - bundleRanges.get(0)) / 2;
+        long splitPosition = mid + 100;
+
+        admin.namespaces().splitNamespaceBundle(namespace, firstBundle, true,
+                "specified_positions_divide", List.of(bundleRanges.get(0), bundleRanges.get(1), splitPosition));
+
+        BundlesData bundlesData = admin.namespaces().getBundles(namespace);
+        assertEquals(bundlesData.getNumBundles(), numBundles + 1);
+        String lowBundle = String.format("0x%08x", bundleRanges.get(0));
+        String midBundle = String.format("0x%08x", splitPosition);
+        String highBundle = String.format("0x%08x", bundleRanges.get(1));
+        assertTrue(bundlesData.getBoundaries().contains(lowBundle));
+        assertTrue(bundlesData.getBoundaries().contains(midBundle));
+        assertTrue(bundlesData.getBoundaries().contains(highBundle));
+    }
 
     @Test
     public void testMoreThenOneFilter() throws Exception {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
index e0bd69fce64c7..c05c8ac741565 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
@@ -54,7 +54,6 @@
 import static org.testng.Assert.assertTrue;
 import static org.testng.AssertJUnit.assertNotNull;
 import java.lang.reflect.Field;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -297,7 +296,7 @@ private int validateChannelStart(ServiceUnitStateChannelImpl channel)
             }
         }
         try {
-            channel.publishSplitEventAsync(new Split(bundle, lookupServiceAddress1, Map.of()))
+            channel.publishSplitEventAsync(new Split(bundle, lookupServiceAddress1))
                     .get(2, TimeUnit.SECONDS);
         } catch (ExecutionException e) {
             if (e.getCause() instanceof IllegalStateException) {
@@ -554,11 +553,15 @@ public void splitAndRetryTest() throws Exception {
             }
             // Call the real method
             reset(namespaceService);
+            doReturn(CompletableFuture.completedFuture(List.of("test-topic-1", "test-topic-2")))
+                    .when(namespaceService).getOwnedTopicListForNamespaceBundle(any());
             return future;
         }).when(namespaceService).updateNamespaceBundles(any(), any());
         doReturn(namespaceService).when(pulsar1).getNamespaceService();
+        doReturn(CompletableFuture.completedFuture(List.of("test-topic-1", "test-topic-2")))
+                .when(namespaceService).getOwnedTopicListForNamespaceBundle(any());
 
-        Split split = new Split(bundle, ownerAddr1.get(), new HashMap<>());
+        Split split = new Split(bundle, ownerAddr1.get());
         channel1.publishSplitEventAsync(split);
 
         waitUntilState(channel1, bundle, Deleted);
@@ -570,7 +573,7 @@ public void splitAndRetryTest() throws Exception {
         validateEventCounters(channel2, 0, 0, 0, 0, 0, 0);
         // Verify the retry count
         verify(((ServiceUnitStateChannelImpl) channel1), times(badVersionExceptionCount + 1))
-                .splitServiceUnitOnceAndRetry(any(), any(), any(), any(), any(), any(), anyLong(), any());
+                .splitServiceUnitOnceAndRetry(any(), any(), any(), any(), any(), any(), any(), any(), anyLong(), any());
 
         // Assert child bundle ownerships in the channels.
         String childBundle1 = "public/default/0x7fffffff_0xffffffff";
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/models/SplitTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/models/SplitTest.java
index 67c9555c88f01..1e23ea74dca5e 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/models/SplitTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/models/SplitTest.java
@@ -19,6 +19,7 @@
 package org.apache.pulsar.broker.loadbalance.extensions.channel.models;
 
 import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNull;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Optional;
@@ -32,6 +33,7 @@ public class SplitTest {
     public void testConstructor() {
         Map> map = new HashMap<>();
         map.put("C", Optional.of("test"));
+        map.put("D", Optional.of("test"));
 
         Split split = new Split("A", "B", map);
         assertEquals(split.serviceUnit(), "A");
@@ -44,4 +46,22 @@ public void testNullBundle() {
         new Split(null, "A", Map.of());
     }
 
-}
\ No newline at end of file
+
+    @Test(expectedExceptions = IllegalArgumentException.class)
+    public void testInvalidSplitServiceUnitToDestBroker() {
+        Map> map = new HashMap<>();
+        map.put("C", Optional.of("test"));
+        map.put("D", Optional.of("test"));
+        map.put("E", Optional.of("test"));
+        new Split("A", "B", map);
+    }
+
+    @Test
+    public void testNullSplitServiceUnitToDestBroker() {
+        var split = new Split("A", "B");
+        assertEquals(split.serviceUnit(), "A");
+        assertEquals(split.sourceBroker(), "B");
+        assertNull(split.splitServiceUnitToDestBroker());
+    }
+
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitSchedulerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitSchedulerTest.java
index 7988aa413366f..f7726f8e7a1aa 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitSchedulerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitSchedulerTest.java
@@ -27,7 +27,6 @@
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.testng.Assert.assertEquals;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
@@ -78,11 +77,11 @@ public void setUp() {
         doReturn(CompletableFuture.completedFuture(null)).when(channel).publishSplitEventAsync(any());
 
         decision1 = new SplitDecision();
-        decision1.setSplit(new Split(bundle1, broker, new HashMap<>()));
+        decision1.setSplit(new Split(bundle1, broker));
         decision1.succeed(SplitDecision.Reason.MsgRate);
 
         decision2 = new SplitDecision();
-        decision2.setSplit(new Split(bundle2, broker, new HashMap<>()));
+        decision2.setSplit(new Split(bundle2, broker));
         decision2.succeed(SplitDecision.Reason.Sessions);
         Set decisions = Set.of(decision1, decision2);
         doReturn(decisions).when(strategy).findBundlesToSplit(any(), any());
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java
index 71606bb85a3fe..deff6fb00fafb 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java
@@ -28,7 +28,6 @@
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.testng.Assert.assertEquals;
-import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Set;
@@ -172,14 +171,14 @@ public void testMaxMsgRate() {
             var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
             if (i == threshold) {
                 SplitDecision decision1 = new SplitDecision();
-                decision1.setSplit(new Split(bundle1, broker, new HashMap<>()));
+                decision1.setSplit(new Split(bundle1, broker));
                 decision1.succeed(SplitDecision.Reason.MsgRate);
 
                 assertEquals(actual, Set.of(decision1));
                 verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
             } else if (i == threshold + 1) {
                 SplitDecision decision1 = new SplitDecision();
-                decision1.setSplit(new Split(bundle2, broker, new HashMap<>()));
+                decision1.setSplit(new Split(bundle2, broker));
                 decision1.succeed(SplitDecision.Reason.MsgRate);
 
                 assertEquals(actual, Set.of(decision1));
@@ -200,14 +199,14 @@ public void testMaxTopics() {
             var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
             if (i == threshold) {
                 SplitDecision decision1 = new SplitDecision();
-                decision1.setSplit(new Split(bundle1, broker, new HashMap<>()));
+                decision1.setSplit(new Split(bundle1, broker));
                 decision1.succeed(SplitDecision.Reason.Topics);
 
                 assertEquals(actual, Set.of(decision1));
                 verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
             } else if (i == threshold + 1) {
                 SplitDecision decision1 = new SplitDecision();
-                decision1.setSplit(new Split(bundle2, broker, new HashMap<>()));
+                decision1.setSplit(new Split(bundle2, broker));
                 decision1.succeed(SplitDecision.Reason.Topics);
 
                 assertEquals(actual, Set.of(decision1));
@@ -231,14 +230,14 @@ public void testMaxSessions() {
             var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
             if (i == threshold) {
                 SplitDecision decision1 = new SplitDecision();
-                decision1.setSplit(new Split(bundle1, broker, new HashMap<>()));
+                decision1.setSplit(new Split(bundle1, broker));
                 decision1.succeed(SplitDecision.Reason.Sessions);
 
                 assertEquals(actual, Set.of(decision1));
                 verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
             } else if (i == threshold + 1) {
                 SplitDecision decision1 = new SplitDecision();
-                decision1.setSplit(new Split(bundle2, broker, new HashMap<>()));
+                decision1.setSplit(new Split(bundle2, broker));
                 decision1.succeed(SplitDecision.Reason.Sessions);
 
                 assertEquals(actual, Set.of(decision1));
@@ -262,14 +261,14 @@ public void testMaxBandwidthMbytes() {
             var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
             if (i == threshold) {
                 SplitDecision decision1 = new SplitDecision();
-                decision1.setSplit(new Split(bundle1, broker, new HashMap<>()));
+                decision1.setSplit(new Split(bundle1, broker));
                 decision1.succeed(SplitDecision.Reason.Bandwidth);
 
                 assertEquals(actual, Set.of(decision1));
                 verify(counter, times(0)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
             } else if (i == threshold + 1) {
                 SplitDecision decision1 = new SplitDecision();
-                decision1.setSplit(new Split(bundle2, broker, new HashMap<>()));
+                decision1.setSplit(new Split(bundle2, broker));
                 decision1.succeed(SplitDecision.Reason.Bandwidth);
 
                 assertEquals(actual, Set.of(decision1));

From f20dc93bc7d3defcc824857a558f2f78726f13a1 Mon Sep 17 00:00:00 2001
From: tison 
Date: Sun, 19 Mar 2023 12:13:10 +0800
Subject: [PATCH 125/174] [improve][misc] Add idea icon (#19855)

Signed-off-by: tison 
---
 .github/changes-filter.yaml |  6 +++---
 .idea/icon.svg              | 33 +++++++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+), 3 deletions(-)
 create mode 100644 .idea/icon.svg

diff --git a/.github/changes-filter.yaml b/.github/changes-filter.yaml
index 3ec2fc22946da..be6faa957887d 100644
--- a/.github/changes-filter.yaml
+++ b/.github/changes-filter.yaml
@@ -3,13 +3,13 @@
 all:
   - '**'
 docs:
-  - 'site2/**'
-  - 'deployment/**'
-  - '.asf.yaml'
   - '*.md'
   - '**/*.md'
+  - '.asf.yaml'
   - '.github/changes-filter.yaml'
   - '.github/ISSUE_TEMPLATE/**'
+  - '.idea/**'
+  - 'deployment/**'
   - 'wiki/**'
 tests:
   - added|modified: '**/src/test/java/**/*.java'
diff --git a/.idea/icon.svg b/.idea/icon.svg
new file mode 100644
index 0000000000000..bf9b232def4a4
--- /dev/null
+++ b/.idea/icon.svg
@@ -0,0 +1,33 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file

From 3c8ab7bc1fe1851ca24b0f12063cf2442276f656 Mon Sep 17 00:00:00 2001
From: ran 
Date: Mon, 20 Mar 2023 10:16:59 +0800
Subject: [PATCH 126/174] [fix][test] Fix flaky MetadataStoreStatsTest 
 (#19829)

---
 .../service/ManagedLedgerCompressionTest.java |  2 +-
 .../broker/stats/MetadataStoreStatsTest.java  | 81 +++++++++++++------
 2 files changed, 58 insertions(+), 25 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ManagedLedgerCompressionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ManagedLedgerCompressionTest.java
index 4991d0a75363d..c13c8bd9fdc94 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ManagedLedgerCompressionTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ManagedLedgerCompressionTest.java
@@ -50,7 +50,7 @@ protected void cleanup() throws Exception {
         super.internalCleanup();
     }
 
-    @Test(timeOut = 1000 * 20)
+    @Test(timeOut = 1000 * 30)
     public void testRestartBrokerEnableManagedLedgerInfoCompression() throws Exception {
         String topic = newTopicName();
         @Cleanup
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/MetadataStoreStatsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/MetadataStoreStatsTest.java
index f4deddf0cec9e..8ae0242c6232a 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/MetadataStoreStatsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/MetadataStoreStatsTest.java
@@ -21,9 +21,13 @@
 import com.google.common.collect.Multimap;
 import java.io.ByteArrayOutputStream;
 import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 import lombok.Cleanup;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.authentication.AuthenticationProviderToken;
 import org.apache.pulsar.broker.service.BrokerTestBase;
@@ -107,13 +111,17 @@ public void testMetadataStoreStats() throws Exception {
         Assert.assertTrue(opsLatency.size() > 1, metricsDebugMessage);
         Assert.assertTrue(putBytes.size() > 1, metricsDebugMessage);
 
+        Set expectedMetadataStoreName = new HashSet<>();
+        expectedMetadataStoreName.add(MetadataStoreConfig.METADATA_STORE);
+        expectedMetadataStoreName.add(MetadataStoreConfig.CONFIGURATION_METADATA_STORE);
+
+        AtomicInteger matchCount = new AtomicInteger(0);
         for (PrometheusMetricsTest.Metric m : opsLatency) {
             Assert.assertEquals(m.tags.get("cluster"), "test", metricsDebugMessage);
             String metadataStoreName = m.tags.get("name");
-            Assert.assertNotNull(metadataStoreName, metricsDebugMessage);
-            Assert.assertTrue(metadataStoreName.equals(MetadataStoreConfig.METADATA_STORE)
-                    || metadataStoreName.equals(MetadataStoreConfig.CONFIGURATION_METADATA_STORE)
-                    || metadataStoreName.equals(MetadataStoreConfig.STATE_METADATA_STORE), metricsDebugMessage);
+            if (!isExpectedLabel(metadataStoreName, expectedMetadataStoreName, matchCount)) {
+                continue;
+            }
             Assert.assertNotNull(m.tags.get("status"), metricsDebugMessage);
 
             if (m.tags.get("status").equals("success")) {
@@ -138,15 +146,19 @@ public void testMetadataStoreStats() throws Exception {
                 }
             }
         }
+        // Because the combination quantity between status(success, fail) and type(get, del, put) is 6.
+        Assert.assertEquals(matchCount.get(), expectedMetadataStoreName.size() * 6);
+
+        matchCount = new AtomicInteger(0);
         for (PrometheusMetricsTest.Metric m : putBytes) {
             Assert.assertEquals(m.tags.get("cluster"), "test", metricsDebugMessage);
             String metadataStoreName = m.tags.get("name");
-            Assert.assertNotNull(metadataStoreName, metricsDebugMessage);
-            Assert.assertTrue(metadataStoreName.equals(MetadataStoreConfig.METADATA_STORE)
-                    || metadataStoreName.equals(MetadataStoreConfig.CONFIGURATION_METADATA_STORE)
-                    || metadataStoreName.equals(MetadataStoreConfig.STATE_METADATA_STORE), metricsDebugMessage);
+            if (!isExpectedLabel(metadataStoreName, expectedMetadataStoreName, matchCount)) {
+                continue;
+            }
             Assert.assertTrue(m.value >= 0, metricsDebugMessage);
         }
+        Assert.assertEquals(matchCount.get(), expectedMetadataStoreName.size());
     }
 
     @Test
@@ -193,43 +205,64 @@ public void testBatchMetadataStoreMetrics() throws Exception {
         Assert.assertTrue(batchExecuteTime.size() > 0, metricsDebugMessage);
         Assert.assertTrue(opsPerBatch.size() > 0, metricsDebugMessage);
 
+        Set expectedMetadataStoreName = new HashSet<>();
+        expectedMetadataStoreName.add(MetadataStoreConfig.METADATA_STORE);
+        expectedMetadataStoreName.add(MetadataStoreConfig.CONFIGURATION_METADATA_STORE);
+
+        AtomicInteger matchCount = new AtomicInteger(0);
         for (PrometheusMetricsTest.Metric m : executorQueueSize) {
             Assert.assertEquals(m.tags.get("cluster"), "test", metricsDebugMessage);
             String metadataStoreName = m.tags.get("name");
-            Assert.assertNotNull(metadataStoreName, metricsDebugMessage);
-            Assert.assertTrue(metadataStoreName.equals(MetadataStoreConfig.METADATA_STORE)
-                    || metadataStoreName.equals(MetadataStoreConfig.CONFIGURATION_METADATA_STORE)
-                    || metadataStoreName.equals(MetadataStoreConfig.STATE_METADATA_STORE), metricsDebugMessage);
+            if (isExpectedLabel(metadataStoreName, expectedMetadataStoreName, matchCount)) {
+                continue;
+            }
             Assert.assertTrue(m.value >= 0, metricsDebugMessage);
         }
+        Assert.assertEquals(matchCount.get(), expectedMetadataStoreName.size());
+
+        matchCount = new AtomicInteger(0);
         for (PrometheusMetricsTest.Metric m : opsWaiting) {
             Assert.assertEquals(m.tags.get("cluster"), "test", metricsDebugMessage);
             String metadataStoreName = m.tags.get("name");
-            Assert.assertNotNull(metadataStoreName, metricsDebugMessage);
-            Assert.assertTrue(metadataStoreName.equals(MetadataStoreConfig.METADATA_STORE)
-                    || metadataStoreName.equals(MetadataStoreConfig.CONFIGURATION_METADATA_STORE)
-                    || metadataStoreName.equals(MetadataStoreConfig.STATE_METADATA_STORE), metricsDebugMessage);
+            if (isExpectedLabel(metadataStoreName, expectedMetadataStoreName, matchCount)) {
+                continue;
+            }
             Assert.assertTrue(m.value >= 0, metricsDebugMessage);
         }
+        Assert.assertEquals(matchCount.get(), expectedMetadataStoreName.size());
 
+        matchCount = new AtomicInteger(0);
         for (PrometheusMetricsTest.Metric m : batchExecuteTime) {
             Assert.assertEquals(m.tags.get("cluster"), "test", metricsDebugMessage);
             String metadataStoreName = m.tags.get("name");
-            Assert.assertNotNull(metadataStoreName, metricsDebugMessage);
-            Assert.assertTrue(metadataStoreName.equals(MetadataStoreConfig.METADATA_STORE)
-                    || metadataStoreName.equals(MetadataStoreConfig.CONFIGURATION_METADATA_STORE)
-                    || metadataStoreName.equals(MetadataStoreConfig.STATE_METADATA_STORE), metricsDebugMessage);
+            if (isExpectedLabel(metadataStoreName, expectedMetadataStoreName, matchCount)) {
+                continue;
+            }
             Assert.assertTrue(m.value >= 0, metricsDebugMessage);
         }
+        Assert.assertEquals(matchCount.get(), expectedMetadataStoreName.size());
 
+        matchCount = new AtomicInteger(0);
         for (PrometheusMetricsTest.Metric m : opsPerBatch) {
             Assert.assertEquals(m.tags.get("cluster"), "test", metricsDebugMessage);
             String metadataStoreName = m.tags.get("name");
-            Assert.assertNotNull(metadataStoreName, metricsDebugMessage);
-            Assert.assertTrue(metadataStoreName.equals(MetadataStoreConfig.METADATA_STORE)
-                    || metadataStoreName.equals(MetadataStoreConfig.CONFIGURATION_METADATA_STORE)
-                    || metadataStoreName.equals(MetadataStoreConfig.STATE_METADATA_STORE), metricsDebugMessage);
+            if (isExpectedLabel(metadataStoreName, expectedMetadataStoreName, matchCount)) {
+                continue;
+            }
             Assert.assertTrue(m.value >= 0, metricsDebugMessage);
         }
+        Assert.assertEquals(matchCount.get(), expectedMetadataStoreName.size());
     }
+
+    private boolean isExpectedLabel(String metadataStoreName, Set expectedLabel,
+                                    AtomicInteger expectedLabelCount) {
+        if (StringUtils.isEmpty(metadataStoreName)
+                || !expectedLabel.contains(metadataStoreName)) {
+            return false;
+        } else {
+            expectedLabelCount.incrementAndGet();
+            return true;
+        }
+    }
+
 }

From 7f5ba2bcbca2a712d11b02d3e2ef4dc2f3052a92 Mon Sep 17 00:00:00 2001
From: Zixuan Liu 
Date: Mon, 20 Mar 2023 10:29:50 +0800
Subject: [PATCH 127/174] [fix][broker] Fix create cluster with empty url
 (#19762)

---
 .../broker/admin/AdminApiClusterTest.java     | 34 +++++++++++++++++++
 .../pulsar/common/util/URIPreconditions.java  |  2 +-
 .../common/util/URIPreconditionsTest.java     |  2 ++
 3 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiClusterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiClusterTest.java
index c37cc234b1213..28ebe39e0aa39 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiClusterTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiClusterTest.java
@@ -22,6 +22,8 @@
 import static org.testng.Assert.assertNotNull;
 import static org.testng.Assert.assertThrows;
 import static org.testng.Assert.fail;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Set;
 import java.util.UUID;
 import lombok.extern.slf4j.Slf4j;
@@ -107,4 +109,36 @@ public void testDeleteExistFailureDomain() throws PulsarAdminException {
 
         admin.clusters().deleteFailureDomain(CLUSTER, domainName);
     }
+
+    @Test
+    public void testCreateCluster() throws PulsarAdminException {
+        List clusterDataList = new ArrayList<>();
+        clusterDataList.add(ClusterData.builder()
+                .serviceUrl("http://pulsar.app:8080")
+                .serviceUrlTls("")
+                .brokerServiceUrl("pulsar://pulsar.app:6650")
+                .brokerServiceUrlTls("")
+                .build());
+        clusterDataList.add(ClusterData.builder()
+                .serviceUrl("")
+                .serviceUrlTls("https://pulsar.app:8443")
+                .brokerServiceUrl("")
+                .brokerServiceUrlTls("pulsar+ssl://pulsar.app:6651")
+                .build());
+        clusterDataList.add(ClusterData.builder()
+                .serviceUrl("")
+                .serviceUrlTls("")
+                .brokerServiceUrl("")
+                .brokerServiceUrlTls("")
+                .build());
+        clusterDataList.add(ClusterData.builder()
+                .serviceUrl(null)
+                .serviceUrlTls(null)
+                .brokerServiceUrl(null)
+                .brokerServiceUrlTls(null)
+                .build());
+        for (int i = 0; i < clusterDataList.size(); i++) {
+            admin.clusters().createCluster("cluster-test-" + i, clusterDataList.get(i));
+        }
+    }
 }
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/URIPreconditions.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/URIPreconditions.java
index f68ed5e41a430..ea7fae2fdf0e9 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/URIPreconditions.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/URIPreconditions.java
@@ -67,7 +67,7 @@ public static void checkURIIfPresent(@Nullable String uri,
     public static void checkURIIfPresent(@Nullable String uri,
                                          @Nonnull Predicate predicate,
                                          @Nullable String errorMessage) throws IllegalArgumentException {
-        if (uri == null) {
+        if (uri == null || uri.length() == 0) {
             return;
         }
         checkURI(uri, predicate, errorMessage);
diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/URIPreconditionsTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/URIPreconditionsTest.java
index d5809e8fdd0d3..f5b9e454e081d 100644
--- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/URIPreconditionsTest.java
+++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/URIPreconditionsTest.java
@@ -30,6 +30,7 @@ public void testCheckURI() {
         // normal
         checkURI("http://pulsar.apache.org", uri -> true);
         checkURI("http://pulsar.apache.org", uri -> Objects.equals(uri.getScheme(), "http"));
+        checkURI("", uri -> true);
         // illegal
         try {
             checkURI("pulsar.apache.org", uri -> Objects.equals(uri.getScheme(), "http"));
@@ -48,6 +49,7 @@ public void testCheckURI() {
     @Test
     public void testCheckURIIfPresent() {
         checkURIIfPresent(null, uri -> false);
+        checkURIIfPresent("", uri -> false);
         checkURIIfPresent("http://pulsar.apache.org", uri -> true);
         try {
             checkURIIfPresent("http/pulsar.apache.org", uri -> uri.getScheme() != null, "Error");

From 2637bda91d46f443762b22ac9963935725af2c8a Mon Sep 17 00:00:00 2001
From: Qiang Zhao 
Date: Mon, 20 Mar 2023 16:39:09 +0800
Subject: [PATCH 128/174] [fix][meta] Follow up #19817, Fix race condition
 between ResourceLock update and invalidation (#19844)

---
 .../apache/pulsar/common/util/FutureUtil.java |  35 ++++++
 .../pulsar/common/util/FutureUtilTest.java    |  72 ++++++++++++
 .../coordination/impl/LockManagerImpl.java    |   2 +-
 .../coordination/impl/ResourceLockImpl.java   | 106 +++++++-----------
 4 files changed, 147 insertions(+), 68 deletions(-)

diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java
index 162ef1e52ffd6..531769a1e452b 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java
@@ -22,6 +22,7 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
@@ -35,6 +36,7 @@
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
+import javax.annotation.concurrent.ThreadSafe;
 
 /**
  * This class is aimed at simplifying work with {@code CompletableFuture}.
@@ -166,6 +168,39 @@ public static Throwable unwrapCompletionException(Throwable ex) {
         }
     }
 
+    @ThreadSafe
+    public static class Sequencer {
+        private CompletableFuture sequencerFuture = CompletableFuture.completedFuture(null);
+        private final boolean allowExceptionBreakChain;
+
+        public Sequencer(boolean allowExceptionBreakChain) {
+            this.allowExceptionBreakChain = allowExceptionBreakChain;
+        }
+
+        public static  Sequencer create(boolean allowExceptionBreakChain) {
+            return new Sequencer<>(allowExceptionBreakChain);
+        }
+        public static  Sequencer create() {
+            return new Sequencer<>(false);
+        }
+
+        /**
+         * @throws NullPointerException NPE when param is null
+         */
+        public synchronized CompletableFuture sequential(Supplier> newTask) {
+            Objects.requireNonNull(newTask);
+            if (sequencerFuture.isDone()) {
+                if (sequencerFuture.isCompletedExceptionally() && allowExceptionBreakChain) {
+                    return sequencerFuture;
+                }
+                return sequencerFuture = newTask.get();
+            }
+            return sequencerFuture = allowExceptionBreakChain
+                    ? sequencerFuture.thenCompose(__ -> newTask.get())
+                    : sequencerFuture.exceptionally(ex -> null).thenCompose(__ -> newTask.get());
+        }
+    }
+
     /**
      * Creates a new {@link CompletableFuture} instance with timeout handling.
      *
diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/FutureUtilTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/FutureUtilTest.java
index 856ec49da6e9d..6df4494edf886 100644
--- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/FutureUtilTest.java
+++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/FutureUtilTest.java
@@ -21,6 +21,7 @@
 import java.io.PrintWriter;
 import java.io.StringWriter;
 import java.time.Duration;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
@@ -32,6 +33,7 @@
 import lombok.Cleanup;
 import org.assertj.core.util.Lists;
 import org.awaitility.Awaitility;
+import org.testng.Assert;
 import org.testng.annotations.Test;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertFalse;
@@ -178,4 +180,74 @@ public void testWaitForAny() {
             assertTrue(ex.getCause() instanceof RuntimeException);
         }
     }
+
+    @Test
+    public void testSequencer() {
+        int concurrentNum = 1000;
+        final ScheduledExecutorService executor = Executors.newScheduledThreadPool(concurrentNum);
+        final FutureUtil.Sequencer sequencer = FutureUtil.Sequencer.create();
+        // normal case -- allowExceptionBreakChain=false
+        final List list = new ArrayList<>();
+        final List> futures = new ArrayList<>();
+        for (int i = 0; i < concurrentNum; i++) {
+            int finalI = i;
+            futures.add(sequencer.sequential(() -> CompletableFuture.runAsync(() -> {
+                list.add(finalI);
+            }, executor)));
+        }
+        FutureUtil.waitForAll(futures).join();
+        for (int i = 0; i < list.size(); i++) {
+            Assert.assertEquals(list.get(i), (Integer) i);
+        }
+
+        // exception case -- allowExceptionBreakChain=false
+        final List list2 = new ArrayList<>();
+        final List> futures2 = new ArrayList<>();
+        for (int i = 0; i < concurrentNum; i++) {
+            int finalI = i;
+            futures2.add(sequencer.sequential(() -> CompletableFuture.runAsync(() -> {
+                if (finalI == 2) {
+                    throw new IllegalStateException();
+                }
+                list2.add(finalI);
+            }, executor)));
+        }
+        try {
+            FutureUtil.waitForAll(futures2).join();
+        } catch (Throwable ignore) {
+
+        }
+        for (int i = 0; i < concurrentNum - 1; i++) {
+            if (i >= 2) {
+                Assert.assertEquals(list2.get(i), i + 1);
+            } else {
+                Assert.assertEquals(list2.get(i), (Integer) i);
+            }
+        }
+        // allowExceptionBreakChain=true
+        final FutureUtil.Sequencer sequencer2 = FutureUtil.Sequencer.create(true);
+        final List list3 = new ArrayList<>();
+        final List> futures3 = new ArrayList<>();
+        for (int i = 0; i < concurrentNum; i++) {
+            int finalI = i;
+            futures3.add(sequencer2.sequential(() -> CompletableFuture.runAsync(() -> {
+                if (finalI == 2) {
+                    throw new IllegalStateException();
+                }
+                list3.add(finalI);
+            }, executor)));
+        }
+        try {
+            FutureUtil.waitForAll(futures3).join();
+        } catch (Throwable ignore) {
+
+        }
+        for (int i = 2; i < concurrentNum; i++) {
+            Assert.assertTrue(futures3.get(i).isCompletedExceptionally());
+        }
+
+        for (int i = 0; i < 2; i++) {
+            Assert.assertEquals(list3.get(i), (Integer) i);
+        }
+    }
 }
\ No newline at end of file
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java
index ca768d38490cc..f5e7e0528bd7b 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java
@@ -124,7 +124,7 @@ private void handleSessionEvent(SessionEvent se) {
             if (se == SessionEvent.SessionReestablished) {
                 log.info("Metadata store session has been re-established. Revalidating all the existing locks.");
                 for (ResourceLockImpl lock : locks.values()) {
-                    futures.add(lock.revalidate(lock.getValue(), true, true));
+                    futures.add(lock.silentRevalidateOnce());
                 }
 
             } else if (se == SessionEvent.Reconnected) {
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java
index 5271a73249d80..93c994b2436b9 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/ResourceLockImpl.java
@@ -44,7 +44,7 @@ public class ResourceLockImpl implements ResourceLock {
     private long version;
     private final CompletableFuture expiredFuture;
     private boolean revalidateAfterReconnection = false;
-    private CompletableFuture pendingOperationFuture;
+    private final FutureUtil.Sequencer sequencer;
 
     private enum State {
         Init,
@@ -61,7 +61,7 @@ public ResourceLockImpl(MetadataStoreExtended store, MetadataSerde serde, Str
         this.path = path;
         this.version = -1;
         this.expiredFuture = new CompletableFuture<>();
-        this.pendingOperationFuture = CompletableFuture.completedFuture(null);
+        this.sequencer = FutureUtil.Sequencer.create();
         this.state = State.Init;
     }
 
@@ -74,11 +74,7 @@ public synchronized T getValue() {
     public synchronized CompletableFuture updateValue(T newValue) {
         // If there is an operation in progress, we're going to let it complete before attempting to
         // update the value
-        if (pendingOperationFuture.isDone()) {
-            pendingOperationFuture = CompletableFuture.completedFuture(null);
-        }
-
-        pendingOperationFuture = pendingOperationFuture.thenCompose(v -> {
+        return sequencer.sequential(() -> {
             synchronized (ResourceLockImpl.this) {
                 if (state != State.Valid) {
                     return CompletableFuture.failedFuture(
@@ -88,8 +84,6 @@ public synchronized CompletableFuture updateValue(T newValue) {
                 return acquire(newValue);
             }
         });
-
-        return pendingOperationFuture;
     }
 
     @Override
@@ -146,7 +140,7 @@ synchronized CompletableFuture acquire(T newValue) {
                 .thenRun(() -> result.complete(null))
                 .exceptionally(ex -> {
                     if (ex.getCause() instanceof LockBusyException) {
-                        revalidate(newValue, false, false)
+                        revalidate(newValue)
                                 .thenAccept(__ -> result.complete(null))
                                 .exceptionally(ex1 -> {
                                    result.completeExceptionally(ex1);
@@ -197,76 +191,54 @@ private CompletableFuture acquireWithNoRevalidation(T newValue) {
     }
 
     synchronized void lockWasInvalidated() {
-        if (state != State.Valid) {
-            // Ignore notifications while we're releasing the lock ourselves
-            return;
-        }
-
-        log.info("Lock on resource {} was invalidated", path);
-        revalidate(value, true, true)
-                .thenRun(() -> log.info("Successfully revalidated the lock on {}", path));
+        log.info("Lock on resource {} was invalidated. state {}", path, state);
+        silentRevalidateOnce();
     }
 
     synchronized CompletableFuture revalidateIfNeededAfterReconnection() {
         if (revalidateAfterReconnection) {
             revalidateAfterReconnection = false;
             log.warn("Revalidate lock at {} after reconnection", path);
-            return revalidate(value, true, true);
+            return silentRevalidateOnce();
         } else {
             return CompletableFuture.completedFuture(null);
         }
     }
 
-    synchronized CompletableFuture revalidate(T newValue, boolean trackPendingOperation,
-                                                    boolean revalidateAfterReconnection) {
-
-        final CompletableFuture trackFuture;
-
-        if (!trackPendingOperation) {
-            trackFuture = doRevalidate(newValue);
-        } else if (pendingOperationFuture.isDone()) {
-            pendingOperationFuture = doRevalidate(newValue);
-            trackFuture = pendingOperationFuture;
-        } else {
-            if (log.isDebugEnabled()) {
-                log.debug("Previous revalidating is not finished while revalidate newValue={}, value={}, version={}",
-                        newValue, value, version);
-            }
-            trackFuture = new CompletableFuture<>();
-            trackFuture.whenComplete((unused, throwable) -> {
-                doRevalidate(newValue).thenRun(() -> trackFuture.complete(null))
-                        .exceptionally(throwable1 -> {
-                            trackFuture.completeExceptionally(throwable1);
-                            return null;
-                        });
-            });
-            pendingOperationFuture = trackFuture;
-        }
-
-        trackFuture.exceptionally(ex -> {
-            synchronized (ResourceLockImpl.this) {
-                Throwable realCause = FutureUtil.unwrapCompletionException(ex);
-                if (!revalidateAfterReconnection || realCause instanceof BadVersionException
-                        || realCause instanceof LockBusyException) {
-                    log.warn("Failed to revalidate the lock at {}. Marked as expired. {}",
-                            path, realCause.getMessage());
-                    state = State.Released;
-                    expiredFuture.complete(null);
-                } else {
-                    // We failed to revalidate the lock due to connectivity issue
-                    // Continue assuming we hold the lock, until we can revalidate it, either
-                    // on Reconnected or SessionReestablished events.
-                    ResourceLockImpl.this.revalidateAfterReconnection = true;
-                    log.warn("Failed to revalidate the lock at {}. Retrying later on reconnection {}", path,
-                            realCause.getMessage());
-                }
-            }
-            return null;
-        });
-        return trackFuture;
+    /**
+     * Revalidate the distributed lock if it is not released.
+     * This method is thread-safe and it will perform multiple re-validation operations in turn.
+     */
+    synchronized CompletableFuture silentRevalidateOnce() {
+        return sequencer.sequential(() -> revalidate(value))
+                .thenRun(() -> log.info("Successfully revalidated the lock on {}", path))
+                .exceptionally(ex -> {
+                    synchronized (ResourceLockImpl.this) {
+                        Throwable realCause = FutureUtil.unwrapCompletionException(ex);
+                        if (realCause instanceof BadVersionException || realCause instanceof LockBusyException) {
+                            log.warn("Failed to revalidate the lock at {}. Marked as expired. {}",
+                                    path, realCause.getMessage());
+                            state = State.Released;
+                            expiredFuture.complete(null);
+                        } else {
+                            // We failed to revalidate the lock due to connectivity issue
+                            // Continue assuming we hold the lock, until we can revalidate it, either
+                            // on Reconnected or SessionReestablished events.
+                            revalidateAfterReconnection = true;
+                            log.warn("Failed to revalidate the lock at {}. Retrying later on reconnection {}", path,
+                                    realCause.getMessage());
+                        }
+                    }
+                    return null;
+                });
     }
 
-    private synchronized CompletableFuture doRevalidate(T newValue) {
+    private synchronized CompletableFuture revalidate(T newValue) {
+        // Since the distributed lock has been expired, we don't need to revalidate it.
+        if (state != State.Valid && state != State.Init) {
+            return CompletableFuture.failedFuture(
+                    new IllegalStateException("Lock was not in valid state: " + state));
+        }
         if (log.isDebugEnabled()) {
             log.debug("doRevalidate with newValue={}, version={}", newValue, version);
         }

From 5d2d61e298e337b93fb178da191bc41d311107c2 Mon Sep 17 00:00:00 2001
From: ran 
Date: Mon, 20 Mar 2023 17:41:20 +0800
Subject: [PATCH 129/174] [feat][txn] Transaction buffer snapshot writer reuse
 (#19641)

---
 .../SystemTopicTxnBufferSnapshotService.java  | 100 ++++++++++++++----
 ...SingleSnapshotAbortedTxnProcessorImpl.java |  21 ++--
 ...napshotSegmentAbortedTxnProcessorImpl.java |  45 ++++----
 .../SegmentAbortedTxnProcessorTest.java       |  19 +++-
 .../TopicTransactionBufferRecoverTest.java    |  17 +--
 .../broker/transaction/TransactionTest.java   |  85 ++++++++++++++-
 .../transaction/TransactionTestBase.java      |  20 ++++
 .../buffer/TransactionBufferCloseTest.java    |  79 ++++++++------
 8 files changed, 293 insertions(+), 93 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicTxnBufferSnapshotService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicTxnBufferSnapshotService.java
index a1b78d89a13eb..332d754cf97d2 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicTxnBufferSnapshotService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicTxnBufferSnapshotService.java
@@ -21,63 +21,127 @@
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory;
 import org.apache.pulsar.broker.systopic.SystemTopicClient;
 import org.apache.pulsar.broker.systopic.SystemTopicClientBase;
 import org.apache.pulsar.client.api.PulsarClient;
 import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.common.events.EventType;
+import org.apache.pulsar.common.naming.NamespaceName;
 import org.apache.pulsar.common.naming.TopicName;
-import org.apache.pulsar.common.util.FutureUtil;
 
+@Slf4j
 public class SystemTopicTxnBufferSnapshotService {
 
-    protected final Map> clients;
+    protected final ConcurrentHashMap> clients;
     protected final NamespaceEventsSystemTopicFactory namespaceEventsSystemTopicFactory;
 
     protected final Class schemaType;
     protected final EventType systemTopicType;
 
+    private final ConcurrentHashMap> refCountedWriterMap;
+
+    // The class ReferenceCountedWriter will maintain the reference count,
+    // when the reference count decrement to 0, it will be removed from writerFutureMap, the writer will be closed.
+    public static class ReferenceCountedWriter {
+
+        private final AtomicLong referenceCount;
+        private final NamespaceName namespaceName;
+        private final CompletableFuture> future;
+        private final SystemTopicTxnBufferSnapshotService snapshotService;
+
+        public ReferenceCountedWriter(NamespaceName namespaceName,
+                                      CompletableFuture> future,
+                                      SystemTopicTxnBufferSnapshotService snapshotService) {
+            this.referenceCount = new AtomicLong(1);
+            this.namespaceName = namespaceName;
+            this.snapshotService = snapshotService;
+            this.future = future;
+            this.future.exceptionally(t -> {
+                        log.error("[{}] Failed to create TB snapshot writer.", namespaceName, t);
+                snapshotService.refCountedWriterMap.remove(namespaceName, this);
+                return null;
+            });
+        }
+
+        public CompletableFuture> getFuture() {
+            return future;
+        }
+
+        private synchronized boolean retain() {
+            return this.referenceCount.incrementAndGet() > 0;
+        }
+
+        public synchronized void release() {
+            if (this.referenceCount.decrementAndGet() == 0) {
+                snapshotService.refCountedWriterMap.remove(namespaceName, this);
+                future.thenAccept(writer -> {
+                    final String topicName = writer.getSystemTopicClient().getTopicName().toString();
+                    writer.closeAsync().exceptionally(t -> {
+                        if (t != null) {
+                            log.error("[{}] Failed to close TB snapshot writer.", topicName, t);
+                        } else {
+                            if (log.isDebugEnabled()) {
+                                log.debug("[{}] Success to close TB snapshot writer.", topicName);
+                            }
+                        }
+                        return null;
+                    });
+                });
+            }
+        }
+
+    }
+
     public SystemTopicTxnBufferSnapshotService(PulsarClient client, EventType systemTopicType,
                                                Class schemaType) {
         this.namespaceEventsSystemTopicFactory = new NamespaceEventsSystemTopicFactory(client);
         this.systemTopicType = systemTopicType;
         this.schemaType = schemaType;
         this.clients = new ConcurrentHashMap<>();
-    }
-
-    public CompletableFuture> createWriter(TopicName topicName) {
-        return getTransactionBufferSystemTopicClient(topicName).thenCompose(SystemTopicClient::newWriterAsync);
+        this.refCountedWriterMap = new ConcurrentHashMap<>();
     }
 
     public CompletableFuture> createReader(TopicName topicName) {
-        return getTransactionBufferSystemTopicClient(topicName).thenCompose(SystemTopicClient::newReaderAsync);
+        return getTransactionBufferSystemTopicClient(topicName.getNamespaceObject()).newReaderAsync();
     }
 
     public void removeClient(TopicName topicName, SystemTopicClientBase transactionBufferSystemTopicClient) {
         if (transactionBufferSystemTopicClient.getReaders().size() == 0
                 && transactionBufferSystemTopicClient.getWriters().size() == 0) {
-            clients.remove(topicName);
+            clients.remove(topicName.getNamespaceObject());
         }
     }
 
-    protected CompletableFuture> getTransactionBufferSystemTopicClient(TopicName topicName) {
+    public ReferenceCountedWriter getReferenceWriter(NamespaceName namespaceName) {
+        return refCountedWriterMap.compute(namespaceName, (k, v) -> {
+            if (v != null && v.retain()) {
+                return v;
+            } else {
+                return new ReferenceCountedWriter<>(namespaceName,
+                        getTransactionBufferSystemTopicClient(namespaceName).newWriterAsync(), this);
+            }
+        });
+    }
+
+    private SystemTopicClient getTransactionBufferSystemTopicClient(NamespaceName namespaceName) {
         TopicName systemTopicName = NamespaceEventsSystemTopicFactory
-                .getSystemTopicName(topicName.getNamespaceObject(), systemTopicType);
+                .getSystemTopicName(namespaceName, systemTopicType);
         if (systemTopicName == null) {
-            return FutureUtil.failedFuture(
-                    new PulsarClientException
-                            .InvalidTopicNameException("Can't create SystemTopicBaseTxnBufferSnapshotIndexService, "
-                            + "because the topicName is null!"));
+            throw new RuntimeException(new PulsarClientException.InvalidTopicNameException(
+                    "Can't get the TB system topic client for namespace " + namespaceName
+                            + " with type " + systemTopicType + "."));
         }
-        return CompletableFuture.completedFuture(clients.computeIfAbsent(systemTopicName,
+
+        return clients.computeIfAbsent(namespaceName,
                 (v) -> namespaceEventsSystemTopicFactory
-                        .createTransactionBufferSystemTopicClient(systemTopicName,
-                                this, schemaType)));
+                        .createTransactionBufferSystemTopicClient(systemTopicName, this, schemaType));
     }
 
     public void close() throws Exception {
-        for (Map.Entry> entry : clients.entrySet()) {
+        for (Map.Entry> entry : clients.entrySet()) {
             entry.getValue().close();
         }
     }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java
index 87161e97512b9..5d582d564eadd 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java
@@ -28,6 +28,7 @@
 import org.apache.bookkeeper.mledger.impl.PositionImpl;
 import org.apache.commons.collections4.map.LinkedMap;
 import org.apache.pulsar.broker.service.BrokerServiceException;
+import org.apache.pulsar.broker.service.SystemTopicTxnBufferSnapshotService.ReferenceCountedWriter;
 import org.apache.pulsar.broker.service.persistent.PersistentTopic;
 import org.apache.pulsar.broker.systopic.SystemTopicClient;
 import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor;
@@ -42,7 +43,7 @@
 @Slf4j
 public class SingleSnapshotAbortedTxnProcessorImpl implements AbortedTxnProcessor {
     private final PersistentTopic topic;
-    private final CompletableFuture> takeSnapshotWriter;
+    private final ReferenceCountedWriter takeSnapshotWriter;
     /**
      * Aborts, map for jude message is aborted, linked for remove abort txn in memory when this
      * position have been deleted.
@@ -51,12 +52,14 @@ public class SingleSnapshotAbortedTxnProcessorImpl implements AbortedTxnProcesso
 
     private volatile long lastSnapshotTimestamps;
 
+    private volatile boolean isClosed = false;
+
     public SingleSnapshotAbortedTxnProcessorImpl(PersistentTopic topic) {
         this.topic = topic;
         this.takeSnapshotWriter = this.topic.getBrokerService().getPulsar()
                 .getTransactionBufferSnapshotServiceFactory()
-                .getTxnBufferSnapshotService().createWriter(TopicName.get(topic.getName()));
-        this.takeSnapshotWriter.exceptionally((ex) -> {
+                .getTxnBufferSnapshotService().getReferenceWriter(TopicName.get(topic.getName()).getNamespaceObject());
+        this.takeSnapshotWriter.getFuture().exceptionally((ex) -> {
                     log.error("{} Failed to create snapshot writer", topic.getName());
                     topic.close();
                     return null;
@@ -132,7 +135,7 @@ public CompletableFuture recoverFromSnapshot() {
 
     @Override
     public CompletableFuture clearAbortedTxnSnapshot() {
-        return this.takeSnapshotWriter.thenCompose(writer -> {
+        return this.takeSnapshotWriter.getFuture().thenCompose(writer -> {
             TransactionBufferSnapshot snapshot = new TransactionBufferSnapshot();
             snapshot.setTopicName(topic.getName());
             return writer.deleteAsync(snapshot.getTopicName(), snapshot);
@@ -141,7 +144,7 @@ public CompletableFuture clearAbortedTxnSnapshot() {
 
     @Override
     public CompletableFuture takeAbortedTxnsSnapshot(PositionImpl maxReadPosition) {
-        return takeSnapshotWriter.thenCompose(writer -> {
+        return takeSnapshotWriter.getFuture().thenCompose(writer -> {
             TransactionBufferSnapshot snapshot = new TransactionBufferSnapshot();
             snapshot.setTopicName(topic.getName());
             snapshot.setMaxReadPositionLedgerId(maxReadPosition.getLedgerId());
@@ -175,8 +178,12 @@ public long getLastSnapshotTimestamps() {
     }
 
     @Override
-    public CompletableFuture closeAsync() {
-        return takeSnapshotWriter.thenCompose(SystemTopicClient.Writer::closeAsync);
+    public synchronized CompletableFuture closeAsync() {
+        if (!isClosed) {
+            isClosed = true;
+            takeSnapshotWriter.release();
+        }
+        return CompletableFuture.completedFuture(null);
     }
 
     private void closeReader(SystemTopicClient.Reader reader) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java
index 7a9e0e1abedd9..751c03aff95a9 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java
@@ -43,6 +43,7 @@
 import org.apache.commons.lang3.tuple.MutablePair;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.pulsar.broker.service.BrokerServiceException;
+import org.apache.pulsar.broker.service.SystemTopicTxnBufferSnapshotService.ReferenceCountedWriter;
 import org.apache.pulsar.broker.service.persistent.PersistentTopic;
 import org.apache.pulsar.broker.systopic.SystemTopicClient;
 import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor;
@@ -442,10 +443,10 @@ public class PersistentWorker {
         private final PersistentTopic topic;
 
         //Persistent snapshot segment and index at the single thread.
-        private final CompletableFuture>
-                snapshotSegmentsWriterFuture;
-        private final CompletableFuture>
-                snapshotIndexWriterFuture;
+        private final ReferenceCountedWriter snapshotSegmentsWriter;
+        private final ReferenceCountedWriter snapshotIndexWriter;
+
+        private volatile boolean closed = false;
 
         private enum OperationState {
             None,
@@ -470,18 +471,20 @@ public enum OperationType {
 
         public PersistentWorker(PersistentTopic topic) {
             this.topic = topic;
-            this.snapshotSegmentsWriterFuture =  this.topic.getBrokerService().getPulsar()
+            this.snapshotSegmentsWriter = this.topic.getBrokerService().getPulsar()
                     .getTransactionBufferSnapshotServiceFactory()
-                    .getTxnBufferSnapshotSegmentService().createWriter(TopicName.get(topic.getName()));
-            this.snapshotSegmentsWriterFuture.exceptionally(ex -> {
+                    .getTxnBufferSnapshotSegmentService()
+                    .getReferenceWriter(TopicName.get(topic.getName()).getNamespaceObject());
+            this.snapshotSegmentsWriter.getFuture().exceptionally(ex -> {
                         log.error("{} Failed to create snapshot index writer", topic.getName());
                         topic.close();
                         return null;
                     });
-            this.snapshotIndexWriterFuture =  this.topic.getBrokerService().getPulsar()
+            this.snapshotIndexWriter =  this.topic.getBrokerService().getPulsar()
                     .getTransactionBufferSnapshotServiceFactory()
-                    .getTxnBufferSnapshotIndexService().createWriter(TopicName.get(topic.getName()));
-            this.snapshotIndexWriterFuture.exceptionally((ex) -> {
+                    .getTxnBufferSnapshotIndexService()
+                    .getReferenceWriter(TopicName.get(topic.getName()).getNamespaceObject());
+            this.snapshotIndexWriter.getFuture().exceptionally((ex) -> {
                         log.error("{} Failed to create snapshot writer", topic.getName());
                         topic.close();
                         return null;
@@ -631,7 +634,7 @@ private CompletableFuture writeSnapshotSegmentAsync(LinkedList segm
             transactionBufferSnapshotSegment.setPersistentPositionLedgerId(
                     abortedMarkerPersistentPosition.getLedgerId());
 
-            return snapshotSegmentsWriterFuture.thenCompose(segmentWriter -> {
+            return snapshotSegmentsWriter.getFuture().thenCompose(segmentWriter -> {
                 transactionBufferSnapshotSegment.setSequenceId(this.sequenceID.get());
                 return segmentWriter.writeAsync(buildKey(this.sequenceID.get()), transactionBufferSnapshotSegment);
             }).thenCompose((messageId) -> {
@@ -668,7 +671,7 @@ private CompletableFuture deleteSnapshotSegment(List positio
             List> results = new ArrayList<>();
             for (PositionImpl positionNeedToDelete : positionNeedToDeletes) {
                 long sequenceIdNeedToDelete = indexes.get(positionNeedToDelete).getSequenceID();
-                CompletableFuture res = snapshotSegmentsWriterFuture
+                CompletableFuture res = snapshotSegmentsWriter.getFuture()
                         .thenCompose(writer -> writer.deleteAsync(buildKey(sequenceIdNeedToDelete), null))
                         .thenCompose(messageId -> {
                             if (log.isDebugEnabled()) {
@@ -695,7 +698,7 @@ private CompletableFuture deleteSnapshotSegment(List positio
 
         private CompletableFuture updateSnapshotIndex(TransactionBufferSnapshotIndexesMetadata snapshotSegment) {
             TransactionBufferSnapshotIndexes snapshotIndexes = new TransactionBufferSnapshotIndexes();
-            CompletableFuture res = snapshotIndexWriterFuture
+            CompletableFuture res = snapshotIndexWriter.getFuture()
                     .thenCompose((indexesWriter) -> {
                         snapshotIndexes.setIndexList(indexes.values().stream().toList());
                         snapshotIndexes.setSnapshot(snapshotSegment);
@@ -712,7 +715,7 @@ private CompletableFuture updateSnapshotIndex(TransactionBufferSnapshotInd
 
         private CompletableFuture clearSnapshotSegmentAndIndexes() {
             CompletableFuture res = persistentWorker.clearAllSnapshotSegments()
-                    .thenCompose((ignore) -> snapshotIndexWriterFuture
+                    .thenCompose((ignore) -> snapshotIndexWriter.getFuture()
                             .thenCompose(indexesWriter -> indexesWriter.writeAsync(topic.getName(), null)))
                     .thenRun(() ->
                             log.debug("Successes to clear the snapshot segment and indexes for the topic [{}]",
@@ -747,7 +750,7 @@ private CompletableFuture clearAllSnapshotSegments() {
                                 Message message = reader.readNextAsync()
                                         .get(getSystemClientOperationTimeoutMs(), TimeUnit.MILLISECONDS);
                                 if (topic.getName().equals(message.getValue().getTopicName())) {
-                                   snapshotSegmentsWriterFuture.get().write(message.getKey(), null);
+                                   snapshotSegmentsWriter.getFuture().get().write(message.getKey(), null);
                                 }
                             }
                             return CompletableFuture.completedFuture(null);
@@ -760,11 +763,13 @@ private CompletableFuture clearAllSnapshotSegments() {
            });
         }
 
-
-        CompletableFuture closeAsync() {
-            return CompletableFuture.allOf(
-                    this.snapshotIndexWriterFuture.thenCompose(SystemTopicClient.Writer::closeAsync),
-                    this.snapshotSegmentsWriterFuture.thenCompose(SystemTopicClient.Writer::closeAsync));
+        synchronized CompletableFuture closeAsync() {
+            if (!closed) {
+                closed = true;
+                snapshotSegmentsWriter.release();
+                snapshotIndexWriter.release();
+            }
+            return CompletableFuture.completedFuture(null);
         }
     }
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java
index ffc059de8e656..c157d7cf8c527 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java
@@ -19,6 +19,8 @@
 package org.apache.pulsar.broker.transaction;
 
 import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
 import java.lang.reflect.Field;
 import java.util.LinkedList;
 import java.util.NavigableMap;
@@ -34,6 +36,7 @@
 import org.apache.commons.lang3.tuple.MutablePair;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.service.BrokerServiceException;
+import org.apache.pulsar.broker.service.SystemTopicTxnBufferSnapshotService.ReferenceCountedWriter;
 import org.apache.pulsar.broker.service.persistent.PersistentTopic;
 import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory;
 import org.apache.pulsar.broker.systopic.SystemTopicClient;
@@ -125,6 +128,7 @@ public void testPutAbortedTxnIntoProcessor() throws Exception {
         newProcessor.trimExpiredAbortedTxns();
         //4. Verify the two sealed segment will be deleted.
         Awaitility.await().untilAsserted(() -> verifyAbortedTxnIDAndSegmentIndex(newProcessor, 11, 4));
+        processor.closeAsync().get(5, TimeUnit.SECONDS);
     }
 
     private void waitTaskExecuteCompletely(AbortedTxnProcessor processor) throws Exception {
@@ -177,8 +181,11 @@ public void testFuturesCanCompleteWhenItIsCanceled() throws Exception {
                 new MutablePair<>(new CompletableFuture<>(), task)));
         try {
             processor.takeAbortedTxnsSnapshot(new PositionImpl(1, 10)).get(2, TimeUnit.SECONDS);
+            fail("The update index operation should fail.");
         } catch (Exception e) {
             Assert.assertTrue(e.getCause() instanceof BrokerServiceException.ServiceUnitNotReadyException);
+        } finally {
+            processor.closeAsync().get(5, TimeUnit.SECONDS);
         }
     }
 
@@ -200,12 +207,13 @@ public void testClearSnapshotSegments() throws Exception {
         SnapshotSegmentAbortedTxnProcessorImpl.PersistentWorker worker =
                 (SnapshotSegmentAbortedTxnProcessorImpl.PersistentWorker) field.get(processor);
         Field indexWriteFutureField = SnapshotSegmentAbortedTxnProcessorImpl
-                .PersistentWorker.class.getDeclaredField("snapshotIndexWriterFuture");
+                .PersistentWorker.class.getDeclaredField("snapshotIndexWriter");
         indexWriteFutureField.setAccessible(true);
-        CompletableFuture> snapshotIndexWriterFuture =
-                (CompletableFuture>)
-                        indexWriteFutureField.get(worker);
-        snapshotIndexWriterFuture.get().close();
+        ReferenceCountedWriter snapshotIndexWriter =
+                (ReferenceCountedWriter) indexWriteFutureField.get(worker);
+        snapshotIndexWriter.release();
+        // After release, the writer should be closed, call close method again to make sure the writer was closed.
+        snapshotIndexWriter.getFuture().get().close();
         //3. Try to write a snapshot segment that will fail to update indexes.
         for (int j = 0; j < SEGMENT_SIZE; j++) {
             TxnID txnID = new TxnID(0, j);
@@ -233,6 +241,7 @@ public void testClearSnapshotSegments() throws Exception {
         //7. Verify the snapshot segments and index after clearing.
         verifySnapshotSegmentsSize(PROCESSOR_TOPIC, 0);
         verifySnapshotSegmentsIndexSize(PROCESSOR_TOPIC, 1);
+        processor.closeAsync().get(5, TimeUnit.SECONDS);
     }
 
     private void verifySnapshotSegmentsSize(String topic, int size) throws Exception {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java
index d4ddb26e014ca..2d6622571c033 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java
@@ -57,6 +57,7 @@
 import org.apache.pulsar.broker.service.AbstractTopic;
 import org.apache.pulsar.broker.service.BrokerService;
 import org.apache.pulsar.broker.service.SystemTopicTxnBufferSnapshotService;
+import org.apache.pulsar.broker.service.SystemTopicTxnBufferSnapshotService.ReferenceCountedWriter;
 import org.apache.pulsar.broker.service.Topic;
 import org.apache.pulsar.broker.service.TransactionBufferSnapshotServiceFactory;
 import org.apache.pulsar.broker.service.persistent.PersistentTopic;
@@ -602,11 +603,12 @@ public void testTransactionBufferRecoverThrowException() throws Exception {
                 mock(SystemTopicTxnBufferSnapshotService.class);
         SystemTopicClient.Reader reader = mock(SystemTopicClient.Reader.class);
         SystemTopicClient.Writer writer = mock(SystemTopicClient.Writer.class);
+        ReferenceCountedWriter refCounterWriter = mock(ReferenceCountedWriter.class);
+        doReturn(CompletableFuture.completedFuture(writer)).when(refCounterWriter).getFuture();
 
         doReturn(CompletableFuture.completedFuture(reader))
                 .when(systemTopicTxnBufferSnapshotService).createReader(any());
-        doReturn(CompletableFuture.completedFuture(writer))
-                .when(systemTopicTxnBufferSnapshotService).createWriter(any());
+        doReturn(refCounterWriter).when(systemTopicTxnBufferSnapshotService).getReferenceWriter(any());
         TransactionBufferSnapshotServiceFactory transactionBufferSnapshotServiceFactory =
                 mock(TransactionBufferSnapshotServiceFactory.class);
         doReturn(systemTopicTxnBufferSnapshotService)
@@ -645,8 +647,9 @@ public void testTransactionBufferRecoverThrowException() throws Exception {
         originalTopic = (PersistentTopic) getPulsarServiceList().get(0)
                 .getBrokerService().getTopic(TopicName.get(topic).toString(), false).get().get();
         // mock create writer fail
-        doReturn(FutureUtil.failedFuture(new PulsarClientException("test")))
-                .when(systemTopicTxnBufferSnapshotService).createWriter(any());
+        ReferenceCountedWriter failedCountedWriter = mock(ReferenceCountedWriter.class);
+        doReturn(FutureUtil.failedFuture(new PulsarClientException("test"))).when(failedCountedWriter).getFuture();
+        doReturn(failedCountedWriter).when(systemTopicTxnBufferSnapshotService).getReferenceWriter(any());
         checkCloseTopic(pulsarClient, transactionBufferSnapshotServiceFactoryOriginal,
                 transactionBufferSnapshotServiceFactory, originalTopic, field, producer);
     }
@@ -703,7 +706,8 @@ public void testTransactionBufferIndexSystemTopic() throws Exception {
                 new TransactionBufferSnapshotServiceFactory(pulsarClient).getTxnBufferSnapshotIndexService();
 
         SystemTopicClient.Writer indexesWriter =
-                transactionBufferSnapshotIndexService.createWriter(TopicName.get(SNAPSHOT_INDEX)).get();
+                transactionBufferSnapshotIndexService.getReferenceWriter(
+                        TopicName.get(SNAPSHOT_INDEX).getNamespaceObject()).getFuture().get();
 
         SystemTopicClient.Reader indexesReader =
                 transactionBufferSnapshotIndexService.createReader(TopicName.get(SNAPSHOT_INDEX)).get();
@@ -764,7 +768,8 @@ public void testTransactionBufferSegmentSystemTopic() throws Exception {
                 new TransactionBufferSnapshotServiceFactory(pulsarClient).getTxnBufferSnapshotSegmentService();
 
         SystemTopicClient.Writer
-                segmentWriter = transactionBufferSnapshotSegmentService.createWriter(snapshotSegmentTopicName).get();
+                segmentWriter = transactionBufferSnapshotSegmentService
+                .getReferenceWriter(snapshotSegmentTopicName.getNamespaceObject()).getFuture().get();
 
         // write two snapshot to snapshot segment topic
         TransactionBufferSnapshotSegment snapshot =
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java
index 20aeac0ed648f..c3533e70cf8be 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java
@@ -86,6 +86,7 @@
 import org.apache.pulsar.broker.service.BrokerService;
 import org.apache.pulsar.broker.service.BrokerServiceException;
 import org.apache.pulsar.broker.service.SystemTopicTxnBufferSnapshotService;
+import org.apache.pulsar.broker.service.SystemTopicTxnBufferSnapshotService.ReferenceCountedWriter;
 import org.apache.pulsar.broker.service.Topic;
 import org.apache.pulsar.broker.service.TransactionBufferSnapshotServiceFactory;
 import org.apache.pulsar.broker.service.persistent.PersistentSubscription;
@@ -1532,8 +1533,10 @@ public void testTBRecoverChangeStateError() throws InterruptedException, Timeout
                 = mock(SystemTopicTxnBufferSnapshotService.class);
         SystemTopicClient.Writer writer = mock(SystemTopicClient.Writer.class);
         when(writer.closeAsync()).thenReturn(CompletableFuture.completedFuture(null));
-        when(systemTopicTxnBufferSnapshotService.createWriter(any()))
-                .thenReturn(CompletableFuture.completedFuture(writer));
+        ReferenceCountedWriter refCounterWriter = mock(ReferenceCountedWriter.class);
+        doReturn(CompletableFuture.completedFuture(writer)).when(refCounterWriter).getFuture();
+        when(systemTopicTxnBufferSnapshotService.getReferenceWriter(any()))
+                .thenReturn(refCounterWriter);
         TransactionBufferSnapshotServiceFactory transactionBufferSnapshotServiceFactory =
                 mock(TransactionBufferSnapshotServiceFactory.class);
         when(transactionBufferSnapshotServiceFactory.getTxnBufferSnapshotService())
@@ -1676,4 +1679,82 @@ public void testDeleteNamespace() throws Exception {
         admin.namespaces().deleteNamespace(namespace, true);
     }
 
+
+    @Test(timeOut = 10_000)
+    public void testTBSnapshotWriter() throws Exception {
+        String namespace = TENANT + "/ns-" + RandomStringUtils.randomAlphabetic(5);
+        admin.namespaces().createNamespace(namespace, 16);
+        String topic = namespace + "/test-create-snapshot-writer-failed";
+        int partitionCount = 20;
+        admin.topics().createPartitionedTopic(topic, partitionCount);
+
+        Class clazz = SystemTopicTxnBufferSnapshotService.class;
+        Field field = clazz.getDeclaredField("refCountedWriterMap");
+        field.setAccessible(true);
+        // inject a failed writer future
+        CompletableFuture> writerFuture = new CompletableFuture<>();
+        for (PulsarService pulsarService : pulsarServiceList) {
+            SystemTopicTxnBufferSnapshotService bufferSnapshotService =
+                    pulsarService.getTransactionBufferSnapshotServiceFactory().getTxnBufferSnapshotService();
+            ConcurrentHashMap writerMap1 =
+                    ((ConcurrentHashMap) field.get(bufferSnapshotService));
+            ReferenceCountedWriter failedCountedWriter =
+                    new ReferenceCountedWriter(NamespaceName.get(namespace), writerFuture, bufferSnapshotService);
+            writerMap1.put(NamespaceName.get(namespace), failedCountedWriter);
+
+            SystemTopicTxnBufferSnapshotService segmentSnapshotService =
+                    pulsarService.getTransactionBufferSnapshotServiceFactory().getTxnBufferSnapshotSegmentService();
+            ConcurrentHashMap writerMap2 =
+                    ((ConcurrentHashMap) field.get(segmentSnapshotService));
+            ReferenceCountedWriter failedCountedWriter2 =
+                    new ReferenceCountedWriter(NamespaceName.get(namespace), writerFuture, segmentSnapshotService);
+            writerMap2.put(NamespaceName.get(namespace), failedCountedWriter2);
+
+            SystemTopicTxnBufferSnapshotService indexSnapshotService =
+                    pulsarService.getTransactionBufferSnapshotServiceFactory().getTxnBufferSnapshotIndexService();
+            ConcurrentHashMap writerMap3 =
+                    ((ConcurrentHashMap) field.get(indexSnapshotService));
+            ReferenceCountedWriter failedCountedWriter3 =
+                    new ReferenceCountedWriter(NamespaceName.get(namespace), writerFuture, indexSnapshotService);
+            writerMap3.put(NamespaceName.get(namespace), failedCountedWriter3);
+        }
+
+        CompletableFuture> producerFuture = pulsarClient.newProducer()
+                .topic(topic)
+                .sendTimeout(0, TimeUnit.SECONDS)
+                .createAsync();
+        getTopic("persistent://" + topic + "-partition-0");
+        Thread.sleep(3000);
+        // the producer shouldn't be created, because the transaction buffer snapshot writer future didn't finish.
+        assertFalse(producerFuture.isDone());
+
+        // The topic will be closed, because the transaction buffer snapshot writer future is failed,
+        // the failed writer future will be removed, the producer will be reconnected and work well.
+        writerFuture.completeExceptionally(new PulsarClientException.TopicTerminatedException("failed writer"));
+        Producer producer = producerFuture.get();
+
+        for (int i = 0; i < partitionCount * 2; i++) {
+            Transaction txn = pulsarClient.newTransaction()
+                    .withTransactionTimeout(1, TimeUnit.MINUTES).build().get();
+            producer.newMessage(txn).value("test".getBytes()).sendAsync();
+            txn.commit().get();
+        }
+        checkSnapshotPublisherCount(namespace, 1);
+        producer.close();
+        admin.topics().unload(topic);
+        checkSnapshotPublisherCount(namespace, 0);
+    }
+
+    private void getTopic(String topicName) {
+        Awaitility.await().atMost(5, TimeUnit.SECONDS).pollInterval(100, TimeUnit.MILLISECONDS)
+                .until(() -> {
+                    for (PulsarService pulsarService : pulsarServiceList) {
+                        if (pulsarService.getBrokerService().getTopicReference(topicName).isPresent()) {
+                            return true;
+                        }
+                    }
+                    return false;
+                });
+    }
+
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java
index fd49354342fa0..f45eda8d21fbe 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java
@@ -41,12 +41,17 @@
 import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.common.naming.NamespaceName;
 import org.apache.pulsar.common.naming.SystemTopicNames;
+import org.apache.pulsar.common.naming.TopicDomain;
+import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.partition.PartitionedTopicMetadata;
 import org.apache.pulsar.common.policies.data.ClusterData;
+import org.apache.pulsar.common.policies.data.PublisherStats;
 import org.apache.pulsar.common.policies.data.TenantInfoImpl;
 import org.apache.pulsar.common.policies.data.TopicType;
 import org.apache.pulsar.metadata.api.MetadataStoreException;
 import org.apache.pulsar.tests.TestRetrySupport;
+import org.awaitility.Awaitility;
+import org.testng.Assert;
 
 @Slf4j
 public abstract class TransactionTestBase extends TestRetrySupport {
@@ -223,4 +228,19 @@ protected void deleteNamespaceWithRetry(String ns, boolean force, PulsarAdmin ad
             throws Exception {
         MockedPulsarServiceBaseTest.deleteNamespaceWithRetry(ns, force, admin, pulsarServiceList);
     }
+
+    public void checkSnapshotPublisherCount(String namespace, int expectCount) {
+        TopicName snTopicName = TopicName.get(TopicDomain.persistent.value(), NamespaceName.get(namespace),
+                SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT);
+        Awaitility.await()
+                .atMost(5, TimeUnit.SECONDS)
+                .pollInterval(100, TimeUnit.MILLISECONDS)
+                .untilAsserted(() -> {
+                    List publisherStatsList =
+                            (List) admin.topics()
+                                    .getStats(snTopicName.getPartitionedTopicName()).getPublishers();
+                    Assert.assertEquals(publisherStatsList.size(), expectCount);
+                });
+    }
+
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferCloseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferCloseTest.java
index cd3a14da5967d..e92cf29521e34 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferCloseTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferCloseTest.java
@@ -19,6 +19,9 @@
 package org.apache.pulsar.broker.transaction.buffer;
 
 import com.google.common.collect.Sets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.pulsar.broker.transaction.TransactionTestBase;
@@ -26,20 +29,13 @@
 import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.client.api.transaction.TransactionCoordinatorClient;
 import org.apache.pulsar.client.impl.PulsarClientImpl;
-import org.apache.pulsar.common.naming.NamespaceName;
-import org.apache.pulsar.common.naming.SystemTopicNames;
-import org.apache.pulsar.common.naming.TopicDomain;
 import org.apache.pulsar.common.naming.TopicName;
-import org.apache.pulsar.common.policies.data.PublisherStats;
 import org.apache.pulsar.common.policies.data.TenantInfoImpl;
 import org.awaitility.Awaitility;
-import org.testng.Assert;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
 
 /**
  * Transaction buffer close test.
@@ -71,49 +67,62 @@ public Object[][] isPartition() {
 
     @Test(timeOut = 10_000, dataProvider = "isPartition")
     public void deleteTopicCloseTransactionBufferTest(boolean isPartition) throws Exception {
-        int expectedCount = isPartition ? 30 : 1;
-        TopicName topicName = createAndLoadTopic(isPartition, expectedCount);
-        checkSnapshotPublisherCount(topicName.getNamespace(), expectedCount);
+        int partitionCount = isPartition ? 30 : 1;
+        List topicNames = createAndLoadTopics(isPartition, partitionCount);
+        String namespaceName = topicNames.get(0).getNamespace();
+        checkSnapshotPublisherCount(namespaceName, 1);
+
+        for (int i = 0; i < topicNames.size(); i++) {
+            deleteTopic(isPartition, topicNames.get(i));
+            // When delete all topics of the namespace, the publisher count should be 0.
+            int expectCount = i == topicNames.size() - 1 ? 0 : 1;
+            checkSnapshotPublisherCount(namespaceName, expectCount);
+        }
+    }
+
+    private void deleteTopic(boolean isPartition, TopicName topicName) throws PulsarAdminException {
         if (isPartition) {
             admin.topics().deletePartitionedTopic(topicName.getPartitionedTopicName(), true);
         } else {
             admin.topics().delete(topicName.getPartitionedTopicName(), true);
         }
-        checkSnapshotPublisherCount(topicName.getNamespace(), 0);
     }
 
     @Test(timeOut = 10_000, dataProvider = "isPartition")
     public void unloadTopicCloseTransactionBufferTest(boolean isPartition) throws Exception {
-        int expectedCount = isPartition ? 30 : 1;
-        TopicName topicName = createAndLoadTopic(isPartition, expectedCount);
-        checkSnapshotPublisherCount(topicName.getNamespace(), expectedCount);
-        admin.topics().unload(topicName.getPartitionedTopicName());
-        checkSnapshotPublisherCount(topicName.getNamespace(), 0);
+        int partitionCount = isPartition ? 30 : 1;
+        List topicNames = createAndLoadTopics(isPartition, partitionCount);
+        String namespaceName = topicNames.get(0).getNamespace();
+        checkSnapshotPublisherCount(namespaceName, 1);
+
+        for (int i = 0; i < topicNames.size(); i++) {
+            admin.topics().unload(topicNames.get(i).getPartitionedTopicName());
+            // When unload all topics of the namespace, the publisher count should be 0.
+            int expectCount = i == topicNames.size() - 1 ? 0 : 1;
+            checkSnapshotPublisherCount(namespaceName, expectCount);
+        }
     }
 
-    private TopicName createAndLoadTopic(boolean isPartition, int partitionCount)
+    private List createAndLoadTopics(boolean isPartition, int partitionCount)
             throws PulsarAdminException, PulsarClientException {
         String namespace = TENANT + "/ns-" + RandomStringUtils.randomAlphabetic(5);
         admin.namespaces().createNamespace(namespace, 3);
-        String topic = namespace + "/tb-close-test-";
-        if (isPartition) {
-            admin.topics().createPartitionedTopic(topic, partitionCount);
-        }
-        pulsarClient.newProducer()
-                .topic(topic)
-                .sendTimeout(0, TimeUnit.SECONDS)
-                .create()
-                .close();
-        return TopicName.get(topic);
-    }
+        String topic = namespace + "/tb-close-test";
+        List topics = new ArrayList<>();
 
-    private void checkSnapshotPublisherCount(String namespace, int expectCount) throws PulsarAdminException {
-        TopicName snTopicName = TopicName.get(TopicDomain.persistent.value(), NamespaceName.get(namespace),
-                SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT);
-        List publisherStatsList =
-                (List) admin.topics()
-                        .getStats(snTopicName.getPartitionedTopicName()).getPublishers();
-        Assert.assertEquals(publisherStatsList.size(), expectCount);
+        for (int i = 0; i < 2; i++) {
+            String t = topic + "-" + i;
+            if (isPartition) {
+                admin.topics().createPartitionedTopic(t, partitionCount);
+            }
+            pulsarClient.newProducer()
+                    .topic(t)
+                    .sendTimeout(0, TimeUnit.SECONDS)
+                    .create()
+                    .close();
+            topics.add(TopicName.get(t));
+        }
+        return topics;
     }
 
 }

From c4abe7bd3c5fe59596decb0b6ce1badd40de0eeb Mon Sep 17 00:00:00 2001
From: fengyubiao 
Date: Mon, 20 Mar 2023 23:09:49 +0800
Subject: [PATCH 130/174] [fix][sec] Upgrade dependency-check-maven and remove
 javax.el (#19764)

---
 pom.xml                                       | 20 ++++++++++++++++++-
 ...owasp-dependency-check-false-positives.xml | 10 ++++++++++
 2 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index dc27ed54274fb..257018b0b0598 100644
--- a/pom.xml
+++ b/pom.xml
@@ -291,7 +291,7 @@ flexible messaging model and an intuitive client API.
     0.1.4
     1.3
     0.4
-    8.0.1
+    8.1.2
     0.9.15
     1.6.1
     6.4.0
@@ -878,6 +878,24 @@ flexible messaging model and an intuitive client API.
         ${caffeine.version}
       
 
+      
+        org.bouncycastle
+        bcpkix-jdk15on
+        ${bouncycastle.version}
+      
+
+      
+        com.cronutils
+        cron-utils
+        ${cron-utils.version}
+        
+          
+            org.glassfish
+            javax.el
+          
+        
+      
+
       
         com.yahoo.athenz
         athenz-zts-java-client-core
diff --git a/src/owasp-dependency-check-false-positives.xml b/src/owasp-dependency-check-false-positives.xml
index 21a3679c0d8f7..345be8f4d2c06 100644
--- a/src/owasp-dependency-check-false-positives.xml
+++ b/src/owasp-dependency-check-false-positives.xml
@@ -182,6 +182,16 @@
     CVE-2021-4277
   
 
+  
+    It treat pulsar-io-kafka-connect-adaptor as a lib of Kafka, CVE-2021-25194 is a false positive.
+    CVE-2023-25194
+  
+
+  
+    It treat pulsar-io-kafka-connect-adaptor as a lib of Kafka, CVE-2021-34917 is a false positive.
+    CVE-2022-34917
+  
+
   
     yaml_project is not used at all. Any CVEs reported for yaml_project are false positives.
     cpe:/a:yaml_project:yaml

From 164add6236eccb34f16683e86904f8d8d0b90c54 Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Mon, 20 Mar 2023 13:55:02 -0500
Subject: [PATCH 131/174] [improve][proxy] Remove unnecessary executor
 callback; use assert (#19670)

### Motivation

The `DirectProxyHandler` and the `ProxyConnection` are run in the same event loop to prevent context switching. As such, we do not need to schedule an event onto an event loop that is in fact the same event loop. Further, scheduling on that event loop could have resulted in uncaught failures because the method was run without any error handling.

Additionally, we can use `assert` to verify that we are in the event loop. Netty makes extensive use of this paradigm, as described in this PR #19653. The primary benefit here is that we skip some unnecessary volatile reads when running the code in production.

### Modifications

* Replace `checkState` with `assert` in `ProxyConnection` class
* Remove unnecessary event execution in callback when starting up the `DirectProxyHandler`.

### Verifying this change

This change is covered by the assertions that are added.

### Does this pull request potentially affect one of the following parts:

This is a minor improvement that should not break anything.

### Documentation

- [x] `doc-not-needed`

### Matching PR in forked repository

PR in forked repository: https://github.com/michaeljmarshall/pulsar/pull/33
---
 .../apache/pulsar/proxy/server/ProxyConnection.java    | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java
index 5a53f6ec014a2..1e19d760c6d7b 100644
--- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java
+++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java
@@ -19,7 +19,6 @@
 package org.apache.pulsar.proxy.server;
 
 import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkState;
 import io.netty.buffer.ByteBuf;
 import io.netty.channel.Channel;
 import io.netty.channel.ChannelFutureListener;
@@ -380,7 +379,7 @@ private synchronized void completeConnect() throws PulsarClientException {
     }
 
     private void handleBrokerConnected(DirectProxyHandler directProxyHandler, CommandConnected connected) {
-        checkState(ctx.executor().inEventLoop(), "This method should be called in the event loop");
+        assert ctx.executor().inEventLoop();
         if (state == State.ProxyConnectingToBroker && ctx.channel().isOpen() && this.directProxyHandler == null) {
             this.directProxyHandler = directProxyHandler;
             state = State.ProxyConnectionToBroker;
@@ -401,7 +400,7 @@ private void handleBrokerConnected(DirectProxyHandler directProxyHandler, Comman
     }
 
     private void connectToBroker(InetSocketAddress brokerAddress) {
-        checkState(ctx.executor().inEventLoop(), "This method should be called in the event loop");
+        assert ctx.executor().inEventLoop();
         DirectProxyHandler directProxyHandler = new DirectProxyHandler(service, this);
         directProxyHandler.connect(proxyToBrokerUrl, brokerAddress, protocolVersionToAdvertise);
     }
@@ -409,10 +408,13 @@ private void connectToBroker(InetSocketAddress brokerAddress) {
     public void brokerConnected(DirectProxyHandler directProxyHandler, CommandConnected connected) {
         try {
             final CommandConnected finalConnected = new CommandConnected().copyFrom(connected);
-            ctx.executor().execute(() -> handleBrokerConnected(directProxyHandler, finalConnected));
+            handleBrokerConnected(directProxyHandler, finalConnected);
         } catch (RejectedExecutionException e) {
             LOG.error("Event loop was already closed. Closing broker connection.", e);
             directProxyHandler.close();
+        } catch (AssertionError e) {
+            LOG.error("Failed assertion, closing direct proxy handler.", e);
+            directProxyHandler.close();
         }
     }
 

From c6de57c4b4510c5064c6e192df8fa9d51cede01d Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Mon, 20 Mar 2023 14:07:04 -0500
Subject: [PATCH 132/174] [improve][broker] Test AuthorizationService to cover
 proxyRoles behavior (#19845)

Relates to: #19455 #19830

### Motivation

When I added the requirement for the proxy to use a role in the `proxyRoles` set, I didn't add a test that checked the negative case. This new test was first added in #19830 with one small difference. The goal of this test is to ensure that authorization of the client role and the original role is handled correctly.

### Modifications

* Add new test class named `AuthorizationServiceTest`. We use `pass.proxy` and `fail.proxy` as proxy roles to simulate cases where the proxy's role passes and fails authorization, which is always possible.
* Add new mock authorization provider named `MockAuthorizationProvider`. The logic is to let any role that starts with `pass` be considered authorized.

### Verifying this change

This is a new test. It simply verifies the existing behavior to prevent future regressions.

### Documentation

- [x] `doc-not-needed`

### Matching PR in forked repository

PR in forked repository: Skipping forked test since the new tests pass locally and there are no other modifications.
---
 .../AuthorizationServiceTest.java             | 135 ++++++++++++++++
 .../MockAuthorizationProvider.java            | 144 ++++++++++++++++++
 2 files changed, 279 insertions(+)
 create mode 100644 pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/AuthorizationServiceTest.java
 create mode 100644 pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MockAuthorizationProvider.java

diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/AuthorizationServiceTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/AuthorizationServiceTest.java
new file mode 100644
index 0000000000000..6f9dffa11b948
--- /dev/null
+++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/AuthorizationServiceTest.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.authorization;
+
+import static org.testng.AssertJUnit.assertFalse;
+import static org.testng.AssertJUnit.assertTrue;
+import java.util.HashSet;
+import org.apache.pulsar.broker.PulsarServerException;
+import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.common.naming.NamespaceName;
+import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.policies.data.NamespaceOperation;
+import org.apache.pulsar.common.policies.data.PolicyName;
+import org.apache.pulsar.common.policies.data.PolicyOperation;
+import org.apache.pulsar.common.policies.data.TenantOperation;
+import org.apache.pulsar.common.policies.data.TopicOperation;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+public class AuthorizationServiceTest {
+
+    AuthorizationService authorizationService;
+
+    @BeforeClass
+    void beforeClass() throws PulsarServerException {
+        ServiceConfiguration conf = new ServiceConfiguration();
+        conf.setAuthorizationEnabled(true);
+        // Consider both of these proxy roles to make testing more comprehensive
+        HashSet proxyRoles = new HashSet<>();
+        proxyRoles.add("pass.proxy");
+        proxyRoles.add("fail.proxy");
+        conf.setProxyRoles(proxyRoles);
+        conf.setAuthorizationProvider(MockAuthorizationProvider.class.getName());
+        authorizationService = new AuthorizationService(conf, null);
+    }
+
+    /**
+     * See {@link MockAuthorizationProvider} for the implementation of the mock authorization provider.
+     */
+    @DataProvider(name = "roles")
+    public Object[][] encryptionProvider() {
+        return new Object[][]{
+                // Schema: role, originalRole, whether authorization should pass
+
+                // Client conditions where original role isn't passed or is blank
+                {"pass.client", null, Boolean.TRUE},
+                {"pass.client", " ", Boolean.TRUE},
+                {"fail.client", null, Boolean.FALSE},
+                {"fail.client", " ", Boolean.FALSE},
+
+                // Proxy conditions where original role isn't passed or is blank
+                {"pass.proxy", null, Boolean.FALSE},
+                {"pass.proxy", " ", Boolean.FALSE},
+                {"fail.proxy", null, Boolean.FALSE},
+                {"fail.proxy", " ", Boolean.FALSE},
+
+                // Normal proxy and client conditions
+                {"pass.proxy", "pass.client", Boolean.TRUE},
+                {"pass.proxy", "fail.client", Boolean.FALSE},
+                {"fail.proxy", "pass.client", Boolean.FALSE},
+                {"fail.proxy", "fail.client", Boolean.FALSE},
+
+                // Not proxy with original principal
+                {"pass.not-proxy", "pass.client", Boolean.FALSE}, // non proxy role can't pass original role
+                {"pass.not-proxy", "fail.client", Boolean.FALSE},
+                {"fail.not-proxy", "pass.client", Boolean.FALSE},
+                {"fail.not-proxy", "fail.client", Boolean.FALSE},
+
+                // Covers an unlikely scenario, but valid in the context of this test
+                {null, "pass.proxy", Boolean.FALSE},
+        };
+    }
+
+    private void checkResult(boolean expected, boolean actual) {
+        if (expected) {
+            assertTrue(actual);
+        } else {
+            assertFalse(actual);
+        }
+    }
+
+    @Test(dataProvider = "roles")
+    public void testAllowTenantOperationAsync(String role, String originalRole, boolean shouldPass) throws Exception {
+        boolean isAuthorized = authorizationService.allowTenantOperationAsync("tenant",
+                TenantOperation.DELETE_NAMESPACE, originalRole, role, null).get();
+        checkResult(shouldPass, isAuthorized);
+    }
+
+    @Test(dataProvider = "roles")
+    public void testNamespaceOperationAsync(String role, String originalRole, boolean shouldPass) throws Exception {
+        boolean isAuthorized = authorizationService.allowNamespaceOperationAsync(NamespaceName.get("public/default"),
+                NamespaceOperation.PACKAGES, originalRole, role, null).get();
+        checkResult(shouldPass, isAuthorized);
+    }
+
+    @Test(dataProvider = "roles")
+    public void testTopicOperationAsync(String role, String originalRole, boolean shouldPass) throws Exception {
+        boolean isAuthorized = authorizationService.allowTopicOperationAsync(TopicName.get("topic"),
+                TopicOperation.PRODUCE, originalRole, role, null).get();
+        checkResult(shouldPass, isAuthorized);
+    }
+
+    @Test(dataProvider = "roles")
+    public void testNamespacePolicyOperationAsync(String role, String originalRole, boolean shouldPass)
+            throws Exception {
+        boolean isAuthorized = authorizationService.allowNamespacePolicyOperationAsync(
+                NamespaceName.get("public/default"), PolicyName.ALL, PolicyOperation.READ, originalRole, role, null)
+                .get();
+        checkResult(shouldPass, isAuthorized);
+    }
+
+    @Test(dataProvider = "roles")
+    public void testTopicPolicyOperationAsync(String role, String originalRole, boolean shouldPass) throws Exception {
+        boolean isAuthorized = authorizationService.allowTopicPolicyOperationAsync(TopicName.get("topic"),
+                PolicyName.ALL, PolicyOperation.READ, originalRole, role, null).get();
+        checkResult(shouldPass, isAuthorized);
+    }
+}
diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MockAuthorizationProvider.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MockAuthorizationProvider.java
new file mode 100644
index 0000000000000..4c939cbd9723a
--- /dev/null
+++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MockAuthorizationProvider.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.authorization;
+
+import java.io.IOException;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
+import org.apache.pulsar.common.naming.NamespaceName;
+import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.policies.data.AuthAction;
+import org.apache.pulsar.common.policies.data.NamespaceOperation;
+import org.apache.pulsar.common.policies.data.PolicyName;
+import org.apache.pulsar.common.policies.data.PolicyOperation;
+import org.apache.pulsar.common.policies.data.TenantOperation;
+import org.apache.pulsar.common.policies.data.TopicOperation;
+
+/**
+ * Mock implementation of the authorization provider interface used for testing.
+ * A role is authorized if it starts with "pass".
+ */
+public class MockAuthorizationProvider implements AuthorizationProvider {
+
+    private CompletableFuture shouldPass(String role) {
+        return CompletableFuture.completedFuture(role != null && role.startsWith("pass"));
+    }
+
+    @Override
+    public CompletableFuture canProduceAsync(TopicName topicName, String role,
+                                                      AuthenticationDataSource authenticationData) {
+        return shouldPass(role);
+    }
+
+    @Override
+    public CompletableFuture canConsumeAsync(TopicName topicName, String role,
+                                                      AuthenticationDataSource authenticationData, String subscription) {
+        return shouldPass(role);
+    }
+
+    @Override
+    public CompletableFuture canLookupAsync(TopicName topicName, String role,
+                                                     AuthenticationDataSource authenticationData) {
+        return shouldPass(role);
+    }
+
+    @Override
+    public CompletableFuture allowFunctionOpsAsync(NamespaceName namespaceName, String role,
+                                                            AuthenticationDataSource authenticationData) {
+        return shouldPass(role);
+    }
+
+    @Override
+    public CompletableFuture allowSourceOpsAsync(NamespaceName namespaceName, String role,
+                                                          AuthenticationDataSource authenticationData) {
+        return shouldPass(role);
+    }
+
+    @Override
+    public CompletableFuture allowTenantOperationAsync(String tenantName, String role,
+                                                                TenantOperation operation,
+                                                                AuthenticationDataSource authData) {
+        return shouldPass(role);
+    }
+
+    @Override
+    public CompletableFuture allowNamespaceOperationAsync(NamespaceName namespaceName, String role,
+                                                                   NamespaceOperation operation,
+                                                                   AuthenticationDataSource authData) {
+        return shouldPass(role);
+    }
+
+    @Override
+    public CompletableFuture allowNamespacePolicyOperationAsync(NamespaceName namespaceName, PolicyName policy,
+                                                                         PolicyOperation operation, String role,
+                                                                         AuthenticationDataSource authData) {
+        return shouldPass(role);
+    }
+
+    @Override
+    public CompletableFuture allowTopicOperationAsync(TopicName topic, String role, TopicOperation operation,
+                                                               AuthenticationDataSource authData) {
+        return shouldPass(role);
+    }
+
+    @Override
+    public CompletableFuture allowTopicPolicyOperationAsync(TopicName topic, String role, PolicyName policy,
+                                                                     PolicyOperation operation,
+                                                                     AuthenticationDataSource authData) {
+        return shouldPass(role);
+    }
+
+
+    @Override
+    public CompletableFuture allowSinkOpsAsync(NamespaceName namespaceName, String role,
+                                                        AuthenticationDataSource authenticationData) {
+        return null;
+    }
+
+
+    @Override
+    public CompletableFuture grantPermissionAsync(NamespaceName namespace, Set actions, String role,
+                                                        String authDataJson) {
+        return null;
+    }
+
+    @Override
+    public CompletableFuture grantSubscriptionPermissionAsync(NamespaceName namespace, String subscriptionName,
+                                                                    Set roles, String authDataJson) {
+        return null;
+    }
+
+    @Override
+    public CompletableFuture revokeSubscriptionPermissionAsync(NamespaceName namespace, String subscriptionName,
+                                                                     String role, String authDataJson) {
+        return null;
+    }
+
+    @Override
+    public CompletableFuture grantPermissionAsync(TopicName topicName, Set actions, String role,
+                                                        String authDataJson) {
+        return null;
+    }
+
+    @Override
+    public void close() throws IOException {
+
+    }
+}

From ac574d58052fd6aef56175f06fe17017b40e9dc6 Mon Sep 17 00:00:00 2001
From: laminar 
Date: Tue, 21 Mar 2023 07:04:59 +0800
Subject: [PATCH 133/174] [improve][fn] Support processingGuarantees
 "EFFECTIVELY_ONCE" in python function  (#18929)

Signed-off-by: laminar 
---
 .../instance/src/main/python/contextimpl.py   | 13 ++++++
 .../src/main/python/python_instance.py        | 43 ++++++++++++++++---
 .../src/main/python/python_instance_main.py   | 13 +++---
 .../src/test/python/test_python_instance.py   | 12 +++---
 4 files changed, 64 insertions(+), 17 deletions(-)

diff --git a/pulsar-functions/instance/src/main/python/contextimpl.py b/pulsar-functions/instance/src/main/python/contextimpl.py
index 14247e2d647a5..bfe7b23927ba7 100755
--- a/pulsar-functions/instance/src/main/python/contextimpl.py
+++ b/pulsar-functions/instance/src/main/python/contextimpl.py
@@ -89,6 +89,19 @@ def get_message_properties(self):
   def get_current_message_topic_name(self):
     return self.message.topic_name()
 
+  def get_message_sequence_id(self):
+    if not self.get_message_id():
+      return None
+    ledger_id = self.get_message_id().ledger_id()
+    entry_id = self.get_message_id().entry_id()
+    offset = (ledger_id << 28) | entry_id
+    return offset
+
+  def get_message_partition_index(self):
+    if not self.get_message_id():
+      return None
+    return self.get_message_id().partition()
+
   def get_partition_key(self):
     return self.message.partition_key()
 
diff --git a/pulsar-functions/instance/src/main/python/python_instance.py b/pulsar-functions/instance/src/main/python/python_instance.py
index c679288789536..57edbf954b376 100755
--- a/pulsar-functions/instance/src/main/python/python_instance.py
+++ b/pulsar-functions/instance/src/main/python/python_instance.py
@@ -106,6 +106,7 @@ def __init__(self,
     self.execution_thread = None
     self.atmost_once = self.instance_config.function_details.processingGuarantees == Function_pb2.ProcessingGuarantees.Value('ATMOST_ONCE')
     self.atleast_once = self.instance_config.function_details.processingGuarantees == Function_pb2.ProcessingGuarantees.Value('ATLEAST_ONCE')
+    self.effectively_once = self.instance_config.function_details.processingGuarantees == Function_pb2.ProcessingGuarantees.Value('EFFECTIVELY_ONCE')
     self.manual = self.instance_config.function_details.processingGuarantees == Function_pb2.ProcessingGuarantees.Value('MANUAL')
     self.auto_ack = self.instance_config.function_details.autoAck
     self.contextimpl = None
@@ -143,11 +144,15 @@ def run(self):
     if self.instance_config.function_details.source.subscriptionType == Function_pb2.SubscriptionType.Value("FAILOVER"):
       mode = pulsar._pulsar.ConsumerType.Failover
 
+    if self.instance_config.function_details.retainOrdering or \
+      self.instance_config.function_details.processingGuarantees == Function_pb2.ProcessingGuarantees.Value("EFFECTIVELY_ONCE"):
+      mode = pulsar._pulsar.ConsumerType.Failover
+
     position = pulsar._pulsar.InitialPosition.Latest
     if self.instance_config.function_details.source.subscriptionPosition == Function_pb2.SubscriptionPosition.Value("EARLIEST"):
       position = pulsar._pulsar.InitialPosition.Earliest
 
-    subscription_name = self.instance_config.function_details.source.subscriptionName    
+    subscription_name = self.instance_config.function_details.source.subscriptionName
 
     if not (subscription_name and subscription_name.strip()):
       subscription_name = str(self.instance_config.function_details.tenant) + "/" + \
@@ -293,7 +298,7 @@ def actual_execution(self):
 
   def done_producing(self, consumer, orig_message, topic, result, sent_message):
     if result == pulsar.Result.Ok:
-      if self.auto_ack and self.atleast_once:
+      if self.auto_ack and self.atleast_once or self.effectively_once:
         consumer.acknowledge(orig_message)
     else:
       error_msg = "Failed to publish to topic [%s] with error [%s] with src message id [%s]" % (topic, result, orig_message.message_id())
@@ -302,14 +307,27 @@ def done_producing(self, consumer, orig_message, topic, result, sent_message):
       # If producer fails send output then send neg ack for input message back to broker
       consumer.negative_acknowledge(orig_message)
 
-
   def process_result(self, output, msg):
-    if output is not None and self.instance_config.function_details.sink.topic != None and \
+    if output is not None and self.instance_config.function_details.sink.topic is not None and \
             len(self.instance_config.function_details.sink.topic) > 0:
       if self.output_serde is None:
         self.setup_output_serde()
+      if self.effectively_once:
+        if self.contextimpl.get_message_partition_index() is None or \
+                self.contextimpl.get_message_partition_index() >= 0:
+          Log.error("Partitioned topic is not available in effectively_once mode.")
+          raise Exception("Partitioned topic is not available in effectively_once mode.")
+
+        producer_id = self.instance_config.function_details.sink.topic
+        producer = self.contextimpl.publish_producers.get(producer_id)
+        if producer is None:
+          self.setup_producer(producer_name=producer_id)
+          self.contextimpl.publish_producers[producer_id] = self.producer
+          Log.info("Setup producer [%s] successfully in effectively_once mode." % self.producer.producer_name())
+
       if self.producer is None:
         self.setup_producer()
+        Log.info("Setup producer successfully.")
 
       # only serialize function output when output schema is not set
       output_object = output
@@ -318,9 +336,21 @@ def process_result(self, output, msg):
 
       if output_object is not None:
         props = {"__pfn_input_topic__" : str(msg.topic), "__pfn_input_msg_id__" : base64ify(msg.message.message_id().serialize())}
-        self.producer.send_async(output_object, partial(self.done_producing, msg.consumer, msg.message, self.producer.topic()), properties=props)
+        if self.effectively_once:
+          self.producer.send_async(output_object,
+                                   partial(self.done_producing, msg.consumer, msg.message, self.producer.topic()),
+                                   properties=props,
+                                   sequence_id=self.contextimpl.get_message_sequence_id())
+          Log.debug("Send message with sequence ID [%s] using the producer [%s] in effectively_once mode." %
+                    (self.contextimpl.get_message_sequence_id(), self.producer.producer_name()))
+        else:
+          self.producer.send_async(output_object,
+                                   partial(self.done_producing, msg.consumer, msg.message, self.producer.topic()),
+                                   properties=props)
     elif self.auto_ack and self.atleast_once:
       msg.consumer.acknowledge(msg.message)
+    elif self.effectively_once:
+      msg.consumer.acknowledge_cumulative(msg.message)
 
   def setup_output_serde(self):
     if self.instance_config.function_details.sink.serDeClassName != None and \
@@ -332,7 +362,7 @@ def setup_output_serde(self):
       serde_kclass = util.import_class(os.path.dirname(self.user_code), DEFAULT_SERIALIZER)
       self.output_serde = serde_kclass()
 
-  def setup_producer(self):
+  def setup_producer(self, producer_name=None):
     if self.instance_config.function_details.sink.topic != None and \
             len(self.instance_config.function_details.sink.topic) > 0:
       Log.debug("Setting up producer for topic %s" % self.instance_config.function_details.sink.topic)
@@ -366,6 +396,7 @@ def setup_producer(self):
       self.producer = self.pulsar_client.create_producer(
         str(self.instance_config.function_details.sink.topic),
         schema=self.output_schema,
+        producer_name=producer_name,
         block_if_queue_full=True,
         batching_enabled=True,
         batching_type=batch_type,
diff --git a/pulsar-functions/instance/src/main/python/python_instance_main.py b/pulsar-functions/instance/src/main/python/python_instance_main.py
index 2d6520b2e99e9..9a923c7e3a18b 100755
--- a/pulsar-functions/instance/src/main/python/python_instance_main.py
+++ b/pulsar-functions/instance/src/main/python/python_instance_main.py
@@ -164,11 +164,7 @@ def main():
     args.function_details = args.function_details[:-1]
   json_format.Parse(args.function_details, function_details)
 
-  if function_details.processingGuarantees == "EFFECTIVELY_ONCE":
-    print("Python instance current not support EFFECTIVELY_ONCE processing guarantees.")
-    sys.exit(1)
-
-  if function_details.autoAck == False and function_details.processingGuarantees == "ATMOST_ONCE" \
+  if function_details.autoAck is False and function_details.processingGuarantees == "ATMOST_ONCE" \
           or function_details.processingGuarantees == "ATLEAST_ONCE":
     print("When Guarantees == " + function_details.processingGuarantees + ", autoAck must be equal to true, "
           "This is a contradictory configuration, autoAck will be removed later,"
@@ -176,6 +172,13 @@ def main():
           "This is a contradictory configuration, autoAck will be removed later," 
           "Please refer to PIP: https://github.com/apache/pulsar/issues/15560")
     sys.exit(1)
+  if function_details.processingGuarantees == Function_pb2.ProcessingGuarantees.Value('EFFECTIVELY_ONCE'):
+    if len(function_details.source.inputSpecs.keys()) != 1 or function_details.sink.topic == "":
+      print("When Guarantees == EFFECTIVELY_ONCE you need to ensure that the following pre-requisites have been met:"
+            "1. deduplication is enabled"
+            "2. set ProcessingGuarantees to EFFECTIVELY_ONCE"
+            "3. the function has only one source topic and one sink topic (both are non-partitioned)")
+      sys.exit(1)
   if os.path.splitext(str(args.py))[1] == '.whl':
     if args.install_usercode_dependencies:
       cmd = "pip install -t %s" % os.path.dirname(os.path.abspath(str(args.py)))
diff --git a/pulsar-functions/instance/src/test/python/test_python_instance.py b/pulsar-functions/instance/src/test/python/test_python_instance.py
index bbd6ca1fe42f1..975ddfd90288d 100644
--- a/pulsar-functions/instance/src/test/python/test_python_instance.py
+++ b/pulsar-functions/instance/src/test/python/test_python_instance.py
@@ -25,7 +25,7 @@
 sys.modules['prometheus_client'] = Mock()
 
 from contextimpl import ContextImpl
-from python_instance import InstanceConfig
+from python_instance import PythonInstance, InstanceConfig
 from pulsar import Message
 
 import Function_pb2
@@ -49,14 +49,14 @@ def test_context_publish(self):
     function_id = 'test_function_id'
     function_version = 'test_function_version'
     function_details = Function_pb2.FunctionDetails()
-    max_buffered_tuples = 100;
+    max_buffered_tuples = 100
     instance_config = InstanceConfig(instance_id, function_id, function_version, function_details, max_buffered_tuples)
     logger = log.Log
     pulsar_client = Mock()
     producer = Mock()
     producer.send_async = Mock(return_value=None)
     pulsar_client.create_producer = Mock(return_value=producer)
-    user_code=__file__
+    user_code = __file__
     consumers = None
     context_impl = ContextImpl(instance_config, logger, pulsar_client, user_code, consumers, None, None, None, None)
 
@@ -77,11 +77,11 @@ def test_context_ack_partitionedtopic(self):
     function_id = 'test_function_id'
     function_version = 'test_function_version'
     function_details = Function_pb2.FunctionDetails()
-    max_buffered_tuples = 100;
+    max_buffered_tuples = 100
     instance_config = InstanceConfig(instance_id, function_id, function_version, function_details, max_buffered_tuples)
     logger = log.Log
     pulsar_client = Mock()
-    user_code=__file__
+    user_code = __file__
     consumer = Mock()
     consumer.acknowledge = Mock(return_value=None)
     consumers = {"mytopic" : consumer}
@@ -89,4 +89,4 @@ def test_context_ack_partitionedtopic(self):
     context_impl.ack("test_message_id", "mytopic-partition-3")
 
     args, kwargs = consumer.acknowledge.call_args
-    self.assertEqual(args[0], "test_message_id")
\ No newline at end of file
+    self.assertEqual(args[0], "test_message_id")

From 03f8b806ec177cdf6a5a82193a3fc82557000989 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= 
Date: Tue, 21 Mar 2023 03:14:33 +0100
Subject: [PATCH 134/174] [fix][io] KCA sink: handle null values with
 KeyValue schema (#19861)

Co-authored-by: Andrey Yegorov 
---
 .../io/kafka/connect/KafkaConnectSink.java    |  4 +-
 .../connect/schema/KafkaConnectData.java      |  9 +++
 .../schema/PulsarSchemaToKafkaSchema.java     | 80 ++++++++++++++++++-
 .../kafka/connect/KafkaConnectSinkTest.java   | 54 +++++++++++++
 4 files changed, 144 insertions(+), 3 deletions(-)

diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java
index efbad2ef47ae0..06f66f60380d9 100644
--- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java
+++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java
@@ -437,8 +437,8 @@ protected SinkRecord toSinkRecord(Record sourceRecord) {
 
             if (nativeObject instanceof KeyValue) {
                 KeyValue kv = (KeyValue) nativeObject;
-                key = KafkaConnectData.getKafkaConnectData(kv.getKey(), keySchema);
-                value = KafkaConnectData.getKafkaConnectData(kv.getValue(), valueSchema);
+                key = KafkaConnectData.getKafkaConnectDataFromSchema(kv.getKey(), keySchema);
+                value = KafkaConnectData.getKafkaConnectDataFromSchema(kv.getValue(), valueSchema);
             } else if (nativeObject != null) {
                 throw new IllegalStateException("Cannot extract KeyValue data from " + nativeObject.getClass());
             } else {
diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaConnectData.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaConnectData.java
index 757241d411034..a308ef01ddcf1 100644
--- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaConnectData.java
+++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaConnectData.java
@@ -54,6 +54,14 @@ private static List arrayToList(Object nativeObject, Schema kafkaValueSc
         return out;
     }
 
+
+    public static Object getKafkaConnectDataFromSchema(Object nativeObject, Schema kafkaSchema) {
+        if (kafkaSchema != null && nativeObject == null) {
+            return null;
+        }
+        return getKafkaConnectData(nativeObject, kafkaSchema);
+    }
+
     @SuppressWarnings("unchecked")
     public static Object getKafkaConnectData(Object nativeObject, Schema kafkaSchema) {
         if (kafkaSchema == null) {
@@ -380,6 +388,7 @@ private static Object defaultOrThrow(Schema kafkaSchema) {
         if (kafkaSchema.isOptional()) {
             return null;
         }
+
         throw new DataException("Invalid null value for required " + kafkaSchema.type() + " field");
     }
 }
diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/PulsarSchemaToKafkaSchema.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/PulsarSchemaToKafkaSchema.java
index 2eb6573374cdb..faf28585e8aed 100644
--- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/PulsarSchemaToKafkaSchema.java
+++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/PulsarSchemaToKafkaSchema.java
@@ -26,11 +26,14 @@
 import com.google.common.util.concurrent.UncheckedExecutionException;
 import io.confluent.connect.avro.AvroData;
 import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.kafka.connect.data.Date;
 import org.apache.kafka.connect.data.Decimal;
+import org.apache.kafka.connect.data.Field;
 import org.apache.kafka.connect.data.Schema;
 import org.apache.kafka.connect.data.SchemaBuilder;
 import org.apache.kafka.connect.data.Time;
@@ -41,6 +44,76 @@
 
 @Slf4j
 public class PulsarSchemaToKafkaSchema {
+
+    private static class OptionalForcingSchema implements Schema {
+
+        Schema sourceSchema;
+
+        public OptionalForcingSchema(Schema sourceSchema) {
+            this.sourceSchema = sourceSchema;
+        }
+
+        @Override
+        public Type type() {
+            return sourceSchema.type();
+        }
+
+        @Override
+        public boolean isOptional() {
+            return true;
+        }
+
+        @Override
+        public Object defaultValue() {
+            return sourceSchema.defaultValue();
+        }
+
+        @Override
+        public String name() {
+            return sourceSchema.name();
+        }
+
+        @Override
+        public Integer version() {
+            return sourceSchema.version();
+        }
+
+        @Override
+        public String doc() {
+            return sourceSchema.doc();
+        }
+
+        @Override
+        public Map parameters() {
+            return sourceSchema.parameters();
+        }
+
+        @Override
+        public Schema keySchema() {
+            return sourceSchema.keySchema();
+        }
+
+        @Override
+        public Schema valueSchema() {
+            return sourceSchema.valueSchema();
+        }
+
+        @Override
+        public List fields() {
+            return sourceSchema.fields();
+        }
+
+        @Override
+        public Field field(String s) {
+            return sourceSchema.field(s);
+        }
+
+        @Override
+        public Schema schema() {
+            return sourceSchema.schema();
+        }
+    }
+
     private static final ImmutableMap pulsarSchemaTypeToKafkaSchema;
     private static final ImmutableSet kafkaLogicalSchemas;
     private static final AvroData avroData = new AvroData(1000);
@@ -80,6 +153,11 @@ private static org.apache.avro.Schema parseAvroSchema(String schemaJson) {
         return parser.parse(schemaJson);
     }
 
+    public static Schema getOptionalKafkaConnectSchema(org.apache.pulsar.client.api.Schema pulsarSchema) {
+        Schema s = getKafkaConnectSchema(pulsarSchema);
+        return new OptionalForcingSchema(s);
+    }
+
     public static Schema getKafkaConnectSchema(org.apache.pulsar.client.api.Schema pulsarSchema) {
         if (pulsarSchema == null || pulsarSchema.getSchemaInfo() == null) {
             throw logAndThrowOnUnsupportedSchema(pulsarSchema, "Schema is required.", null);
@@ -122,7 +200,7 @@ public static Schema getKafkaConnectSchema(org.apache.pulsar.client.api.Schema p
                 if (pulsarSchema.getSchemaInfo().getType() == SchemaType.KEY_VALUE) {
                     KeyValueSchema kvSchema = (KeyValueSchema) pulsarSchema;
                     return SchemaBuilder.map(getKafkaConnectSchema(kvSchema.getKeySchema()),
-                                             getKafkaConnectSchema(kvSchema.getValueSchema()))
+                                    getOptionalKafkaConnectSchema(kvSchema.getValueSchema()))
                                 .build();
                 }
                 org.apache.avro.Schema avroSchema = parseAvroSchema(
diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java
index 567562d338b98..e9d454ed2fd5a 100644
--- a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java
+++ b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java
@@ -51,6 +51,9 @@
 import org.apache.pulsar.client.api.schema.Field;
 import org.apache.pulsar.client.api.schema.GenericObject;
 import org.apache.pulsar.client.api.schema.GenericRecord;
+import org.apache.pulsar.client.api.schema.GenericSchema;
+import org.apache.pulsar.client.api.schema.RecordSchemaBuilder;
+import org.apache.pulsar.client.api.schema.SchemaBuilder;
 import org.apache.pulsar.client.api.schema.SchemaDefinition;
 import org.apache.pulsar.client.impl.BatchMessageIdImpl;
 import org.apache.pulsar.client.impl.MessageIdImpl;
@@ -60,6 +63,7 @@
 import org.apache.pulsar.client.impl.schema.JSONSchema;
 import org.apache.pulsar.client.impl.schema.SchemaInfoImpl;
 import org.apache.pulsar.client.impl.schema.generic.GenericAvroRecord;
+import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema;
 import org.apache.pulsar.client.util.MessageIdUtils;
 import org.apache.pulsar.common.schema.KeyValue;
 import org.apache.pulsar.common.schema.SchemaInfo;
@@ -734,6 +738,56 @@ public void schemaKeyValueSchemaTest() throws Exception {
         Assert.assertEquals(key, 11);
     }
 
+    @Test
+    public void schemaKeyValueSchemaNullValueTest() throws Exception {
+        RecordSchemaBuilder builder = SchemaBuilder
+                .record("test");
+        builder.field("test").type(SchemaType.STRING);
+        GenericSchema schema = GenericAvroSchema.of(builder.build(SchemaType.AVRO));
+        KeyValue kv = new KeyValue<>(11, null);
+        SinkRecord sinkRecord = recordSchemaTest(kv, Schema.KeyValue(Schema.INT32, schema), 11,
+                "INT32", null, "STRUCT");
+        Assert.assertNull(sinkRecord.value());
+        int key = (int) sinkRecord.key();
+        Assert.assertEquals(key, 11);
+    }
+
+    @Test
+    public void schemaKeyValueSchemaNullValueNoUnwrapTest() throws Exception {
+        props.put("unwrapKeyValueIfAvailable", "false");
+        JSONSchema jsonSchema = JSONSchema
+                .of(SchemaDefinition.builder()
+                        .withPojo(PulsarSchemaToKafkaSchemaTest.StructWithAnnotations.class)
+                        .withAlwaysAllowNull(true)
+                        .build());
+        KeyValue kv = new KeyValue<>(11, null);
+        Map expected = new HashMap();
+        expected.put("11", null);
+        SinkRecord sinkRecord = recordSchemaTest(kv, Schema.KeyValue(Schema.INT32, jsonSchema), "key",
+                "STRING", expected, "MAP");
+        Assert.assertNull(((Map)sinkRecord.value()).get(11));
+        String key =(String)sinkRecord.key();
+        Assert.assertEquals(key, "key");
+    }
+
+    @Test
+    public void schemaKeyValueSchemaNullValueNoUnwrapTestAvro() throws Exception {
+        props.put("unwrapKeyValueIfAvailable", "false");
+        RecordSchemaBuilder builder = SchemaBuilder
+                .record("test");
+        builder.property("op", "test");
+        builder.field("test").type(SchemaType.STRING);
+        GenericSchema schema = GenericAvroSchema.of(builder.build(SchemaType.AVRO));
+        KeyValue kv = new KeyValue<>(11, null);
+        Map expected = new HashMap();
+        expected.put("11", null);
+        SinkRecord sinkRecord = recordSchemaTest(kv, Schema.KeyValue(Schema.INT32, schema), "key",
+                "STRING", expected, "MAP");
+        Assert.assertNull(((Map)sinkRecord.value()).get(11));
+        String key =(String)sinkRecord.key();
+        Assert.assertEquals(key, "key");
+    }
+
     @Test
     public void kafkaLogicalTypesTimestampTest() {
         Schema schema = new TestSchema(SchemaInfoImpl.builder()

From 1e44ba179bfc189d22dff046a1887e12842b0851 Mon Sep 17 00:00:00 2001
From: Zike Yang 
Date: Tue, 21 Mar 2023 16:17:41 +0800
Subject: [PATCH 135/174] [fix][doc] The `messageIds` in `onNegativeAcksSend`
 shouldn't be null (#19852)

### Motivation

The `onNegativeAcksSend` is only called by:
https://github.com/apache/pulsar/blob/80c5791b87482bee3392308ecef45f455f8de885/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NegativeAcksTracker.java#L83

It will not be null. The statement `null if acknowledge fail.` is incorrect.

### Modifications

* Remove the incorrect statement and refine the doc.

Signed-off-by: Zike Yang 
Co-authored-by: Jun Ma <60642177+momo-jun@users.noreply.github.com>
---
 .../java/org/apache/pulsar/client/api/ConsumerInterceptor.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerInterceptor.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerInterceptor.java
index d6d91cd88500b..be2f9b0f10826 100644
--- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerInterceptor.java
+++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerInterceptor.java
@@ -107,7 +107,7 @@ public interface ConsumerInterceptor extends AutoCloseable {
      * 
Any exception thrown by this method will be ignored by the caller.
      *
      * @param consumer the consumer which contains the interceptor
-     * @param messageIds message to ack, null if acknowledge fail.
+     * @param messageIds the set of message ids to negatively ack
      */
     void onNegativeAcksSend(Consumer consumer, Set messageIds);
 

From 905e8ef730c96cbb43ad5c892f1dbfc3cd1521cc Mon Sep 17 00:00:00 2001
From: Qiang Zhao 
Date: Tue, 21 Mar 2023 21:31:22 +0800
Subject: [PATCH 136/174] [improve][meta] Make session notification to be
 async. (#19869)

---
 .../apache/pulsar/common/util/FutureUtil.java | 26 +++++++++++++++++++
 .../coordination/impl/LeaderElectionImpl.java | 23 ++++++++--------
 .../coordination/impl/LockManagerImpl.java    | 22 +++++++---------
 3 files changed, 48 insertions(+), 23 deletions(-)

diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java
index 531769a1e452b..2b082b4a7899b 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java
@@ -28,6 +28,7 @@
 import java.util.concurrent.CompletionException;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executor;
+import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
@@ -36,6 +37,7 @@
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
+import javax.annotation.Nonnull;
 import javax.annotation.concurrent.ThreadSafe;
 
 /**
@@ -238,6 +240,30 @@ public static  CompletableFuture addTimeoutHandling(CompletableFuture f
         return future;
     }
 
+    /**
+     * @throws RejectedExecutionException if this task cannot be accepted for execution
+     * @throws NullPointerException if one of params is null
+     */
+    public static  @Nonnull CompletableFuture composeAsync(Supplier> futureSupplier,
+                                                                 Executor executor) {
+        Objects.requireNonNull(futureSupplier);
+        Objects.requireNonNull(executor);
+        final CompletableFuture future = new CompletableFuture<>();
+        try {
+            executor.execute(() -> futureSupplier.get().whenComplete((result, error) -> {
+                if (error != null) {
+                    future.completeExceptionally(error);
+                    return;
+                }
+                future.complete(result);
+            }));
+        } catch (RejectedExecutionException ex) {
+            future.completeExceptionally(ex);
+        }
+        return future;
+    }
+
+
     /**
      * Creates a low-overhead timeout exception which is performance optimized to minimize allocations
      * and cpu consumption. It sets the stacktrace of the exception to the given source class and
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java
index c1121b1309c2c..11ae62226e7cd 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java
@@ -24,7 +24,6 @@
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
-import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
@@ -32,6 +31,7 @@
 import lombok.extern.slf4j.Slf4j;
 import org.apache.bookkeeper.common.concurrent.FutureUtils;
 import org.apache.bookkeeper.common.util.SafeRunnable;
+import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.metadata.api.GetResult;
 import org.apache.pulsar.metadata.api.MetadataCache;
 import org.apache.pulsar.metadata.api.MetadataCacheConfig;
@@ -62,6 +62,7 @@ class LeaderElectionImpl implements LeaderElection {
     private Optional proposedValue;
 
     private final ScheduledExecutorService executor;
+    private final FutureUtil.Sequencer sequencer;
 
     private enum InternalState {
         Init, ElectionInProgress, LeaderIsPresent, Closed
@@ -85,7 +86,7 @@ private enum InternalState {
         this.internalState = InternalState.Init;
         this.stateChangesListener = stateChangesListener;
         this.executor = executor;
-
+        this.sequencer = FutureUtil.Sequencer.create();
         store.registerListener(this::handlePathNotification);
         store.registerSessionListener(this::handleSessionNotification);
         updateCachedValueFuture = executor.scheduleWithFixedDelay(SafeRunnable.safeRun(this::getLeaderValue),
@@ -277,18 +278,18 @@ public Optional getLeaderValueIfPresent() {
 
     private void handleSessionNotification(SessionEvent event) {
         // Ensure we're only processing one session event at a time.
-        executor.execute(SafeRunnable.safeRun(() -> {
+        sequencer.sequential(() -> FutureUtil.composeAsync(() -> {
             if (event == SessionEvent.SessionReestablished) {
                 log.info("Revalidating leadership for {}", path);
-
-                try {
-                    LeaderElectionState les = elect().get();
-                    log.info("Resynced leadership for {} - State: {}", path, les);
-                } catch (ExecutionException | InterruptedException e) {
-                    log.warn("Failure when processing session event", e);
-                }
+                return elect().thenAccept(leaderState -> {
+                    log.info("Resynced leadership for {} - State: {}", path, leaderState);
+                }).exceptionally(ex -> {
+                    log.warn("Failure when processing session event", ex);
+                    return null;
+                });
             }
-        }));
+            return CompletableFuture.completedFuture(null);
+        }, executor));
     }
 
     private void handlePathNotification(Notification notification) {
diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java
index f5e7e0528bd7b..4da6b7998a0c4 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LockManagerImpl.java
@@ -27,11 +27,9 @@
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.bookkeeper.common.util.SafeRunnable;
 import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.metadata.api.MetadataCache;
 import org.apache.pulsar.metadata.api.MetadataSerde;
@@ -53,6 +51,7 @@ class LockManagerImpl implements LockManager {
     private final MetadataStoreExtended store;
     private final MetadataCache cache;
     private final MetadataSerde serde;
+    private final FutureUtil.Sequencer sequencer;
     private final ExecutorService executor;
 
     private enum State {
@@ -72,6 +71,7 @@ private enum State {
         this.cache = store.getMetadataCache(serde);
         this.serde = serde;
         this.executor = executor;
+        this.sequencer = FutureUtil.Sequencer.create();
         store.registerSessionListener(this::handleSessionEvent);
         store.registerListener(this::handleDataNotification);
     }
@@ -118,9 +118,8 @@ public CompletableFuture> acquireLock(String path, T value) {
     private void handleSessionEvent(SessionEvent se) {
         // We want to make sure we're processing one event at a time and that we're done with one event before going
         // for the next one.
-        executor.execute(SafeRunnable.safeRun(() -> {
-            List> futures = new ArrayList<>();
-
+        sequencer.sequential(() -> FutureUtil.composeAsync(() -> {
+            final List> futures = new ArrayList<>();
             if (se == SessionEvent.SessionReestablished) {
                 log.info("Metadata store session has been re-established. Revalidating all the existing locks.");
                 for (ResourceLockImpl lock : locks.values()) {
@@ -133,13 +132,12 @@ private void handleSessionEvent(SessionEvent se) {
                     futures.add(lock.revalidateIfNeededAfterReconnection());
                 }
             }
-
-            try {
-                FutureUtil.waitForAll(futures).get();
-            } catch (ExecutionException | InterruptedException e) {
-                log.warn("Failure when processing session event", e);
-            }
-        }));
+            return FutureUtil.waitForAll(futures)
+                    .exceptionally(ex -> {
+                        log.warn("Failure when processing session event", ex);
+                        return null;
+                    });
+        }, executor));
     }
 
     private void handleDataNotification(Notification n) {

From a50ecda11f947e5120cfb01adbfd46f22b76f1f2 Mon Sep 17 00:00:00 2001
From: Nihar Rathod 
Date: Tue, 21 Mar 2023 19:30:54 +0530
Subject: [PATCH 137/174] [fix][test]
 PulsarStandaloneTest.testMetadataInitialization (#19842)

---
 .../test/resources/configurations/standalone_no_client_auth.conf | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pulsar-broker/src/test/resources/configurations/standalone_no_client_auth.conf b/pulsar-broker/src/test/resources/configurations/standalone_no_client_auth.conf
index 573d6cdba9933..d9411e655ad5b 100644
--- a/pulsar-broker/src/test/resources/configurations/standalone_no_client_auth.conf
+++ b/pulsar-broker/src/test/resources/configurations/standalone_no_client_auth.conf
@@ -29,3 +29,4 @@ authenticationEnabled=true
 authenticationProviders=org.apache.pulsar.MockTokenAuthenticationProvider
 brokerClientAuthenticationPlugin=
 brokerClientAuthenticationParameters=
+loadBalancerOverrideBrokerNicSpeedGbps=2
\ No newline at end of file

From 30d2469086fea989ac8baf059df8e69c66a68d89 Mon Sep 17 00:00:00 2001
From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com>
Date: Tue, 21 Mar 2023 07:33:00 -0700
Subject: [PATCH 138/174] [improve][broker] PIP-192 Excluded bundles with
 isolation policy or anti-affinity-group policy from topk load bundles
 (#19742)

### Motivation

Raising a PR to implement https://github.com/apache/pulsar/issues/16691.

Bundles with isolation policies or anti-affinity-group policies are not ideal targets to auto-unload as destination brokers are limited.

### Modifications
This PR
- Excluded bundles with isolation policy or anti-affinity-group policy from topk load bundles
- Introduced a config `loadBalancerSheddingBundlesWithPoliciesEnabled ` to control this behavior.
---
 .../pulsar/broker/ServiceConfiguration.java   |  10 +
 .../extensions/models/TopKBundles.java        |  79 ++++++--
 .../AntiAffinityGroupPolicyHelper.java        |  31 +--
 .../reporter/TopBundleLoadDataReporter.java   |   2 +-
 .../extensions/scheduler/TransferShedder.java |   7 +
 ...tiAffinityNamespaceGroupExtensionTest.java |  30 +++
 .../extensions/models/TopKBundlesTest.java    | 182 ++++++++++++++++--
 .../TopBundleLoadDataReporterTest.java        |  39 +++-
 .../scheduler/TransferShedderTest.java        |  42 ++--
 9 files changed, 356 insertions(+), 66 deletions(-)

diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index ff242888ae0b6..ba82667690dc5 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -2600,6 +2600,16 @@ The delayed message index time step(in seconds) in per bucket snapshot segment,
     )
     private long loadBalancerServiceUnitStateCleanUpDelayTimeInSeconds = 604800;
 
+    @FieldContext(
+            category = CATEGORY_LOAD_BALANCER,
+            dynamic = true,
+            doc = "Option to automatically unload namespace bundles with affinity(isolation) "
+                    + "or anti-affinity group policies."
+                    + "Such bundles are not ideal targets to auto-unload as destination brokers are limited."
+                    + "(only used in load balancer extension logics)"
+    )
+    private boolean loadBalancerSheddingBundlesWithPoliciesEnabled = false;
+
     /**** --- Replication. --- ****/
     @FieldContext(
         category = CATEGORY_REPLICATION,
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java
index c189005b9539c..d49361741bec4 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java
@@ -24,10 +24,14 @@
 import java.util.Map;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
-import lombok.NoArgsConstructor;
 import lombok.ToString;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData;
+import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared;
+import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies;
 import org.apache.pulsar.common.naming.NamespaceName;
+import org.apache.pulsar.metadata.api.MetadataStoreException;
 import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats;
 
 /**
@@ -36,7 +40,7 @@
 @Getter
 @ToString
 @EqualsAndHashCode
-@NoArgsConstructor
+@Slf4j
 public class TopKBundles {
 
     // temp array for sorting
@@ -44,6 +48,15 @@ public class TopKBundles {
 
     private final TopBundlesLoadData loadData = new TopBundlesLoadData();
 
+    private final PulsarService pulsar;
+
+    private final SimpleResourceAllocationPolicies allocationPolicies;
+
+    public TopKBundles(PulsarService pulsar) {
+        this.pulsar = pulsar;
+        this.allocationPolicies = new SimpleResourceAllocationPolicies(pulsar);
+    }
+
     /**
      * Update the topK bundles from the input bundleStats.
      *
@@ -52,26 +65,35 @@ public class TopKBundles {
      */
     public void update(Map bundleStats, int topk) {
         arr.clear();
-        for (var etr : bundleStats.entrySet()) {
-            if (etr.getKey().startsWith(NamespaceName.SYSTEM_NAMESPACE.toString())) {
-                continue;
+        try {
+            var isLoadBalancerSheddingBundlesWithPoliciesEnabled =
+                    pulsar.getConfiguration().isLoadBalancerSheddingBundlesWithPoliciesEnabled();
+            for (var etr : bundleStats.entrySet()) {
+                String bundle = etr.getKey();
+                if (bundle.startsWith(NamespaceName.SYSTEM_NAMESPACE.toString())) {
+                    continue;
+                }
+                if (!isLoadBalancerSheddingBundlesWithPoliciesEnabled && hasPolicies(bundle)) {
+                    continue;
+                }
+                arr.add(etr);
             }
-            arr.add(etr);
-        }
-        var topKBundlesLoadData = loadData.getTopBundlesLoadData();
-        topKBundlesLoadData.clear();
-        if (arr.isEmpty()) {
-            return;
-        }
-        topk = Math.min(topk, arr.size());
-        partitionSort(arr, topk);
+            var topKBundlesLoadData = loadData.getTopBundlesLoadData();
+            topKBundlesLoadData.clear();
+            if (arr.isEmpty()) {
+                return;
+            }
+            topk = Math.min(topk, arr.size());
+            partitionSort(arr, topk);
 
-        for (int i = 0; i < topk; i++) {
-            var etr = arr.get(i);
-            topKBundlesLoadData.add(
-                    new TopBundlesLoadData.BundleLoadData(etr.getKey(), (NamespaceBundleStats) etr.getValue()));
+            for (int i = 0; i < topk; i++) {
+                var etr = arr.get(i);
+                topKBundlesLoadData.add(
+                        new TopBundlesLoadData.BundleLoadData(etr.getKey(), (NamespaceBundleStats) etr.getValue()));
+            }
+        } finally {
+            arr.clear();
         }
-        arr.clear();
     }
 
     static void partitionSort(List> arr, int k) {
@@ -109,4 +131,23 @@ static void partitionSort(List> arr, int
         }
         Collections.sort(arr.subList(0, end), (a, b) -> b.getValue().compareTo(a.getValue()));
     }
+
+    private boolean hasPolicies(String bundle) {
+        NamespaceName namespace = NamespaceName.get(LoadManagerShared.getNamespaceNameFromBundleName(bundle));
+        if (allocationPolicies.areIsolationPoliciesPresent(namespace)) {
+            return true;
+        }
+
+        try {
+            var antiAffinityGroupOptional =
+                    LoadManagerShared.getNamespaceAntiAffinityGroup(pulsar, namespace.toString());
+            if (antiAffinityGroupOptional.isPresent()) {
+                return true;
+            }
+        } catch (MetadataStoreException e) {
+            log.error("Failed to get localPolicies for bundle:{}.", bundle, e);
+            throw new RuntimeException(e);
+        }
+        return false;
+    }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java
index c8332a1d7b5e6..69e3302bebd50 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java
@@ -54,32 +54,37 @@ public boolean canUnload(
             String bundle,
             String srcBroker,
             Optional dstBroker) {
+
         try {
             var antiAffinityGroupOptional = LoadManagerShared.getNamespaceAntiAffinityGroup(
                     pulsar, LoadManagerShared.getNamespaceNameFromBundleName(bundle));
-            if (antiAffinityGroupOptional.isPresent()) {
+            if (antiAffinityGroupOptional.isEmpty()) {
+                return true;
+            }
 
-                // copy to retain the input brokers
-                Map candidates = new HashMap<>(brokers);
+            // bundle has anti-affinityGroup
+            if (!pulsar.getConfiguration().isLoadBalancerSheddingBundlesWithPoliciesEnabled()) {
+                return false;
+            }
 
-                filter(candidates, bundle);
+            // copy to retain the input brokers
+            Map candidates = new HashMap<>(brokers);
 
-                candidates.remove(srcBroker);
+            filter(candidates, bundle);
 
-                // unload case
-                if (dstBroker.isEmpty()) {
-                    return !candidates.isEmpty();
-                }
+            candidates.remove(srcBroker);
 
-                // transfer case
-                return candidates.containsKey(dstBroker.get());
+            // unload case
+            if (dstBroker.isEmpty()) {
+                return !candidates.isEmpty();
             }
+
+            // transfer case
+            return candidates.containsKey(dstBroker.get());
         } catch (MetadataStoreException e) {
             log.error("Failed to check unload candidates. Assumes that bundle:{} cannot unload ", bundle, e);
             return false;
         }
-
-        return true;
     }
 
     public void listenFailureDomainUpdate() {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java
index 59e328fc2be80..cc1a39add5d4e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java
@@ -48,7 +48,7 @@ public TopBundleLoadDataReporter(PulsarService pulsar,
         this.lookupServiceAddress = lookupServiceAddress;
         this.bundleLoadDataStore = bundleLoadDataStore;
         this.lastBundleStatsUpdatedAt = 0;
-        this.topKBundles = new TopKBundles();
+        this.topKBundles = new TopKBundles(pulsar);
     }
 
     @Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java
index 3c67479bcc7cb..effddaf9c4159 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java
@@ -456,6 +456,7 @@ private boolean isTransferable(LoadManagerContext context,
         if (pulsar == null || allocationPolicies == null) {
             return true;
         }
+
         String namespace = LoadManagerShared.getNamespaceNameFromBundleName(bundle);
         final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle);
         NamespaceBundle namespaceBundle =
@@ -491,6 +492,12 @@ private boolean canTransferWithIsolationPoliciesToBroker(LoadManagerContext cont
                 || !allocationPolicies.areIsolationPoliciesPresent(namespaceBundle.getNamespaceObject())) {
             return true;
         }
+
+        // bundle has isolation policies.
+        if (!context.brokerConfiguration().isLoadBalancerSheddingBundlesWithPoliciesEnabled()) {
+            return false;
+        }
+
         boolean transfer = context.brokerConfiguration().isLoadBalancerTransferEnabled();
         Set candidates = isolationPoliciesHelper.applyIsolationPolicies(availableBrokers, namespaceBundle);
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java
index 32822c0f5b524..16b87195b1511 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java
@@ -105,6 +105,8 @@ protected void verifyLoadSheddingWithAntiAffinityNamespace(String namespace, Str
             brokersCopy.remove(srcBroker);
             var dstBroker = brokersCopy.entrySet().iterator().next().getKey();
 
+            // test setLoadBalancerSheddingBundlesWithPoliciesEnabled = true
+            conf.setLoadBalancerSheddingBundlesWithPoliciesEnabled(true);
             assertTrue(antiAffinityGroupPolicyHelper.canUnload(brokers,
                     "not-enabled-" + namespace + "/" + bundle,
                     srcBroker, Optional.of(dstBroker)));
@@ -128,6 +130,34 @@ protected void verifyLoadSheddingWithAntiAffinityNamespace(String namespace, Str
             assertFalse(antiAffinityGroupPolicyHelper.canUnload(brokers,
                     namespaceBundle,
                     dstBroker, Optional.empty()));
+
+            // test setLoadBalancerSheddingBundlesWithPoliciesEnabled = false
+            conf.setLoadBalancerSheddingBundlesWithPoliciesEnabled(false);
+            assertTrue(antiAffinityGroupPolicyHelper.canUnload(brokers,
+                    "not-enabled-" + namespace + "/" + bundle,
+                    srcBroker, Optional.of(dstBroker)));
+
+            assertTrue(antiAffinityGroupPolicyHelper.canUnload(brokers,
+                    "not-enabled-" + namespace + "/" + bundle,
+                    srcBroker, Optional.empty()));
+
+            assertFalse(antiAffinityGroupPolicyHelper.canUnload(brokers,
+                    namespaceBundle,
+                    srcBroker, Optional.of(dstBroker)));
+
+            assertFalse(antiAffinityGroupPolicyHelper.canUnload(brokers,
+                    namespaceBundle,
+                    dstBroker, Optional.of(srcBroker)));
+
+            assertFalse(antiAffinityGroupPolicyHelper.canUnload(brokers,
+                    namespaceBundle,
+                    srcBroker, Optional.empty()));
+
+            assertFalse(antiAffinityGroupPolicyHelper.canUnload(brokers,
+                    namespaceBundle,
+                    dstBroker, Optional.empty()));
+
+
         } catch (Exception e) {
             throw new RuntimeException(e);
         }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundlesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundlesTest.java
index d759dd016955a..9b42163bd664b 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundlesTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundlesTest.java
@@ -18,65 +18,221 @@
  */
 package org.apache.pulsar.broker.loadbalance.extensions.models;
 
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertNull;
-import static org.testng.Assert.assertTrue;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Random;
+import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared;
+import org.apache.pulsar.broker.resources.LocalPoliciesResources;
+import org.apache.pulsar.broker.resources.NamespaceResources;
+import org.apache.pulsar.broker.resources.PulsarResources;
+import org.apache.pulsar.common.naming.NamespaceName;
+import org.apache.pulsar.common.policies.data.AutoFailoverPolicyData;
+import org.apache.pulsar.common.policies.data.AutoFailoverPolicyType;
+import org.apache.pulsar.common.policies.data.LocalPolicies;
+import org.apache.pulsar.common.policies.data.NamespaceIsolationData;
+import org.apache.pulsar.common.policies.data.NamespaceIsolationDataImpl;
+import org.apache.pulsar.common.policies.impl.NamespaceIsolationPolicies;
+import org.apache.pulsar.metadata.api.MetadataStoreException;
 import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats;
+import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
 @Test(groups = "broker")
 public class TopKBundlesTest {
+    private PulsarService pulsar;
+    private ServiceConfiguration configuration;
+    private NamespaceResources.IsolationPolicyResources isolationPolicyResources;
+    private PulsarResources pulsarResources;
+    private LocalPoliciesResources localPoliciesResources;
 
+    String bundle1 = "my-tenant/my-namespace1/0x00000000_0x0FFFFFFF";
+    String bundle2 = "my-tenant/my-namespace2/0x00000000_0x0FFFFFFF";
+    String bundle3 = "my-tenant/my-namespace3/0x00000000_0x0FFFFFFF";
+    String bundle4 = "my-tenant/my-namespace4/0x00000000_0x0FFFFFFF";
+
+    @BeforeMethod
+    public void init() throws MetadataStoreException {
+        pulsar = mock(PulsarService.class);
+        configuration = new ServiceConfiguration();
+        doReturn(configuration).when(pulsar).getConfiguration();
+        configuration.setLoadBalancerSheddingBundlesWithPoliciesEnabled(false);
+        pulsarResources = mock(PulsarResources.class);
+        var namespaceResources = mock(NamespaceResources.class);
+
+        isolationPolicyResources = mock(NamespaceResources.IsolationPolicyResources.class);
+        doReturn(pulsarResources).when(pulsar).getPulsarResources();
+        doReturn(namespaceResources).when(pulsarResources).getNamespaceResources();
+        doReturn(isolationPolicyResources).when(namespaceResources).getIsolationPolicies();
+        doReturn(Optional.empty()).when(isolationPolicyResources).getIsolationDataPolicies(any());
+
+        localPoliciesResources = mock(LocalPoliciesResources.class);
+        doReturn(localPoliciesResources).when(pulsarResources).getLocalPolicies();
+        doReturn(Optional.empty()).when(localPoliciesResources).getLocalPolicies(any());
+
+    }
     @Test
     public void testTopBundlesLoadData() {
         Map bundleStats = new HashMap<>();
-        var topKBundles = new TopKBundles();
+        var topKBundles = new TopKBundles(pulsar);
         NamespaceBundleStats stats1 = new NamespaceBundleStats();
         stats1.msgRateIn = 500;
-        bundleStats.put("bundle-1", stats1);
+        bundleStats.put(bundle1, stats1);
 
         NamespaceBundleStats stats2 = new NamespaceBundleStats();
         stats2.msgRateIn = 10000;
-        bundleStats.put("bundle-2", stats2);
+        bundleStats.put(bundle2, stats2);
 
         NamespaceBundleStats stats3 = new NamespaceBundleStats();
         stats3.msgRateIn = 100000;
-        bundleStats.put("bundle-3", stats3);
+        bundleStats.put(bundle3, stats3);
 
         NamespaceBundleStats stats4 = new NamespaceBundleStats();
         stats4.msgRateIn = 0;
-        bundleStats.put("bundle-4", stats4);
+        bundleStats.put(bundle4, stats4);
 
         topKBundles.update(bundleStats, 3);
         var top0 = topKBundles.getLoadData().getTopBundlesLoadData().get(0);
         var top1 = topKBundles.getLoadData().getTopBundlesLoadData().get(1);
         var top2 = topKBundles.getLoadData().getTopBundlesLoadData().get(2);
 
-        assertEquals(top0.bundleName(), "bundle-3");
-        assertEquals(top1.bundleName(), "bundle-2");
-        assertEquals(top2.bundleName(), "bundle-1");
+        assertEquals(top0.bundleName(), bundle3);
+        assertEquals(top1.bundleName(), bundle2);
+        assertEquals(top2.bundleName(), bundle1);
     }
 
     @Test
     public void testSystemNamespace() {
         Map bundleStats = new HashMap<>();
-        var topKBundles = new TopKBundles();
+        var topKBundles = new TopKBundles(pulsar);
         NamespaceBundleStats stats1 = new NamespaceBundleStats();
         stats1.msgRateIn = 500;
-        bundleStats.put("pulsar/system/bundle-1", stats1);
+        bundleStats.put("pulsar/system/0x00000000_0x0FFFFFFF", stats1);
 
         NamespaceBundleStats stats2 = new NamespaceBundleStats();
         stats2.msgRateIn = 10000;
-        bundleStats.put("pulsar/system/bundle-2", stats2);
+        bundleStats.put(bundle1, stats2);
 
         topKBundles.update(bundleStats, 2);
-        assertTrue(topKBundles.getLoadData().getTopBundlesLoadData().isEmpty());
+
+        assertEquals(topKBundles.getLoadData().getTopBundlesLoadData().size(), 1);
+        var top0 = topKBundles.getLoadData().getTopBundlesLoadData().get(0);
+        assertEquals(top0.bundleName(), bundle1);
+    }
+
+
+    private  void setAntiAffinityGroup() throws MetadataStoreException {
+        LocalPolicies localPolicies = new LocalPolicies(null, null, "namespaceAntiAffinityGroup");
+        NamespaceName namespace = NamespaceName.get(LoadManagerShared.getNamespaceNameFromBundleName(bundle2));
+        doReturn(Optional.of(localPolicies)).when(localPoliciesResources).getLocalPolicies(eq(namespace));
+    }
+
+    private void setIsolationPolicy() throws MetadataStoreException {
+        Map parameters = new HashMap<>();
+        parameters.put("min_limit", "3");
+        parameters.put("usage_threshold", "90");
+        var policyData = Map.of("policy", (NamespaceIsolationDataImpl)
+                NamespaceIsolationData.builder()
+                        .namespaces(Collections.singletonList("my-tenant/my-namespace1.*"))
+                        .primary(Collections.singletonList("prod1-broker[1-3].messaging.use.example.com"))
+                        .secondary(Collections.singletonList("prod1-broker.*.use.example.com"))
+                        .autoFailoverPolicy(AutoFailoverPolicyData.builder()
+                                .policyType(AutoFailoverPolicyType.min_available)
+                                .parameters(parameters)
+                                .build()
+                        ).build());
+
+        NamespaceIsolationPolicies policies = new NamespaceIsolationPolicies(policyData);
+        doReturn(Optional.of(policies)).when(isolationPolicyResources).getIsolationDataPolicies(any());
+    }
+
+    @Test
+    public void testIsolationPolicy() throws MetadataStoreException {
+
+        setIsolationPolicy();
+
+        Map bundleStats = new HashMap<>();
+        var topKBundles = new TopKBundles(pulsar);
+        NamespaceBundleStats stats1 = new NamespaceBundleStats();
+        stats1.msgRateIn = 500;
+        bundleStats.put(bundle1, stats1);
+
+        NamespaceBundleStats stats2 = new NamespaceBundleStats();
+        stats2.msgRateIn = 10000;
+        bundleStats.put(bundle2, stats2);
+
+        topKBundles.update(bundleStats, 2);
+        assertEquals(topKBundles.getLoadData().getTopBundlesLoadData().size(), 1);
+        var top0 = topKBundles.getLoadData().getTopBundlesLoadData().get(0);
+        assertEquals(top0.bundleName(), bundle2);
+    }
+
+
+    @Test
+    public void testAntiAffinityGroupPolicy() throws MetadataStoreException {
+
+        setAntiAffinityGroup();
+
+        Map bundleStats = new HashMap<>();
+        var topKBundles = new TopKBundles(pulsar);
+        NamespaceBundleStats stats1 = new NamespaceBundleStats();
+        stats1.msgRateIn = 500;
+        bundleStats.put(bundle1, stats1);
+
+        NamespaceBundleStats stats2 = new NamespaceBundleStats();
+        stats2.msgRateIn = 10000;
+        bundleStats.put(bundle2, stats2);
+
+        topKBundles.update(bundleStats, 2);
+        assertEquals(topKBundles.getLoadData().getTopBundlesLoadData().size(), 1);
+        var top0 = topKBundles.getLoadData().getTopBundlesLoadData().get(0);
+        assertEquals(top0.bundleName(), bundle1);
+
+    }
+
+    @Test
+    public void testLoadBalancerSheddingBundlesWithPoliciesEnabledConfig() throws MetadataStoreException {
+
+        setIsolationPolicy();
+        setAntiAffinityGroup();
+
+        configuration.setLoadBalancerSheddingBundlesWithPoliciesEnabled(true);
+
+        Map bundleStats = new HashMap<>();
+        var topKBundles = new TopKBundles(pulsar);
+        NamespaceBundleStats stats1 = new NamespaceBundleStats();
+        stats1.msgRateIn = 500;
+        bundleStats.put(bundle1, stats1);
+
+        NamespaceBundleStats stats2 = new NamespaceBundleStats();
+        stats2.msgRateIn = 10000;
+        bundleStats.put(bundle2, stats2);
+
+        topKBundles.update(bundleStats, 2);
+
+        assertEquals(topKBundles.getLoadData().getTopBundlesLoadData().size(), 2);
+        var top0 = topKBundles.getLoadData().getTopBundlesLoadData().get(0);
+        var top1 = topKBundles.getLoadData().getTopBundlesLoadData().get(1);
+
+        assertEquals(top0.bundleName(), bundle2);
+        assertEquals(top1.bundleName(), bundle1);
+
+        configuration.setLoadBalancerSheddingBundlesWithPoliciesEnabled(false);
+
+        topKBundles.update(bundleStats, 2);
+
+        assertEquals(topKBundles.getLoadData().getTopBundlesLoadData().size(), 0);
     }
 
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java
index ce2d3d8c3ea93..b5c415c405fbd 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java
@@ -27,6 +27,7 @@
 import static org.testng.Assert.assertNull;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
 import org.apache.commons.lang.reflect.FieldUtils;
 import org.apache.pulsar.broker.PulsarService;
@@ -34,8 +35,12 @@
 import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData;
 import org.apache.pulsar.broker.loadbalance.extensions.models.TopKBundles;
 import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore;
+import org.apache.pulsar.broker.resources.LocalPoliciesResources;
+import org.apache.pulsar.broker.resources.NamespaceResources;
+import org.apache.pulsar.broker.resources.PulsarResources;
 import org.apache.pulsar.broker.service.BrokerService;
 import org.apache.pulsar.broker.service.PulsarStats;
+import org.apache.pulsar.metadata.api.MetadataStoreException;
 import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
@@ -48,26 +53,44 @@ public class TopBundleLoadDataReporterTest {
     PulsarStats pulsarStats;
     Map bundleStats;
     ServiceConfiguration config;
+    private NamespaceResources.IsolationPolicyResources isolationPolicyResources;
+    private PulsarResources pulsarResources;
+    private LocalPoliciesResources localPoliciesResources;
+    String bundle1 = "my-tenant/my-namespace1/0x00000000_0x0FFFFFFF";
+    String bundle2 = "my-tenant/my-namespace2/0x00000000_0x0FFFFFFF";
 
     @BeforeMethod
-    void setup() {
+    void setup() throws MetadataStoreException {
         config = new ServiceConfiguration();
         pulsar = mock(PulsarService.class);
         store = mock(LoadDataStore.class);
         brokerService = mock(BrokerService.class);
         pulsarStats = mock(PulsarStats.class);
+        pulsarResources = mock(PulsarResources.class);
+        isolationPolicyResources = mock(NamespaceResources.IsolationPolicyResources.class);
+        var namespaceResources = mock(NamespaceResources.class);
+        localPoliciesResources = mock(LocalPoliciesResources.class);
+
         doReturn(brokerService).when(pulsar).getBrokerService();
         doReturn(config).when(pulsar).getConfiguration();
         doReturn(pulsarStats).when(brokerService).getPulsarStats();
         doReturn(CompletableFuture.completedFuture(null)).when(store).pushAsync(any(), any());
 
+        doReturn(pulsarResources).when(pulsar).getPulsarResources();
+        doReturn(namespaceResources).when(pulsarResources).getNamespaceResources();
+        doReturn(isolationPolicyResources).when(namespaceResources).getIsolationPolicies();
+        doReturn(Optional.empty()).when(isolationPolicyResources).getIsolationDataPolicies(any());
+
+        doReturn(localPoliciesResources).when(pulsarResources).getLocalPolicies();
+        doReturn(Optional.empty()).when(localPoliciesResources).getLocalPolicies(any());
+
         bundleStats = new HashMap<>();
         NamespaceBundleStats stats1 = new NamespaceBundleStats();
         stats1.msgRateIn = 500;
-        bundleStats.put("bundle-1", stats1);
+        bundleStats.put(bundle1, stats1);
         NamespaceBundleStats stats2 = new NamespaceBundleStats();
         stats2.msgRateIn = 10000;
-        bundleStats.put("bundle-2", stats2);
+        bundleStats.put(bundle2, stats2);
         doReturn(bundleStats).when(brokerService).getBundleStats();
     }
 
@@ -81,25 +104,25 @@ public void testGenerateLoadData() throws IllegalAccessException {
         doReturn(1l).when(pulsarStats).getUpdatedAt();
         config.setLoadBalancerBundleLoadReportPercentage(100);
         var target = new TopBundleLoadDataReporter(pulsar, "", store);
-        var expected = new TopKBundles();
+        var expected = new TopKBundles(pulsar);
         expected.update(bundleStats, 2);
         assertEquals(target.generateLoadData(), expected.getLoadData());
 
         config.setLoadBalancerBundleLoadReportPercentage(50);
         FieldUtils.writeDeclaredField(target, "lastBundleStatsUpdatedAt", 0l, true);
-        expected = new TopKBundles();
+        expected = new TopKBundles(pulsar);
         expected.update(bundleStats, 1);
         assertEquals(target.generateLoadData(), expected.getLoadData());
 
         config.setLoadBalancerBundleLoadReportPercentage(1);
         FieldUtils.writeDeclaredField(target, "lastBundleStatsUpdatedAt", 0l, true);
-        expected = new TopKBundles();
+        expected = new TopKBundles(pulsar);
         expected.update(bundleStats, 1);
         assertEquals(target.generateLoadData(), expected.getLoadData());
 
         doReturn(new HashMap()).when(brokerService).getBundleStats();
         FieldUtils.writeDeclaredField(target, "lastBundleStatsUpdatedAt", 0l, true);
-        expected = new TopKBundles();
+        expected = new TopKBundles(pulsar);
         assertEquals(target.generateLoadData(), expected.getLoadData());
     }
 
@@ -116,7 +139,7 @@ public void testReportForce()  {
     public void testReport(){
         var target = new TopBundleLoadDataReporter(pulsar, "broker-1", store);
         doReturn(1l).when(pulsarStats).getUpdatedAt();
-        var expected = new TopKBundles();
+        var expected = new TopKBundles(pulsar);
         expected.update(bundleStats, 1);
         target.reportAsync(false);
         verify(store, times(1)).pushAsync("broker-1", expected.getLoadData());
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
index a89f33bb98737..5ca9345d6def5 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
@@ -97,8 +97,8 @@ public class TransferShedderTest {
     double setupLoadStd = 0.3982762860126121;
 
     PulsarService pulsar;
-    AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper;
     ServiceConfiguration conf;
+    AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper;
     LocalPoliciesResources localPoliciesResources;
     String bundleD1 = "my-tenant/my-namespaceD/0x00000000_0x0FFFFFFF";
     String bundleD2 = "my-tenant/my-namespaceD/0x0FFFFFFF_0xFFFFFFFF";
@@ -109,8 +109,7 @@ public class TransferShedderTest {
     public void init() throws MetadataStoreException {
         pulsar = mock(PulsarService.class);
         conf = new ServiceConfiguration();
-        doReturn(conf).when(pulsar).getConfiguration();
-
+        conf.setLoadBalancerSheddingBundlesWithPoliciesEnabled(true);
         var pulsarResources = mock(PulsarResources.class);
         var namespaceResources = mock(NamespaceResources.class);
         var isolationPolicyResources = mock(NamespaceResources.IsolationPolicyResources.class);
@@ -118,6 +117,7 @@ public void init() throws MetadataStoreException {
         var namespaceService = mock(NamespaceService.class);
         localPoliciesResources = mock(LocalPoliciesResources.class);
         antiAffinityGroupPolicyHelper = mock(AntiAffinityGroupPolicyHelper.class);
+        doReturn(conf).when(pulsar).getConfiguration();
         doReturn(namespaceService).when(pulsar).getNamespaceService();
         doReturn(pulsarResources).when(pulsar).getPulsarResources();
         doReturn(localPoliciesResources).when(pulsarResources).getLocalPolicies();
@@ -138,9 +138,9 @@ public void init() throws MetadataStoreException {
         }).when(factory).getBundle(anyString(), anyString());
         doReturn(true).when(antiAffinityGroupPolicyHelper).canUnload(any(), any(), any(), any());
     }
+
     public LoadManagerContext setupContext(){
         var ctx = getContext();
-        ctx.brokerConfiguration().setLoadBalancerDebugModeEnabled(true);
 
         var brokerLoadDataStore = ctx.brokerLoadDataStore();
         brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx,  2));
@@ -160,7 +160,6 @@ public LoadManagerContext setupContext(){
 
     public LoadManagerContext setupContext(int clusterSize) {
         var ctx = getContext();
-        ctx.brokerConfiguration().setLoadBalancerDebugModeEnabled(true);
 
         var brokerLoadDataStore = ctx.brokerLoadDataStore();
         var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
@@ -170,7 +169,7 @@ public LoadManagerContext setupContext(int clusterSize) {
             int brokerLoad = rand.nextInt(100);
             brokerLoadDataStore.pushAsync("broker" + i, getCpuLoad(ctx,  brokerLoad));
             int bundleLoad = rand.nextInt(brokerLoad + 1);
-            topBundlesLoadDataStore.pushAsync("broker" + i, getTopBundlesLoad("bundle" + i,
+            topBundlesLoadDataStore.pushAsync("broker" + i, getTopBundlesLoad("my-tenant/my-namespace" + i,
                     bundleLoad, brokerLoad - bundleLoad));
         }
         return ctx;
@@ -199,7 +198,7 @@ public TopBundlesLoadData getTopBundlesLoad(String bundlePrefix, int load1, int
         namespaceBundleStats1.msgThroughputOut = load1;
         var namespaceBundleStats2 = new NamespaceBundleStats();
         namespaceBundleStats2.msgThroughputOut = load2;
-        var topKBundles = new TopKBundles();
+        var topKBundles = new TopKBundles(pulsar);
         topKBundles.update(Map.of(bundlePrefix + "/0x00000000_0x0FFFFFFF", namespaceBundleStats1,
                 bundlePrefix + "/0x0FFFFFFF_0xFFFFFFFF", namespaceBundleStats2), 2);
         return topKBundles.getLoadData();
@@ -208,7 +207,7 @@ public TopBundlesLoadData getTopBundlesLoad(String bundlePrefix, int load1, int
     public TopBundlesLoadData getTopBundlesLoad(String bundlePrefix, int load1) {
         var namespaceBundleStats1 = new NamespaceBundleStats();
         namespaceBundleStats1.msgThroughputOut = load1;
-        var topKBundles = new TopKBundles();
+        var topKBundles = new TopKBundles(pulsar);
         topKBundles.update(Map.of(bundlePrefix + "/0x00000000_0x0FFFFFFF", namespaceBundleStats1), 2);
         return topKBundles.getLoadData();
     }
@@ -220,7 +219,7 @@ public TopBundlesLoadData getTopBundlesLoadWithOutSuffix(String namespace,
         namespaceBundleStats1.msgThroughputOut = load1 * 1e6;
         var namespaceBundleStats2 = new NamespaceBundleStats();
         namespaceBundleStats2.msgThroughputOut = load2 * 1e6;
-        var topKBundles = new TopKBundles();
+        var topKBundles = new TopKBundles(pulsar);
         topKBundles.update(Map.of(namespace + "/0x00000000_0x7FFFFFF", namespaceBundleStats1,
                 namespace + "/0x7FFFFFF_0xFFFFFFF", namespaceBundleStats2), 2);
         return topKBundles.getLoadData();
@@ -229,6 +228,8 @@ public TopBundlesLoadData getTopBundlesLoadWithOutSuffix(String namespace,
     public LoadManagerContext getContext(){
         var ctx = mock(LoadManagerContext.class);
         var conf = new ServiceConfiguration();
+        conf.setLoadBalancerDebugModeEnabled(true);
+        conf.setLoadBalancerSheddingBundlesWithPoliciesEnabled(false);
         var brokerLoadDataStore = new LoadDataStore() {
             Map map = new HashMap<>();
             @Override
@@ -461,8 +462,6 @@ public void testGetAvailableBrokersFailed() {
         BrokerRegistry registry = ctx.brokerRegistry();
         doReturn(FutureUtil.failedFuture(new TimeoutException())).when(registry).getAvailableBrokerLookupDataAsync();
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
-
-        assertTrue(res.isEmpty());
         assertEquals(counter.getBreakdownCounters().get(Skip).get(Unknown).get(), 1);
         assertEquals(counter.getLoadAvg(), setupLoadAvg);
         assertEquals(counter.getLoadStd(), setupLoadStd);
@@ -501,6 +500,25 @@ public void testBundlesWithIsolationPolicies() throws IllegalAccessException {
         assertEquals(res, expected);
         assertEquals(counter.getLoadAvg(), setupLoadAvg);
         assertEquals(counter.getLoadStd(), setupLoadStd);
+
+
+        // test setLoadBalancerSheddingBundlesWithPoliciesEnabled=false;
+        doReturn(true).when(allocationPoliciesSpy).areIsolationPoliciesPresent(any());
+        ctx.brokerConfiguration().setLoadBalancerTransferEnabled(true);
+        ctx.brokerConfiguration().setLoadBalancerSheddingBundlesWithPoliciesEnabled(false);
+        res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
+        assertTrue(res.isEmpty());
+        assertEquals(counter.getBreakdownCounters().get(Skip).get(NoBundles).get(), 1);
+        assertEquals(counter.getLoadAvg(), setupLoadAvg);
+        assertEquals(counter.getLoadStd(), setupLoadStd);
+
+        // Test unload a has isolation policies broker.
+        ctx.brokerConfiguration().setLoadBalancerTransferEnabled(false);
+        res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
+        assertTrue(res.isEmpty());
+        assertEquals(counter.getBreakdownCounters().get(Skip).get(NoBundles).get(), 2);
+        assertEquals(counter.getLoadAvg(), setupLoadAvg);
+        assertEquals(counter.getLoadStd(), setupLoadStd);
     }
 
     public BrokerLookupData getLookupData() {
@@ -602,7 +620,7 @@ public void testBundlesWithAntiAffinityGroup() throws IllegalAccessException, Me
         var expected2 = new HashSet<>();
         expected2.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")),
                 Success, Overloaded));
-        assertEquals(res, expected2);
+        assertEquals(res2, expected2);
         assertEquals(counter.getLoadAvg(), setupLoadAvg);
         assertEquals(counter.getLoadStd(), setupLoadStd);
     }

From f294be37ce12e311a6bd87493cf0cba11fc6c2f8 Mon Sep 17 00:00:00 2001
From: fengyubiao 
Date: Wed, 22 Mar 2023 15:30:00 +0800
Subject: [PATCH 139/174] [fix] [ml] make the result of delete cursor is
 success if cursor is deleted (#19825)

When deleting the zk node of the cursor, if the exception `MetadataStoreException.NotFoundException`  occurs, the deletion is considered successful.
---
 .../mledger/impl/MetaStoreImpl.java           | 16 ++++++++----
 .../mledger/impl/ManagedLedgerTest.java       | 26 +++++++++++++++++++
 2 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java
index bcb73553324dd..3bdaac1e8d80e 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java
@@ -47,6 +47,7 @@
 import org.apache.pulsar.common.allocator.PulsarByteBufAllocator;
 import org.apache.pulsar.common.compression.CompressionCodec;
 import org.apache.pulsar.common.compression.CompressionCodecProvider;
+import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.metadata.api.MetadataStore;
 import org.apache.pulsar.metadata.api.MetadataStoreException;
 import org.apache.pulsar.metadata.api.Notification;
@@ -292,7 +293,7 @@ public void asyncUpdateCursorInfo(String ledgerName, String cursorName, ManagedC
     @Override
     public void asyncRemoveCursor(String ledgerName, String cursorName, MetaStoreCallback callback) {
         String path = PREFIX + ledgerName + "/" + cursorName;
-        log.info("[{}] Remove consumer={}", ledgerName, cursorName);
+        log.info("[{}] Remove cursor={}", ledgerName, cursorName);
 
         store.delete(path, Optional.empty())
                 .thenAcceptAsync(v -> {
@@ -301,11 +302,16 @@ public void asyncRemoveCursor(String ledgerName, String cursorName, MetaStoreCal
                     }
                     callback.operationComplete(null, null);
                 }, executor.chooseThread(ledgerName))
-                .exceptionally(ex -> {
-                    executor.executeOrdered(ledgerName, SafeRunnable.safeRun(() -> callback
-                            .operationFailed(getException(ex))));
+                .exceptionallyAsync(ex -> {
+                    Throwable actEx = FutureUtil.unwrapCompletionException(ex);
+                    if (actEx instanceof MetadataStoreException.NotFoundException){
+                        log.info("[{}] [{}] cursor delete done because it did not exist.", ledgerName, cursorName);
+                        callback.operationComplete(null, null);
+                        return null;
+                    }
+                    SafeRunnable.safeRun(() -> callback.operationFailed(getException(ex)));
                     return null;
-                });
+                }, executor.chooseThread(ledgerName));
     }
 
     @Override
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java
index a4d8b75d00c96..dd30cde72e769 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java
@@ -128,6 +128,7 @@
 import org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition;
 import org.apache.pulsar.common.policies.data.EnsemblePlacementPolicyConfig;
 import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl;
+import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.metadata.api.MetadataStoreException;
 import org.apache.pulsar.metadata.api.Stat;
 import org.apache.pulsar.metadata.api.extended.SessionEvent;
@@ -3969,4 +3970,29 @@ public void testGetEstimatedBacklogSize() throws Exception {
         Assert.assertEquals(ledger.getEstimatedBacklogSize(((PositionImpl) positions.get(9)).getNext()), 0);
         ledger.close();
     }
+
+    @Test
+    public void testDeleteCursorTwice() throws Exception {
+        ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open("ml");
+        String cursorName = "cursor_1";
+        ml.openCursor(cursorName);
+        syncRemoveCursor(ml, cursorName);
+        syncRemoveCursor(ml, cursorName);
+    }
+
+    private void syncRemoveCursor(ManagedLedgerImpl ml, String cursorName){
+        CompletableFuture future = new CompletableFuture<>();
+        ml.getStore().asyncRemoveCursor(ml.name, cursorName, new MetaStoreCallback() {
+            @Override
+            public void operationComplete(Void result, Stat stat) {
+                future.complete(null);
+            }
+
+            @Override
+            public void operationFailed(MetaStoreException e) {
+                future.completeExceptionally(FutureUtil.unwrapCompletionException(e));
+            }
+        });
+        future.join();
+    }
 }

From a9037334a399af905fae94d2aefa5db339cbd5b1 Mon Sep 17 00:00:00 2001
From: fengyubiao 
Date: Wed, 22 Mar 2023 16:49:13 +0800
Subject: [PATCH 140/174] [fix] [admin] Make response code to 400 instead of
 500 when delete topic fails due to enabled geo-replication (#19879)

Motivation: As expected, If geo-replication is enabled, a topic cannot be deleted. However deleting that topic returns a 500, and no further info.
Modifications: Make response code to 400 instead of 500 when delete topic fails due to enabled geo-replication
---
 .../broker/admin/v2/PersistentTopics.java     |  4 ++-
 .../pulsar/broker/service/ReplicatorTest.java | 28 +++++++++++++++++++
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java
index 7eeaec403d842..477cc8b57129d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java
@@ -1106,7 +1106,9 @@ public void deleteTopic(
                         ex = new RestException(Response.Status.PRECONDITION_FAILED,
                                 t.getMessage());
                     }
-                    if (isManagedLedgerNotFoundException(t)) {
+                    if (t instanceof IllegalStateException){
+                        ex = new RestException(422/* Unprocessable entity*/, t.getMessage());
+                    } else if (isManagedLedgerNotFoundException(t)) {
                         ex = new RestException(Response.Status.NOT_FOUND,
                                 getTopicNotFoundErrorMessage(topicName.toString()));
                     } else if (!isRedirectException(ex)) {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java
index ab4f6a5c7f8ba..4086c54a0ba4b 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java
@@ -70,6 +70,7 @@
 import org.apache.pulsar.broker.service.persistent.PersistentReplicator;
 import org.apache.pulsar.broker.service.persistent.PersistentTopic;
 import org.apache.pulsar.client.admin.PulsarAdmin;
+import org.apache.pulsar.client.admin.PulsarAdminException;
 import org.apache.pulsar.client.api.Consumer;
 import org.apache.pulsar.client.api.Message;
 import org.apache.pulsar.client.api.MessageId;
@@ -869,6 +870,33 @@ public void testReplicatorProducerClosing() throws Exception {
         assertNull(producer);
     }
 
+    @Test
+    public void testDeleteTopicFailure() throws Exception {
+        final String topicName = BrokerTestUtil.newUniqueName("persistent://pulsar/ns/tp_" + UUID.randomUUID());
+        admin1.topics().createNonPartitionedTopic(topicName);
+        try {
+            admin1.topics().delete(topicName);
+            fail("Delete topic should fail if enabled replicator");
+        } catch (Exception ex) {
+            assertTrue(ex instanceof PulsarAdminException);
+            assertEquals(((PulsarAdminException) ex).getStatusCode(), 422/* Unprocessable entity*/);
+        }
+    }
+
+    @Test
+    public void testDeletePartitionedTopicFailure() throws Exception {
+        final String topicName = BrokerTestUtil.newUniqueName("persistent://pulsar/ns/tp_" + UUID.randomUUID());
+        admin1.topics().createPartitionedTopic(topicName, 2);
+        admin1.topics().createSubscription(topicName, "sub1", MessageId.earliest);
+        try {
+            admin1.topics().deletePartitionedTopic(topicName);
+            fail("Delete topic should fail if enabled replicator");
+        } catch (Exception ex) {
+            assertTrue(ex instanceof PulsarAdminException);
+            assertEquals(((PulsarAdminException) ex).getStatusCode(), 422/* Unprocessable entity*/);
+        }
+    }
+
     @Test(priority = 4, timeOut = 30000)
     public void testReplicatorProducerName() throws Exception {
         log.info("--- Starting ReplicatorTest::testReplicatorProducerName ---");

From a1fdbd294cab5c48dd2b092085465eb4075cc6a4 Mon Sep 17 00:00:00 2001
From: Jiwei Guo 
Date: Thu, 23 Mar 2023 20:09:57 +0800
Subject: [PATCH 141/174] [fix][broker] Fix NPE when update topic policy.
 (#19875)

---
 .../apache/pulsar/common/policies/data/TopicPolicies.java    | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicPolicies.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicPolicies.java
index caa9cd3fd1daa..4a76170d116a3 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicPolicies.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicPolicies.java
@@ -20,6 +20,7 @@
 
 import com.google.common.collect.Sets;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -188,4 +189,8 @@ public boolean isSchemaValidationEnforced() {
     public Set getReplicationClustersSet() {
         return replicationClusters != null ? Sets.newTreeSet(this.replicationClusters) : null;
     }
+
+    public Map getSubscriptionPolicies() {
+        return subscriptionPolicies == null ? Collections.emptyMap() : subscriptionPolicies;
+    }
 }

From c172a775fa0fc5762a9703669ed8ebcd2efbe042 Mon Sep 17 00:00:00 2001
From: Cong Zhao 
Date: Fri, 24 Mar 2023 15:05:02 +0800
Subject: [PATCH 142/174] [fix][broker] Fix can't send ErrorCommand when
 message is null value (#19899)

---
 .../pulsar/client/api/ClientErrorsTest.java   | 29 +++++++++++++++++--
 .../pulsar/common/protocol/Commands.java      |  4 +--
 2 files changed, 28 insertions(+), 5 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientErrorsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientErrorsTest.java
index e8b9baa992c46..61c7a98602b69 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientErrorsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ClientErrorsTest.java
@@ -22,16 +22,15 @@
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertTrue;
 import static org.testng.Assert.fail;
-
 import io.netty.channel.ChannelHandlerContext;
-
+import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
-
 import lombok.Cleanup;
+import org.apache.bookkeeper.common.util.JsonUtil;
 import org.apache.pulsar.client.impl.ConsumerBase;
 import org.apache.pulsar.client.impl.PartitionedProducerImpl;
 import org.apache.pulsar.client.impl.ProducerBase;
@@ -831,4 +830,28 @@ public void testConsumerReconnect() throws Exception {
         mockBrokerService.resetHandleConnect();
         mockBrokerService.resetHandleSubscribe();
     }
+
+    @Test
+    public void testCommandErrorMessageIsNull() throws Exception {
+        @Cleanup
+        PulsarClient client = PulsarClient.builder().serviceUrl(mockBrokerService.getBrokerAddress()).build();
+
+        mockBrokerService.setHandleProducer((ctx, producer) -> {
+            try {
+                ctx.writeAndFlush(Commands.newError(producer.getRequestId(), ServerError.AuthorizationError, null));
+            } catch (Exception e) {
+                fail("Send error command failed", e);
+            }
+        });
+
+        try {
+            client.newProducer().topic("persistent://prop/use/ns/t1").create();
+            fail();
+        } catch (Exception e) {
+            assertTrue(e instanceof PulsarClientException.AuthorizationException);
+            Map map = JsonUtil.fromJson(e.getMessage(), Map.class);
+            assertEquals(map.get("errorMsg"), "");
+        }
+        mockBrokerService.resetHandleProducer();
+    }
 }
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java
index 8a5684cf676b0..85c4d021fdf22 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java
@@ -368,7 +368,7 @@ public static BaseCommand newErrorCommand(long requestId, ServerError serverErro
         cmd.setError()
                 .setRequestId(requestId)
                 .setError(serverError)
-                .setMessage(message);
+                .setMessage(message != null ? message : "");
         return cmd;
     }
 
@@ -401,7 +401,7 @@ public static BaseCommand newSendErrorCommand(long producerId, long sequenceId,
                 .setProducerId(producerId)
                 .setSequenceId(sequenceId)
                 .setError(error)
-                .setMessage(errorMsg);
+                .setMessage(errorMsg != null ? errorMsg : "");
         return cmd;
     }
 

From f1f8dab972b098be69ad35ab3d307f19284c4e48 Mon Sep 17 00:00:00 2001
From: Abhilash Mandaliya 
Date: Fri, 24 Mar 2023 16:51:22 +0530
Subject: [PATCH 143/174] [improve][io][broker] Updated
 org.reflections-reflections library version (#19898)

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 257018b0b0598..3f33069b8c252 100644
--- a/pom.xml
+++ b/pom.xml
@@ -149,7 +149,7 @@ flexible messaging model and an intuitive client API.
     1.0.6
     1.0.2.3
     2.13.4.20221013
-    0.9.11
+    0.10.2
     1.6.2
     8.37
     0.40.2

From 1af0ff35109af066662324515e59d2b8bff8c899 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Fri, 24 Mar 2023 15:03:47 +0000
Subject: [PATCH 144/174] [improve][ci] Handle retry of maven downloads for
 more failure cases (#19903)

---
 .github/workflows/ci-go-functions.yaml           | 2 +-
 .github/workflows/ci-maven-cache-update.yaml     | 2 +-
 .github/workflows/ci-owasp-dependency-check.yaml | 2 +-
 .github/workflows/pulsar-ci-flaky.yaml           | 2 +-
 .github/workflows/pulsar-ci.yaml                 | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/ci-go-functions.yaml b/.github/workflows/ci-go-functions.yaml
index f4d1ae0b99887..a34cb1a90908e 100644
--- a/.github/workflows/ci-go-functions.yaml
+++ b/.github/workflows/ci-go-functions.yaml
@@ -32,7 +32,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  MAVEN_OPTS: -Xss1500k -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
+  MAVEN_OPTS: -Xss1500k -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
 
 jobs:
   preconditions:
diff --git a/.github/workflows/ci-maven-cache-update.yaml b/.github/workflows/ci-maven-cache-update.yaml
index 87570586fde6e..a5a88e2e0b8a6 100644
--- a/.github/workflows/ci-maven-cache-update.yaml
+++ b/.github/workflows/ci-maven-cache-update.yaml
@@ -42,7 +42,7 @@ on:
     - cron: '30 */12 * * *'
 
 env:
-  MAVEN_OPTS: -Xss1500k -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
+  MAVEN_OPTS: -Xss1500k -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
 
 jobs:
   update-maven-dependencies-cache:
diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml
index 194d88c582d42..dba1c92ca28dd 100644
--- a/.github/workflows/ci-owasp-dependency-check.yaml
+++ b/.github/workflows/ci-owasp-dependency-check.yaml
@@ -24,7 +24,7 @@ on:
   workflow_dispatch:
 
 env:
-  MAVEN_OPTS: -Xss1500k -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
+  MAVEN_OPTS: -Xss1500k -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
 
 jobs:
   run-owasp-dependency-check:
diff --git a/.github/workflows/pulsar-ci-flaky.yaml b/.github/workflows/pulsar-ci-flaky.yaml
index acfa66ff43c74..ea19bc3307819 100644
--- a/.github/workflows/pulsar-ci-flaky.yaml
+++ b/.github/workflows/pulsar-ci-flaky.yaml
@@ -36,7 +36,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  MAVEN_OPTS: -Xss1500k -Xmx1024m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
+  MAVEN_OPTS: -Xss1500k -Xmx1024m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
   # defines the retention period for the intermediate build artifacts needed for rerunning a failed build job
   # it's possible to rerun individual failed jobs when the build artifacts are available
   # if the artifacts have already been expired, the complete workflow can be rerun by closing and reopening the PR or by rebasing the PR
diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml
index 721a1d2eafc72..f7dbc755264d6 100644
--- a/.github/workflows/pulsar-ci.yaml
+++ b/.github/workflows/pulsar-ci.yaml
@@ -36,7 +36,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  MAVEN_OPTS: -Xss1500k -Xmx1024m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
+  MAVEN_OPTS: -Xss1500k -Xmx1024m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
   # defines the retention period for the intermediate build artifacts needed for rerunning a failed build job
   # it's possible to rerun individual failed jobs when the build artifacts are available
   # if the artifacts have already been expired, the complete workflow can be rerun by closing and reopening the PR or by rebasing the PR

From 8081ee26d8d3727c720800a3453a798893763fee Mon Sep 17 00:00:00 2001
From: Cong Zhao 
Date: Sat, 25 Mar 2023 15:39:37 +0800
Subject: [PATCH 145/174] [improve][broker][PIP-195] Make bucket merge
 operation asynchronous (#19873)

---
 .../bucket/BucketDelayedDeliveryTracker.java  | 139 +++++++++++-------
 .../delayed/bucket/ImmutableBucket.java       |  17 ++-
 .../broker/delayed/bucket/MutableBucket.java  |   6 +
 ...ayedMessageIndexBucketSnapshotFormat.proto |   1 +
 .../BookkeeperBucketSnapshotStorageTest.java  |   3 +
 .../BucketDelayedDeliveryTrackerTest.java     |  57 +++++--
 6 files changed, 154 insertions(+), 69 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
index 31fdaa6fb7667..1cf8bd2b20d17 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
@@ -90,6 +90,8 @@ public class BucketDelayedDeliveryTracker extends AbstractDelayedDeliveryTracker
 
     private final Table snapshotSegmentLastIndexTable;
 
+    private static final Long INVALID_BUCKET_ID = -1L;
+
     public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher,
                                  Timer timer, long tickTimeMillis,
                                  boolean isDelayedDeliveryDeliverAtTimeStrict,
@@ -246,12 +248,12 @@ private void afterCreateImmutableBucket(Pair immu
                     immutableBucket);
 
             immutableBucket.getSnapshotCreateFuture().ifPresent(createFuture -> {
-                CompletableFuture future = createFuture.whenComplete((__, ex) -> {
+                CompletableFuture future = createFuture.handle((bucketId, ex) -> {
                     if (ex == null) {
                         immutableBucket.setSnapshotSegments(null);
                         log.info("[{}] Creat bucket snapshot finish, bucketKey: {}", dispatcher.getName(),
                                 immutableBucket.bucketKey());
-                        return;
+                        return bucketId;
                     }
 
                     //TODO Record create snapshot failed
@@ -277,6 +279,7 @@ private void afterCreateImmutableBucket(Pair immu
                         snapshotSegmentLastIndexTable.remove(lastDelayedIndex.getLedgerId(),
                                 lastDelayedIndex.getTimestamp());
                     }
+                    return INVALID_BUCKET_ID;
                 });
                 immutableBucket.setSnapshotCreateFuture(future);
             });
@@ -308,12 +311,7 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver
             lastMutableBucket.resetLastMutableBucketRange();
 
             if (immutableBuckets.asMapOfRanges().size() > maxNumBuckets) {
-                try {
-                    asyncMergeBucketSnapshot().get(2 * AsyncOperationTimeoutSeconds * MaxRetryTimes, TimeUnit.SECONDS);
-                } catch (Exception e) {
-                    // Ignore exception to merge bucket on the next schedule.
-                    log.error("[{}] An exception occurs when merge bucket snapshot.", dispatcher.getName(), e);
-                }
+                asyncMergeBucketSnapshot();
             }
         }
 
@@ -341,18 +339,26 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver
     private synchronized CompletableFuture asyncMergeBucketSnapshot() {
         List values = immutableBuckets.asMapOfRanges().values().stream().toList();
         long minNumberMessages = Long.MAX_VALUE;
+        long minScheduleTimestamp = Long.MAX_VALUE;
         int minIndex = -1;
         for (int i = 0; i + 1 < values.size(); i++) {
             ImmutableBucket bucketL = values.get(i);
             ImmutableBucket bucketR = values.get(i + 1);
-            long numberMessages = bucketL.numberBucketDelayedMessages + bucketR.numberBucketDelayedMessages;
-            if (numberMessages < minNumberMessages) {
-                minNumberMessages = (int) numberMessages;
-                if (bucketL.lastSegmentEntryId > bucketL.getCurrentSegmentEntryId()
-                        && bucketR.lastSegmentEntryId > bucketR.getCurrentSegmentEntryId()
-                        && bucketL.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE).isDone()
-                        && bucketR.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE).isDone()) {
-                    minIndex = i;
+            // We should skip the bucket which last segment already been load to memory, avoid record replicated index.
+            if (bucketL.lastSegmentEntryId > bucketL.getCurrentSegmentEntryId()
+                    && bucketR.lastSegmentEntryId > bucketR.getCurrentSegmentEntryId()
+                    // Skip the bucket that is merging
+                    && !bucketL.merging && !bucketR.merging){
+                long scheduleTimestamp =
+                        Math.min(bucketL.firstScheduleTimestamps.get(bucketL.currentSegmentEntryId + 1),
+                                bucketR.firstScheduleTimestamps.get(bucketR.currentSegmentEntryId + 1));
+                long numberMessages = bucketL.numberBucketDelayedMessages + bucketR.numberBucketDelayedMessages;
+                if (scheduleTimestamp <= minScheduleTimestamp) {
+                    minScheduleTimestamp = scheduleTimestamp;
+                    if (numberMessages < minNumberMessages) {
+                        minNumberMessages = numberMessages;
+                        minIndex = i;
+                    }
                 }
             }
         }
@@ -369,7 +375,14 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot() {
             log.info("[{}] Merging bucket snapshot, bucketAKey: {}, bucketBKey: {}", dispatcher.getName(),
                     immutableBucketA.bucketKey(), immutableBucketB.bucketKey());
         }
+
+        immutableBucketA.merging = true;
+        immutableBucketB.merging = true;
         return asyncMergeBucketSnapshot(immutableBucketA, immutableBucketB).whenComplete((__, ex) -> {
+            synchronized (this) {
+                immutableBucketA.merging = false;
+                immutableBucketB.merging = false;
+            }
             if (ex != null) {
                 log.error("[{}] Failed to merge bucket snapshot, bucketAKey: {}, bucketBKey: {}",
                         dispatcher.getName(), immutableBucketA.bucketKey(), immutableBucketB.bucketKey(), ex);
@@ -382,46 +395,58 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot() {
 
     private synchronized CompletableFuture asyncMergeBucketSnapshot(ImmutableBucket bucketA,
                                                                           ImmutableBucket bucketB) {
-        CompletableFuture> futureA =
-                bucketA.getRemainSnapshotSegment();
-        CompletableFuture> futureB =
-                bucketB.getRemainSnapshotSegment();
-        return futureA.thenCombine(futureB, CombinedSegmentDelayedIndexQueue::wrap)
-                .thenAccept(combinedDelayedIndexQueue -> {
-                    Pair immutableBucketDelayedIndexPair =
-                            lastMutableBucket.createImmutableBucketAndAsyncPersistent(
-                                    timeStepPerBucketSnapshotSegmentInMillis, maxIndexesPerBucketSnapshotSegment,
-                                    sharedBucketPriorityQueue, combinedDelayedIndexQueue, bucketA.startLedgerId,
-                                    bucketB.endLedgerId);
-
-                    // Merge bit map to new bucket
-                    Map delayedIndexBitMapA = bucketA.getDelayedIndexBitMap();
-                    Map delayedIndexBitMapB = bucketB.getDelayedIndexBitMap();
-                    Map delayedIndexBitMap = new HashMap<>(delayedIndexBitMapA);
-                    delayedIndexBitMapB.forEach((ledgerId, bitMapB) -> {
-                        delayedIndexBitMap.compute(ledgerId, (k, bitMapA) -> {
-                            if (bitMapA == null) {
-                                return bitMapB;
-                            }
+        CompletableFuture createAFuture = bucketA.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE);
+        CompletableFuture createBFuture = bucketB.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE);
 
-                            bitMapA.or(bitMapB);
-                            return bitMapA;
-                        });
-                    });
-                    immutableBucketDelayedIndexPair.getLeft().setDelayedIndexBitMap(delayedIndexBitMap);
-
-                    afterCreateImmutableBucket(immutableBucketDelayedIndexPair);
+        return CompletableFuture.allOf(createAFuture, createBFuture).thenCompose(bucketId -> {
+            if (INVALID_BUCKET_ID.equals(createAFuture.join()) || INVALID_BUCKET_ID.equals(createBFuture.join())) {
+                return FutureUtil.failedFuture(new RuntimeException("Can't merge buckets due to bucket create failed"));
+            }
 
-                    immutableBucketDelayedIndexPair.getLeft().getSnapshotCreateFuture()
-                            .orElse(NULL_LONG_PROMISE).thenCompose(___ -> {
-                        CompletableFuture removeAFuture = bucketA.asyncDeleteBucketSnapshot();
-                        CompletableFuture removeBFuture = bucketB.asyncDeleteBucketSnapshot();
-                        return CompletableFuture.allOf(removeAFuture, removeBFuture);
+            CompletableFuture> futureA =
+                    bucketA.getRemainSnapshotSegment();
+            CompletableFuture> futureB =
+                    bucketB.getRemainSnapshotSegment();
+            return futureA.thenCombine(futureB, CombinedSegmentDelayedIndexQueue::wrap)
+                    .thenAccept(combinedDelayedIndexQueue -> {
+                        synchronized (BucketDelayedDeliveryTracker.this) {
+                            Pair immutableBucketDelayedIndexPair =
+                                    lastMutableBucket.createImmutableBucketAndAsyncPersistent(
+                                            timeStepPerBucketSnapshotSegmentInMillis,
+                                            maxIndexesPerBucketSnapshotSegment,
+                                            sharedBucketPriorityQueue, combinedDelayedIndexQueue, bucketA.startLedgerId,
+                                            bucketB.endLedgerId);
+
+                            // Merge bit map to new bucket
+                            Map delayedIndexBitMapA = bucketA.getDelayedIndexBitMap();
+                            Map delayedIndexBitMapB = bucketB.getDelayedIndexBitMap();
+                            Map delayedIndexBitMap = new HashMap<>(delayedIndexBitMapA);
+                            delayedIndexBitMapB.forEach((ledgerId, bitMapB) -> {
+                                delayedIndexBitMap.compute(ledgerId, (k, bitMapA) -> {
+                                    if (bitMapA == null) {
+                                        return bitMapB;
+                                    }
+
+                                    bitMapA.or(bitMapB);
+                                    return bitMapA;
+                                });
+                            });
+                            immutableBucketDelayedIndexPair.getLeft().setDelayedIndexBitMap(delayedIndexBitMap);
+
+                            afterCreateImmutableBucket(immutableBucketDelayedIndexPair);
+
+                            immutableBucketDelayedIndexPair.getLeft().getSnapshotCreateFuture()
+                                    .orElse(NULL_LONG_PROMISE).thenCompose(___ -> {
+                                        CompletableFuture removeAFuture = bucketA.asyncDeleteBucketSnapshot();
+                                        CompletableFuture removeBFuture = bucketB.asyncDeleteBucketSnapshot();
+                                        return CompletableFuture.allOf(removeAFuture, removeBFuture);
+                                    });
+
+                            immutableBuckets.remove(Range.closed(bucketA.startLedgerId, bucketA.endLedgerId));
+                            immutableBuckets.remove(Range.closed(bucketB.startLedgerId, bucketB.endLedgerId));
+                        }
                     });
-
-                    immutableBuckets.remove(Range.closed(bucketA.startLedgerId, bucketA.endLedgerId));
-                    immutableBuckets.remove(Range.closed(bucketB.startLedgerId, bucketB.endLedgerId));
-                });
+        });
     }
 
     @Override
@@ -477,6 +502,12 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa
 
             ImmutableBucket bucket = snapshotSegmentLastIndexTable.get(ledgerId, entryId);
             if (bucket != null && immutableBuckets.asMapOfRanges().containsValue(bucket)) {
+                if (bucket.merging) {
+                    log.info("[{}] Skip load to wait for bucket snapshot merge finish, bucketKey:{}",
+                            dispatcher.getName(), bucket.bucketKey());
+                    break;
+                }
+
                 final int preSegmentEntryId = bucket.currentSegmentEntryId;
                 if (log.isDebugEnabled()) {
                     log.debug("[{}] Loading next bucket snapshot segment, bucketKey: {}, nextSegmentEntryId: {}",
@@ -525,7 +556,6 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa
                                     dispatcher.getName(), bucket.bucketKey(), bucket.currentSegmentEntryId);
                         }
                     }).get(AsyncOperationTimeoutSeconds * MaxRetryTimes, TimeUnit.SECONDS);
-                    snapshotSegmentLastIndexTable.remove(ledgerId, entryId);
                 } catch (Exception e) {
                     // Ignore exception to reload this segment on the next schedule.
                     log.error("[{}] An exception occurs when load next bucket snapshot, bucketKey:{}",
@@ -533,6 +563,7 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa
                     break;
                 }
             }
+            snapshotSegmentLastIndexTable.remove(ledgerId, entryId);
 
             positions.add(new PositionImpl(ledgerId, entryId));
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java
index c9223efa09243..ab1c285011d20 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java
@@ -22,6 +22,7 @@
 import static org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker.AsyncOperationTimeoutSeconds;
 import static org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker.NULL_LONG_PROMISE;
 import com.google.protobuf.ByteString;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -44,7 +45,12 @@
 class ImmutableBucket extends Bucket {
 
     @Setter
-    private volatile List snapshotSegments;
+    private List snapshotSegments;
+
+    boolean merging = false;
+
+    @Setter
+    List firstScheduleTimestamps = new ArrayList<>();
 
     ImmutableBucket(String dispatcherName, ManagedCursor cursor,
                     BucketSnapshotStorage storage, long startLedgerId, long endLedgerId) {
@@ -92,6 +98,9 @@ private CompletableFuture> asyncLoadNextBucketSnapshotEntry(b
 
                         this.setLastSegmentEntryId(metadataList.size());
                         this.recoverDelayedIndexBitMapAndNumber(nextSnapshotEntryIndex, metadataList);
+                        List firstScheduleTimestamps = metadataList.stream().map(
+                                        SnapshotSegmentMetadata::getMinScheduleTimestamp).toList();
+                        this.setFirstScheduleTimestamps(firstScheduleTimestamps);
 
                         return nextSnapshotEntryIndex + 1;
                     });
@@ -157,8 +166,10 @@ CompletableFuture>
             return bucketSnapshotStorage.getBucketSnapshotSegment(getAndUpdateBucketId(), nextSegmentEntryId,
                     lastSegmentEntryId).whenComplete((__, ex) -> {
                 if (ex != null) {
-                    log.warn("[{}] Failed to get remain bucket snapshot segment, bucketKey: {}.",
-                            dispatcherName, bucketKey(), ex);
+                    log.warn(
+                            "[{}] Failed to get remain bucket snapshot segment, bucketKey: {},"
+                                    + " nextSegmentEntryId: {}, lastSegmentEntryId: {}",
+                            dispatcherName, bucketKey(), nextSegmentEntryId, lastSegmentEntryId, ex);
                 }
             });
         }, BucketSnapshotPersistenceException.class, MaxRetryTimes);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
index e743f39e6920d..1577bf8fa51d2 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
@@ -75,11 +75,15 @@ Pair createImmutableBucketAndAsyncPersistent(
         SnapshotSegment.Builder snapshotSegmentBuilder = SnapshotSegment.newBuilder();
         SnapshotSegmentMetadata.Builder segmentMetadataBuilder = SnapshotSegmentMetadata.newBuilder();
 
+        List firstScheduleTimestamps = new ArrayList<>();
         long currentTimestampUpperLimit = 0;
+        long currentFirstTimestamp = 0L;
         while (!delayedIndexQueue.isEmpty()) {
             DelayedIndex delayedIndex = delayedIndexQueue.peek();
             long timestamp = delayedIndex.getTimestamp();
             if (currentTimestampUpperLimit == 0) {
+                currentFirstTimestamp = timestamp;
+                firstScheduleTimestamps.add(currentFirstTimestamp);
                 currentTimestampUpperLimit = timestamp + timeStepPerBucketSnapshotSegment - 1;
             }
 
@@ -104,6 +108,7 @@ Pair createImmutableBucketAndAsyncPersistent(
                     || (maxIndexesPerBucketSnapshotSegment != -1
                     && snapshotSegmentBuilder.getIndexesCount() >= maxIndexesPerBucketSnapshotSegment)) {
                 segmentMetadataBuilder.setMaxScheduleTimestamp(timestamp);
+                segmentMetadataBuilder.setMinScheduleTimestamp(currentFirstTimestamp);
                 currentTimestampUpperLimit = 0;
 
                 Iterator> iterator = bitMap.entrySet().iterator();
@@ -134,6 +139,7 @@ Pair createImmutableBucketAndAsyncPersistent(
         bucket.setCurrentSegmentEntryId(1);
         bucket.setNumberBucketDelayedMessages(numMessages);
         bucket.setLastSegmentEntryId(lastSegmentEntryId);
+        bucket.setFirstScheduleTimestamps(firstScheduleTimestamps);
 
         // Skip first segment, because it has already been loaded
         List snapshotSegments = bucketSnapshotSegments.subList(1, bucketSnapshotSegments.size());
diff --git a/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSnapshotFormat.proto b/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSnapshotFormat.proto
index 8414a583fe5b0..6996b860c5249 100644
--- a/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSnapshotFormat.proto
+++ b/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSnapshotFormat.proto
@@ -31,6 +31,7 @@ message DelayedIndex {
 message SnapshotSegmentMetadata {
     map delayed_index_bit_map = 1;
     required uint64 max_schedule_timestamp = 2;
+    required uint64 min_schedule_timestamp = 3;
 }
 
 message SnapshotSegment {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java
index 72052d22b859d..a628b58e10d32 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java
@@ -73,6 +73,7 @@ public void testCreateSnapshot() throws ExecutionException, InterruptedException
     public void testGetSnapshot() throws ExecutionException, InterruptedException {
         DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata =
                 DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder()
+                        .setMinScheduleTimestamp(System.currentTimeMillis())
                         .setMaxScheduleTimestamp(System.currentTimeMillis())
                         .putDelayedIndexBitMap(100L, ByteString.copyFrom(new byte[1])).build();
 
@@ -122,6 +123,7 @@ public void testGetSnapshotMetadata() throws ExecutionException, InterruptedExce
         DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata =
                 DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder()
                         .setMaxScheduleTimestamp(timeMillis)
+                        .setMinScheduleTimestamp(timeMillis)
                         .putAllDelayedIndexBitMap(map).build();
 
         DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata =
@@ -172,6 +174,7 @@ public void testDeleteSnapshot() throws ExecutionException, InterruptedException
     public void testGetBucketSnapshotLength() throws ExecutionException, InterruptedException {
         DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata =
                 DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder()
+                        .setMinScheduleTimestamp(System.currentTimeMillis())
                         .setMaxScheduleTimestamp(System.currentTimeMillis())
                         .putDelayedIndexBitMap(100L, ByteString.copyFrom(new byte[1])).build();
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
index 74101f00b960c..0d53c278fd2f6 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
@@ -52,6 +52,7 @@
 import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
 import org.testcontainers.shaded.org.apache.commons.lang3.mutable.MutableLong;
 import org.testcontainers.shaded.org.awaitility.Awaitility;
+import org.testng.Assert;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
@@ -251,18 +252,28 @@ public void testRoaringBitmapSerialize() {
     }
 
     @Test(dataProvider = "delayedTracker")
-    public void testMergeSnapshot(BucketDelayedDeliveryTracker tracker) {
+    public void testMergeSnapshot(final BucketDelayedDeliveryTracker tracker) {
         for (int i = 1; i <= 110; i++) {
             tracker.addMessage(i, i, i * 10);
+            Awaitility.await().untilAsserted(() -> {
+                Assert.assertTrue(
+                        tracker.getImmutableBuckets().asMapOfRanges().values().stream().noneMatch(x -> x.merging));
+            });
         }
 
         assertEquals(110, tracker.getNumberOfDelayedMessages());
 
         int size = tracker.getImmutableBuckets().asMapOfRanges().size();
 
-        assertEquals(10, size);
+        Awaitility.await().untilAsserted(() -> {
+            assertEquals(10, size);
+        });
 
         tracker.addMessage(111, 1011, 111 * 10);
+        Awaitility.await().untilAsserted(() -> {
+            Assert.assertTrue(
+                    tracker.getImmutableBuckets().asMapOfRanges().values().stream().noneMatch(x -> x.merging));
+        });
 
         MutableLong delayedMessagesInSnapshot = new MutableLong();
         tracker.getImmutableBuckets().asMapOfRanges().forEach((k, v) -> {
@@ -271,26 +282,28 @@ public void testMergeSnapshot(BucketDelayedDeliveryTracker tracker) {
 
         tracker.close();
 
-        tracker = new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock,
-                true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1,10);
+        BucketDelayedDeliveryTracker tracker2 = new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock,
+                true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1, 10);
 
-        assertEquals(tracker.getNumberOfDelayedMessages(), delayedMessagesInSnapshot.getValue());
+        assertEquals(tracker2.getNumberOfDelayedMessages(), delayedMessagesInSnapshot.getValue());
 
         for (int i = 1; i <= 110; i++) {
-            tracker.addMessage(i, i, i * 10);
+            tracker2.addMessage(i, i, i * 10);
         }
 
         clockTime.set(110 * 10);
 
-        NavigableSet scheduledMessages = tracker.getScheduledMessages(110);
+        NavigableSet scheduledMessages = tracker2.getScheduledMessages(110);
         for (int i = 1; i <= 110; i++) {
             PositionImpl position = scheduledMessages.pollFirst();
             assertEquals(position, PositionImpl.get(i, i));
         }
+
+        tracker2.close();
     }
 
     @Test(dataProvider = "delayedTracker")
-    public void testWithBkException(BucketDelayedDeliveryTracker tracker) {
+    public void testWithBkException(final BucketDelayedDeliveryTracker tracker) {
         MockBucketSnapshotStorage mockBucketSnapshotStorage = (MockBucketSnapshotStorage) bucketSnapshotStorage;
         mockBucketSnapshotStorage.injectCreateException(
                 new BucketSnapshotPersistenceException("Bookie operation timeout, op: Create entry"));
@@ -308,11 +321,25 @@ public void testWithBkException(BucketDelayedDeliveryTracker tracker) {
 
         for (int i = 1; i <= 110; i++) {
             tracker.addMessage(i, i, i * 10);
+            Awaitility.await().untilAsserted(() -> {
+                Assert.assertTrue(
+                        tracker.getImmutableBuckets().asMapOfRanges().values().stream().noneMatch(x -> x.merging));
+            });
         }
 
         assertEquals(110, tracker.getNumberOfDelayedMessages());
 
+        int size = tracker.getImmutableBuckets().asMapOfRanges().size();
+
+        Awaitility.await().untilAsserted(() -> {
+            assertEquals(10, size);
+        });
+
         tracker.addMessage(111, 1011, 111 * 10);
+        Awaitility.await().untilAsserted(() -> {
+            Assert.assertTrue(
+                    tracker.getImmutableBuckets().asMapOfRanges().values().stream().noneMatch(x -> x.merging));
+        });
 
         MutableLong delayedMessagesInSnapshot = new MutableLong();
         tracker.getImmutableBuckets().asMapOfRanges().forEach((k, v) -> {
@@ -321,11 +348,11 @@ public void testWithBkException(BucketDelayedDeliveryTracker tracker) {
 
         tracker.close();
 
-        tracker = new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock,
+        BucketDelayedDeliveryTracker tracker2 = new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock,
                 true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1,10);
 
         Long delayedMessagesInSnapshotValue = delayedMessagesInSnapshot.getValue();
-        assertEquals(tracker.getNumberOfDelayedMessages(), delayedMessagesInSnapshotValue);
+        assertEquals(tracker2.getNumberOfDelayedMessages(), delayedMessagesInSnapshotValue);
 
         clockTime.set(110 * 10);
 
@@ -338,14 +365,16 @@ public void testWithBkException(BucketDelayedDeliveryTracker tracker) {
         mockBucketSnapshotStorage.injectGetSegmentException(
                 new BucketSnapshotPersistenceException("Bookie operation timeout4, op: Get entry"));
 
-        assertEquals(tracker.getScheduledMessages(100).size(), 0);
+        assertEquals(tracker2.getScheduledMessages(100).size(), 0);
 
-        assertEquals(tracker.getScheduledMessages(100).size(), delayedMessagesInSnapshotValue);
+        assertEquals(tracker2.getScheduledMessages(100).size(), delayedMessagesInSnapshotValue);
 
         assertTrue(mockBucketSnapshotStorage.createExceptionQueue.isEmpty());
         assertTrue(mockBucketSnapshotStorage.getMetaDataExceptionQueue.isEmpty());
         assertTrue(mockBucketSnapshotStorage.getSegmentExceptionQueue.isEmpty());
         assertTrue(mockBucketSnapshotStorage.deleteExceptionQueue.isEmpty());
+
+        tracker2.close();
     }
 
     @Test(dataProvider = "delayedTracker")
@@ -390,6 +419,8 @@ public void testMaxIndexesPerSegment(BucketDelayedDeliveryTracker tracker) {
         tracker.getImmutableBuckets().asMapOfRanges().forEach((k, bucket) -> {
             assertEquals(bucket.getLastSegmentEntryId(), 4);
         });
+
+        tracker.close();
     }
     
     @Test(dataProvider = "delayedTracker")
@@ -408,5 +439,7 @@ public void testClear(BucketDelayedDeliveryTracker tracker) {
       assertEquals(tracker.getImmutableBuckets().asMapOfRanges().size(), 0);
       assertEquals(tracker.getLastMutableBucket().size(), 0);
       assertEquals(tracker.getSharedBucketPriorityQueue().size(), 0);
+
+      tracker.close();
     }
 }

From 329e80b847bd6584cbb031a96d5416d4df488432 Mon Sep 17 00:00:00 2001
From: Cong Zhao 
Date: Sun, 26 Mar 2023 00:30:40 +0800
Subject: [PATCH 146/174] [improve][broker] Clear delayed message when
 unsubscribe &  Make clear operation asynchronous (#19901)

---
 .../delayed/DelayedDeliveryTracker.java       |  5 +-
 .../InMemoryDelayedDeliveryTracker.java       |  4 +-
 .../pulsar/broker/delayed/bucket/Bucket.java  | 21 +++++--
 .../bucket/BucketDelayedDeliveryTracker.java  | 63 +++++++++++++------
 .../delayed/bucket/ImmutableBucket.java       | 57 +++--------------
 .../broker/delayed/bucket/MutableBucket.java  |  7 ++-
 .../pulsar/broker/service/Dispatcher.java     |  4 +-
 ...PersistentDispatcherMultipleConsumers.java | 23 ++++++-
 .../persistent/PersistentSubscription.java    | 11 +++-
 .../service/persistent/PersistentTopic.java   | 46 +++++++++++++-
 .../BucketDelayedDeliveryTrackerTest.java     |  7 ++-
 .../persistent/BucketDelayedDeliveryTest.java | 59 +++++++++++++++++
 12 files changed, 217 insertions(+), 90 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java
index 2f248a441cdee..3cc2da8db1e4d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java
@@ -20,6 +20,7 @@
 
 import com.google.common.annotations.Beta;
 import java.util.NavigableSet;
+import java.util.concurrent.CompletableFuture;
 import org.apache.bookkeeper.mledger.impl.PositionImpl;
 
 /**
@@ -81,8 +82,10 @@ public interface DelayedDeliveryTracker extends AutoCloseable {
 
     /**
      * Clear all delayed messages from the tracker.
+     *
+     * @return CompletableFuture
      */
-    void clear();
+    CompletableFuture clear();
 
     /**
      * Close the subscription tracker and release all resources.
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java
index f55d5fd11694b..8de6ee58e2ce5 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java
@@ -23,6 +23,7 @@
 import java.time.Clock;
 import java.util.NavigableSet;
 import java.util.TreeSet;
+import java.util.concurrent.CompletableFuture;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.bookkeeper.mledger.impl.PositionImpl;
@@ -147,8 +148,9 @@ public NavigableSet getScheduledMessages(int maxMessages) {
     }
 
     @Override
-    public void clear() {
+    public CompletableFuture clear() {
         this.priorityQueue.clear();
+        return CompletableFuture.completedFuture(null);
     }
 
     @Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java
index 7cfccff7ba328..5b7023be5034e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java
@@ -33,6 +33,7 @@
 import org.apache.bookkeeper.mledger.ManagedLedgerException;
 import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat;
 import org.apache.pulsar.common.util.Codec;
+import org.apache.pulsar.common.util.FutureUtil;
 import org.roaringbitmap.RoaringBitmap;
 
 @Slf4j
@@ -47,6 +48,9 @@ abstract class Bucket {
     protected final String dispatcherName;
 
     protected final ManagedCursor cursor;
+
+    protected final FutureUtil.Sequencer sequencer;
+
     protected final BucketSnapshotStorage bucketSnapshotStorage;
 
     long startLedgerId;
@@ -67,9 +71,10 @@ abstract class Bucket {
     private volatile CompletableFuture snapshotCreateFuture;
 
 
-    Bucket(String dispatcherName, ManagedCursor cursor,
+    Bucket(String dispatcherName, ManagedCursor cursor, FutureUtil.Sequencer sequencer,
            BucketSnapshotStorage storage, long startLedgerId, long endLedgerId) {
-        this(dispatcherName, cursor, storage, startLedgerId, endLedgerId, new HashMap<>(), -1, -1, 0, 0, null, null);
+        this(dispatcherName, cursor, sequencer, storage, startLedgerId, endLedgerId, new HashMap<>(), -1, -1, 0, 0,
+                null, null);
     }
 
     boolean containsMessage(long ledgerId, long entryId) {
@@ -154,12 +159,16 @@ CompletableFuture asyncSaveBucketSnapshot(
 
     private CompletableFuture putBucketKeyId(String bucketKey, Long bucketId) {
         Objects.requireNonNull(bucketId);
-        return executeWithRetry(() -> cursor.putCursorProperty(bucketKey, String.valueOf(bucketId)),
-                ManagedLedgerException.BadVersionException.class, MaxRetryTimes);
+        return sequencer.sequential(() -> {
+            return executeWithRetry(() -> cursor.putCursorProperty(bucketKey, String.valueOf(bucketId)),
+                    ManagedLedgerException.BadVersionException.class, MaxRetryTimes);
+        });
     }
 
     protected CompletableFuture removeBucketCursorProperty(String bucketKey) {
-        return executeWithRetry(() -> cursor.removeCursorProperty(bucketKey),
-                ManagedLedgerException.BadVersionException.class, MaxRetryTimes);
+        return sequencer.sequential(() -> {
+            return executeWithRetry(() -> cursor.removeCursorProperty(bucketKey),
+                    ManagedLedgerException.BadVersionException.class, MaxRetryTimes);
+        });
     }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
index 1cf8bd2b20d17..39973928abf5d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
@@ -31,6 +31,7 @@
 import io.netty.util.Timeout;
 import io.netty.util.Timer;
 import java.time.Clock;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -117,19 +118,23 @@ public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispat
         this.sharedBucketPriorityQueue = new TripleLongPriorityQueue();
         this.immutableBuckets = TreeRangeMap.create();
         this.snapshotSegmentLastIndexTable = HashBasedTable.create();
-        this.lastMutableBucket = new MutableBucket(dispatcher.getName(), dispatcher.getCursor(), bucketSnapshotStorage);
+        this.lastMutableBucket =
+                new MutableBucket(dispatcher.getName(), dispatcher.getCursor(), FutureUtil.Sequencer.create(),
+                        bucketSnapshotStorage);
         this.numberDelayedMessages = recoverBucketSnapshot();
     }
 
     private synchronized long recoverBucketSnapshot() throws RuntimeException {
         ManagedCursor cursor = this.lastMutableBucket.getCursor();
+        FutureUtil.Sequencer sequencer = this.lastMutableBucket.getSequencer();
         Map, ImmutableBucket> toBeDeletedBucketMap = new HashMap<>();
         cursor.getCursorProperties().keySet().forEach(key -> {
             if (key.startsWith(DELAYED_BUCKET_KEY_PREFIX)) {
                 String[] keys = key.split(DELIMITER);
                 checkArgument(keys.length == 3);
                 ImmutableBucket immutableBucket =
-                        new ImmutableBucket(dispatcher.getName(), cursor, this.lastMutableBucket.bucketSnapshotStorage,
+                        new ImmutableBucket(dispatcher.getName(), cursor, sequencer,
+                                this.lastMutableBucket.bucketSnapshotStorage,
                                 Long.parseLong(keys[1]), Long.parseLong(keys[2]));
                 putAndCleanOverlapRange(Range.closed(immutableBucket.startLedgerId, immutableBucket.endLedgerId),
                         immutableBucket, toBeDeletedBucketMap);
@@ -274,7 +279,7 @@ private void afterCreateImmutableBucket(Pair immu
                         });
 
                         immutableBucket.setCurrentSegmentEntryId(immutableBucket.lastSegmentEntryId);
-                        immutableBuckets.remove(
+                        immutableBuckets.asMapOfRanges().remove(
                                 Range.closed(immutableBucket.startLedgerId, immutableBucket.endLedgerId));
                         snapshotSegmentLastIndexTable.remove(lastDelayedIndex.getLedgerId(),
                                 lastDelayedIndex.getTimestamp());
@@ -442,8 +447,9 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot(ImmutableB
                                         return CompletableFuture.allOf(removeAFuture, removeBFuture);
                                     });
 
-                            immutableBuckets.remove(Range.closed(bucketA.startLedgerId, bucketA.endLedgerId));
-                            immutableBuckets.remove(Range.closed(bucketB.startLedgerId, bucketB.endLedgerId));
+                            Map, ImmutableBucket> immutableBucketMap = immutableBuckets.asMapOfRanges();
+                            immutableBucketMap.remove(Range.closed(bucketA.startLedgerId, bucketA.endLedgerId));
+                            immutableBucketMap.remove(Range.closed(bucketB.startLedgerId, bucketB.endLedgerId));
                         }
                     });
         });
@@ -525,14 +531,15 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa
                     }
 
                     if (bucket.currentSegmentEntryId == bucket.lastSegmentEntryId) {
-                        immutableBuckets.remove(Range.closed(bucket.startLedgerId, bucket.endLedgerId));
+                        immutableBuckets.asMapOfRanges().remove(Range.closed(bucket.startLedgerId, bucket.endLedgerId));
                         bucket.asyncDeleteBucketSnapshot();
                         continue;
                     }
 
                     bucket.asyncLoadNextBucketSnapshotEntry().thenAccept(indexList -> {
                         if (CollectionUtils.isEmpty(indexList)) {
-                            immutableBuckets.remove(Range.closed(bucket.startLedgerId, bucket.endLedgerId));
+                            immutableBuckets.asMapOfRanges()
+                                    .remove(Range.closed(bucket.startLedgerId, bucket.endLedgerId));
                             bucket.asyncDeleteBucketSnapshot();
                             return;
                         }
@@ -549,13 +556,13 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa
                             // Back bucket state
                             bucket.setCurrentSegmentEntryId(preSegmentEntryId);
 
-                            log.error("[{}] Failed to load bucket snapshot segment, bucketKey: {}",
-                                    dispatcher.getName(), bucket.bucketKey(), ex);
+                            log.error("[{}] Failed to load bucket snapshot segment, bucketKey: {}, segmentEntryId: {}",
+                                    dispatcher.getName(), bucket.bucketKey(), preSegmentEntryId + 1, ex);
                         } else {
                             log.info("[{}] Load next bucket snapshot segment finish, bucketKey: {}, segmentEntryId: {}",
-                                    dispatcher.getName(), bucket.bucketKey(), bucket.currentSegmentEntryId);
+                                    dispatcher.getName(), bucket.bucketKey(), preSegmentEntryId + 1);
                         }
-                    }).get(AsyncOperationTimeoutSeconds * MaxRetryTimes, TimeUnit.SECONDS);
+                    }).get(AsyncOperationTimeoutSeconds * (MaxRetryTimes + 1), TimeUnit.SECONDS);
                 } catch (Exception e) {
                     // Ignore exception to reload this segment on the next schedule.
                     log.error("[{}] An exception occurs when load next bucket snapshot, bucketKey:{}",
@@ -585,30 +592,46 @@ public boolean shouldPauseAllDeliveries() {
     }
 
     @Override
-    public synchronized void clear() {
-        cleanImmutableBuckets(true);
-        sharedBucketPriorityQueue.clear();
-        lastMutableBucket.clear();
-        snapshotSegmentLastIndexTable.clear();
-        numberDelayedMessages = 0;
+    public synchronized CompletableFuture clear() {
+        return cleanImmutableBuckets(true).thenRun(() -> {
+            synchronized (this) {
+                sharedBucketPriorityQueue.clear();
+                lastMutableBucket.clear();
+                snapshotSegmentLastIndexTable.clear();
+                numberDelayedMessages = 0;
+            }
+        });
     }
 
     @Override
     public synchronized void close() {
         super.close();
         lastMutableBucket.close();
-        cleanImmutableBuckets(false);
         sharedBucketPriorityQueue.close();
+        try {
+            cleanImmutableBuckets(false).get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS);
+        } catch (Exception e) {
+            log.warn("[{}] Failed wait to snapshot generate", dispatcher.getName(), e);
+        }
     }
 
-    private void cleanImmutableBuckets(boolean delete) {
+    private CompletableFuture cleanImmutableBuckets(boolean delete) {
         if (immutableBuckets != null) {
+            List> futures = new ArrayList<>();
             Iterator iterator = immutableBuckets.asMapOfRanges().values().iterator();
             while (iterator.hasNext()) {
                 ImmutableBucket bucket = iterator.next();
-                bucket.clear(delete);
+                if (delete) {
+                    futures.add(bucket.clear());
+                } else {
+                    bucket.getSnapshotCreateFuture().ifPresent(future -> futures.add(future.thenApply(x -> null)));
+                }
+                numberDelayedMessages -= bucket.getNumberBucketDelayedMessages();
                 iterator.remove();
             }
+            return FutureUtil.waitForAll(futures);
+        } else {
+            return CompletableFuture.completedFuture(null);
         }
     }
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java
index ab1c285011d20..969d326e28187 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java
@@ -19,7 +19,6 @@
 package org.apache.pulsar.broker.delayed.bucket;
 
 import static org.apache.bookkeeper.mledger.util.Futures.executeWithRetry;
-import static org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker.AsyncOperationTimeoutSeconds;
 import static org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker.NULL_LONG_PROMISE;
 import com.google.protobuf.ByteString;
 import java.util.ArrayList;
@@ -28,7 +27,6 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.TimeUnit;
 import java.util.function.Supplier;
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
@@ -38,6 +36,7 @@
 import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat;
 import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex;
 import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata;
+import org.apache.pulsar.common.util.FutureUtil;
 import org.roaringbitmap.RoaringBitmap;
 import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
 
@@ -52,9 +51,9 @@ class ImmutableBucket extends Bucket {
     @Setter
     List firstScheduleTimestamps = new ArrayList<>();
 
-    ImmutableBucket(String dispatcherName, ManagedCursor cursor,
+    ImmutableBucket(String dispatcherName, ManagedCursor cursor, FutureUtil.Sequencer sequencer,
                     BucketSnapshotStorage storage, long startLedgerId, long endLedgerId) {
-        super(dispatcherName, cursor, storage, startLedgerId, endLedgerId);
+        super(dispatcherName, cursor, sequencer, storage, startLedgerId, endLedgerId);
     }
 
     public Optional> getSnapshotSegments() {
@@ -117,8 +116,9 @@ private CompletableFuture> asyncLoadNextBucketSnapshotEntry(b
                     () -> bucketSnapshotStorage.getBucketSnapshotSegment(bucketId, nextSegmentEntryId,
                             nextSegmentEntryId).whenComplete((___, ex) -> {
                         if (ex != null) {
-                            log.warn("[{}] Failed to get bucket snapshot segment. bucketKey: {}, bucketId: {}",
-                                    dispatcherName, bucketKey(), bucketId, ex);
+                            log.warn("[{}] Failed to get bucket snapshot segment. bucketKey: {},"
+                                            + " bucketId: {}, segmentEntryId: {}", dispatcherName, bucketKey(),
+                                    bucketId, nextSegmentEntryId, ex);
                         }
                     }), BucketSnapshotPersistenceException.class, MaxRetryTimes)
                     .thenApply(bucketSnapshotSegments -> {
@@ -191,48 +191,9 @@ CompletableFuture asyncDeleteBucketSnapshot() {
         });
     }
 
-    void clear(boolean delete) {
+    CompletableFuture clear() {
         delayedIndexBitMap.clear();
-        if (delete) {
-            final String bucketKey = bucketKey();
-            try {
-                getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE).exceptionally(e -> null).thenCompose(__ -> {
-                        if (getSnapshotCreateFuture().isPresent() && getBucketId().isEmpty()) {
-                            log.error("[{}] Can't found bucketId, don't execute delete operate, bucketKey: {}",
-                                    dispatcherName, bucketKey);
-                            return CompletableFuture.completedFuture(null);
-                        }
-                        long bucketId = getAndUpdateBucketId();
-                        return removeBucketCursorProperty(bucketKey()).thenAccept(___ -> {
-                            executeWithRetry(() -> bucketSnapshotStorage.deleteBucketSnapshot(bucketId),
-                            BucketSnapshotPersistenceException.class, MaxRetryTimes)
-                            .whenComplete((____, ex) -> {
-                                if (ex != null) {
-                                    log.error("[{}] Failed to delete bucket snapshot, bucketId: {}, bucketKey: {}",
-                                            dispatcherName, bucketId, bucketKey, ex);
-                                } else {
-                                    log.info("[{}] Delete bucket snapshot finish, bucketId: {}, bucketKey: {}",
-                                            dispatcherName, bucketId, bucketKey);
-                                }
-                            });
-                    });
-                }).get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS);
-            } catch (Exception e) {
-                log.error("Failed to clear bucket snapshot, bucketKey: {}", bucketKey, e);
-                if (e instanceof InterruptedException) {
-                    Thread.currentThread().interrupt();
-                }
-                throw new RuntimeException(e);
-            }
-        } else {
-            getSnapshotCreateFuture().ifPresent(snapshotGenerateFuture -> {
-                try {
-                    snapshotGenerateFuture.get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS);
-                } catch (Exception e) {
-                    log.warn("[{}] Failed wait to snapshot generate, bucketId: {}, bucketKey: {}", dispatcherName,
-                            getBucketId(), bucketKey());
-                }
-            });
-        }
+        return getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE).exceptionally(e -> null)
+                .thenCompose(__ -> asyncDeleteBucketSnapshot());
     }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
index 1577bf8fa51d2..f8a4ecc7a4ddb 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
@@ -34,6 +34,7 @@
 import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata;
 import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment;
 import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata;
+import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue;
 import org.roaringbitmap.RoaringBitmap;
 
@@ -42,9 +43,9 @@ class MutableBucket extends Bucket implements AutoCloseable {
 
     private final TripleLongPriorityQueue priorityQueue;
 
-    MutableBucket(String dispatcherName, ManagedCursor cursor,
+    MutableBucket(String dispatcherName, ManagedCursor cursor, FutureUtil.Sequencer sequencer,
                   BucketSnapshotStorage bucketSnapshotStorage) {
-        super(dispatcherName, cursor, bucketSnapshotStorage, -1L, -1L);
+        super(dispatcherName, cursor, sequencer, bucketSnapshotStorage, -1L, -1L);
         this.priorityQueue = new TripleLongPriorityQueue();
     }
 
@@ -134,7 +135,7 @@ Pair createImmutableBucketAndAsyncPersistent(
 
         final int lastSegmentEntryId = segmentMetadataList.size();
 
-        ImmutableBucket bucket = new ImmutableBucket(dispatcherName, cursor, bucketSnapshotStorage,
+        ImmutableBucket bucket = new ImmutableBucket(dispatcherName, cursor, sequencer, bucketSnapshotStorage,
                 startLedgerId, endLedgerId);
         bucket.setCurrentSegmentEntryId(1);
         bucket.setNumberBucketDelayedMessages(numMessages);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java
index 48877ce53f801..9b0c4e885e64d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java
@@ -110,8 +110,8 @@ default long getNumberOfDelayedMessages() {
         return 0;
     }
 
-    default void clearDelayedMessages() {
-        //No-op
+    default CompletableFuture clearDelayedMessages() {
+        return CompletableFuture.completedFuture(null);
     }
 
     default void cursorIsReset() {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
index 059820b1b66c3..e5c9e85bac3f2 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
@@ -48,6 +48,7 @@
 import org.apache.bookkeeper.mledger.impl.PositionImpl;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.pulsar.broker.delayed.AbstractDelayedDeliveryTracker;
+import org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTrackerFactory;
 import org.apache.pulsar.broker.delayed.DelayedDeliveryTracker;
 import org.apache.pulsar.broker.delayed.InMemoryDelayedDeliveryTracker;
 import org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker;
@@ -1089,8 +1090,26 @@ public synchronized long getNumberOfDelayedMessages() {
     }
 
     @Override
-    public void clearDelayedMessages() {
-        this.delayedDeliveryTracker.ifPresent(DelayedDeliveryTracker::clear);
+    public CompletableFuture clearDelayedMessages() {
+        if (!topic.isDelayedDeliveryEnabled()) {
+            return CompletableFuture.completedFuture(null);
+        }
+
+        if (delayedDeliveryTracker.isEmpty() && topic.getBrokerService()
+                .getDelayedDeliveryTrackerFactory() instanceof BucketDelayedDeliveryTrackerFactory) {
+            synchronized (this) {
+                if (delayedDeliveryTracker.isEmpty()) {
+                    delayedDeliveryTracker = Optional
+                            .of(topic.getBrokerService().getDelayedDeliveryTrackerFactory().newTracker(this));
+                }
+            }
+        }
+
+        if (delayedDeliveryTracker.isPresent()) {
+            return this.delayedDeliveryTracker.get().clear();
+        } else {
+            return CompletableFuture.completedFuture(null);
+        }
     }
 
     @Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
index e07d7bee500a7..bdb3c9fc391ce 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
@@ -649,9 +649,16 @@ public void clearBacklogComplete(Object ctx) {
                             cursor.getNumberOfEntriesInBacklog(false));
                 }
                 if (dispatcher != null) {
-                    dispatcher.clearDelayedMessages();
+                    dispatcher.clearDelayedMessages().whenComplete((__, ex) -> {
+                        if (ex != null) {
+                            future.completeExceptionally(ex);
+                        } else {
+                            future.complete(null);
+                        }
+                    });
+                } else {
+                    future.complete(null);
                 }
-                future.complete(null);
             }
 
             @Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
index b3b6526eea54d..82a4f5312357a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
@@ -79,6 +79,7 @@
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.pulsar.broker.PulsarServerException;
+import org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTrackerFactory;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateCompactionStrategy;
 import org.apache.pulsar.broker.namespace.NamespaceService;
@@ -1078,13 +1079,13 @@ public CompletableFuture unsubscribe(String subscriptionName) {
                     new AsyncCallbacks.DeleteLedgerCallback() {
                         @Override
                         public void deleteLedgerComplete(Object ctx) {
-                            asyncDeleteCursor(subscriptionName, unsubscribeFuture);
+                            asyncDeleteCursorWithClearDelayedMessage(subscriptionName, unsubscribeFuture);
                         }
 
                         @Override
                         public void deleteLedgerFailed(ManagedLedgerException exception, Object ctx) {
                             if (exception instanceof MetadataNotFoundException) {
-                                asyncDeleteCursor(subscriptionName, unsubscribeFuture);
+                                asyncDeleteCursorWithClearDelayedMessage(subscriptionName, unsubscribeFuture);
                                 return;
                             }
 
@@ -1094,12 +1095,51 @@ public void deleteLedgerFailed(ManagedLedgerException exception, Object ctx) {
                         }
                     }, null);
         } else {
-            asyncDeleteCursor(subscriptionName, unsubscribeFuture);
+            asyncDeleteCursorWithClearDelayedMessage(subscriptionName, unsubscribeFuture);
         }
 
         return unsubscribeFuture;
     }
 
+    private void asyncDeleteCursorWithClearDelayedMessage(String subscriptionName,
+                                                          CompletableFuture unsubscribeFuture) {
+        if (!isDelayedDeliveryEnabled()
+                || !(brokerService.getDelayedDeliveryTrackerFactory() instanceof BucketDelayedDeliveryTrackerFactory)) {
+            asyncDeleteCursor(subscriptionName, unsubscribeFuture);
+            return;
+        }
+
+        PersistentSubscription persistentSubscription = subscriptions.get(subscriptionName);
+        if (persistentSubscription == null) {
+            log.warn("[{}][{}] Can't find subscription, skip clear delayed message", topic, subscriptionName);
+            asyncDeleteCursor(subscriptionName, unsubscribeFuture);
+            return;
+        }
+
+        Dispatcher dispatcher = persistentSubscription.getDispatcher();
+        final Dispatcher temporaryDispatcher;
+        if (dispatcher == null) {
+            log.info("[{}][{}] Dispatcher is null, try to create temporary dispatcher to clear delayed message", topic,
+                    subscriptionName);
+            dispatcher = temporaryDispatcher =
+                    new PersistentDispatcherMultipleConsumers(this, persistentSubscription.cursor,
+                            persistentSubscription);
+        } else {
+            temporaryDispatcher = null;
+        }
+
+        dispatcher.clearDelayedMessages().whenComplete((__, ex) -> {
+            if (ex != null) {
+                unsubscribeFuture.completeExceptionally(ex);
+            } else {
+                asyncDeleteCursor(subscriptionName, unsubscribeFuture);
+            }
+            if (temporaryDispatcher != null) {
+                temporaryDispatcher.close();
+            }
+        });
+    }
+
     private void asyncDeleteCursor(String subscriptionName, CompletableFuture unsubscribeFuture) {
         ledger.asyncDeleteCursor(Codec.encode(subscriptionName), new DeleteCursorCallback() {
             @Override
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
index 0d53c278fd2f6..fbb866d48ed69 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
@@ -39,7 +39,9 @@
 import java.util.NavigableSet;
 import java.util.Set;
 import java.util.TreeMap;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicLong;
 import org.apache.bookkeeper.mledger.ManagedCursor;
 import org.apache.bookkeeper.mledger.impl.PositionImpl;
@@ -424,7 +426,8 @@ public void testMaxIndexesPerSegment(BucketDelayedDeliveryTracker tracker) {
     }
     
     @Test(dataProvider = "delayedTracker")
-    public void testClear(BucketDelayedDeliveryTracker tracker) {
+    public void testClear(BucketDelayedDeliveryTracker tracker)
+            throws ExecutionException, InterruptedException, TimeoutException {
       for (int i = 1; i <= 1001; i++) {
           tracker.addMessage(i, i, i * 10);
       }
@@ -433,7 +436,7 @@ public void testClear(BucketDelayedDeliveryTracker tracker) {
       assertTrue(tracker.getImmutableBuckets().asMapOfRanges().size() > 0);
       assertEquals(tracker.getLastMutableBucket().size(), 1);
 
-      tracker.clear();
+      tracker.clear().get(1, TimeUnit.MINUTES);
 
       assertEquals(tracker.getNumberOfDelayedMessages(), 0);
       assertEquals(tracker.getImmutableBuckets().asMapOfRanges().size(), 0);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java
index 292889e8c159a..09b7cbbf1b99d 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java
@@ -18,9 +18,15 @@
  */
 package org.apache.pulsar.broker.service.persistent;
 
+import static org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.CURSOR_INTERNAL_PROPERTY_PREFIX;
+import static org.testng.Assert.assertTrue;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import lombok.Cleanup;
+import org.apache.bookkeeper.client.BKException;
+import org.apache.bookkeeper.client.BookKeeper;
+import org.apache.bookkeeper.client.LedgerHandle;
 import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl;
 import org.apache.pulsar.broker.BrokerTestUtil;
 import org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTrackerFactory;
@@ -103,4 +109,57 @@ public void testBucketDelayedDeliveryWithAllConsumersDisconnecting() throws Exce
         Awaitility.await().untilAsserted(() -> Assert.assertEquals(dispatcher2.getNumberOfDelayedMessages(), 1000));
         Assert.assertEquals(bucketKeys, bucketKeys2);
     }
+
+
+    @Test
+    public void testUnsubscribe() throws Exception {
+        String topic = BrokerTestUtil.newUniqueName("persistent://public/default/testUnsubscribes");
+
+        @Cleanup
+        Consumer c1 = pulsarClient.newConsumer(Schema.STRING)
+                .topic(topic)
+                .subscriptionName("sub")
+                .subscriptionType(SubscriptionType.Shared)
+                .subscribe();
+
+        @Cleanup
+        Producer producer = pulsarClient.newProducer(Schema.STRING)
+                .topic(topic)
+                .create();
+
+        for (int i = 0; i < 1000; i++) {
+            producer.newMessage()
+                    .value("msg")
+                    .deliverAfter(1, TimeUnit.HOURS)
+                    .send();
+        }
+
+        Dispatcher dispatcher = pulsar.getBrokerService().getTopicReference(topic).get().getSubscription("sub").getDispatcher();
+        Awaitility.await().untilAsserted(() -> Assert.assertEquals(dispatcher.getNumberOfDelayedMessages(), 1000));
+
+        Map cursorProperties =
+                ((PersistentDispatcherMultipleConsumers) dispatcher).getCursor().getCursorProperties();
+        List bucketIds = cursorProperties.entrySet().stream()
+                .filter(x -> x.getKey().startsWith(CURSOR_INTERNAL_PROPERTY_PREFIX + "delayed.bucket")).map(
+                        x -> Long.valueOf(x.getValue())).toList();
+
+        assertTrue(bucketIds.size() > 0);
+
+        c1.close();
+
+        restartBroker();
+
+        admin.topics().deleteSubscription(topic, "sub");
+
+        for (Long bucketId : bucketIds) {
+            try {
+                LedgerHandle ledgerHandle =
+                        pulsarTestContext.getBookKeeperClient()
+                                .openLedger(bucketId, BookKeeper.DigestType.CRC32C, new byte[]{});
+                Assert.fail("Should fail");
+            } catch (BKException.BKNoSuchLedgerExistsException e) {
+                // ignore it
+            }
+        }
+    }
 }

From 2b4a3c14458f564e7e0178f71979a41e7b7a42b7 Mon Sep 17 00:00:00 2001
From: Zixuan Liu 
Date: Sun, 26 Mar 2023 22:36:34 +0800
Subject: [PATCH 147/174] [fix][broker] Fix RetentionPolicies constructor
 (#19777)

Signed-off-by: Zixuan Liu 
---
 .../apache/bookkeeper/mledger/ManagedLedgerConfig.java    | 8 ++++----
 .../pulsar/broker/service/ConsumedLedgersTrimTest.java    | 2 +-
 .../pulsar/common/policies/data/RetentionPolicies.java    | 6 +++---
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java
index 6e88a8e650d58..0c93a5b642cf6 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java
@@ -62,7 +62,7 @@ public class ManagedLedgerConfig {
     private int ledgerRolloverTimeout = 4 * 3600;
     private double throttleMarkDelete = 0;
     private long retentionTimeMs = 0;
-    private int retentionSizeInMB = 0;
+    private long retentionSizeInMB = 0;
     private boolean autoSkipNonRecoverableData;
     private boolean lazyCursorRecovery = false;
     private long metadataOperationsTimeoutSeconds = 60;
@@ -396,7 +396,7 @@ public ManagedLedgerConfig setThrottleMarkDelete(double throttleMarkDelete) {
     /**
      * Set the retention time for the ManagedLedger.
      * 

-     * Retention time and retention size ({@link #setRetentionSizeInMB(int)}) are together used to retain the
+     * Retention time and retention size ({@link #setRetentionSizeInMB(long)}) are together used to retain the
      * ledger data when there are no cursors or when all the cursors have marked the data for deletion.
      * Data will be deleted in this case when both retention time and retention size settings don't prevent deleting
      * the data marked for deletion.
@@ -438,7 +438,7 @@ public long getRetentionTimeMillis() {
      * @param retentionSizeInMB
      *            quota for message retention
      */
-    public ManagedLedgerConfig setRetentionSizeInMB(int retentionSizeInMB) {
+    public ManagedLedgerConfig setRetentionSizeInMB(long retentionSizeInMB) {
         this.retentionSizeInMB = retentionSizeInMB;
         return this;
     }
@@ -447,7 +447,7 @@ public ManagedLedgerConfig setRetentionSizeInMB(int retentionSizeInMB) {
      * @return quota for message retention
      *
      */
-    public int getRetentionSizeInMB() {
+    public long getRetentionSizeInMB() {
         return retentionSizeInMB;
     }
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumedLedgersTrimTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumedLedgersTrimTest.java
index 099a9028c4645..80db4c30f454d 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumedLedgersTrimTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumedLedgersTrimTest.java
@@ -86,7 +86,7 @@ public void TestConsumedLedgersTrim() throws Exception {
         PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topicName).get();
 
         ManagedLedgerConfig managedLedgerConfig = persistentTopic.getManagedLedger().getConfig();
-        managedLedgerConfig.setRetentionSizeInMB(1);
+        managedLedgerConfig.setRetentionSizeInMB(1L);
         managedLedgerConfig.setRetentionTime(1, TimeUnit.SECONDS);
         managedLedgerConfig.setMaxEntriesPerLedger(2);
         managedLedgerConfig.setMinimumRolloverTime(1, TimeUnit.MILLISECONDS);
diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/RetentionPolicies.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/RetentionPolicies.java
index 4206220c5ee58..8d5b25da43153 100644
--- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/RetentionPolicies.java
+++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/RetentionPolicies.java
@@ -29,13 +29,13 @@
  */
 public class RetentionPolicies {
     private int retentionTimeInMinutes;
-    private int retentionSizeInMB;
+    private long retentionSizeInMB;
 
     public RetentionPolicies() {
         this(0, 0);
     }
 
-    public RetentionPolicies(int retentionTimeInMinutes, int retentionSizeInMB) {
+    public RetentionPolicies(int retentionTimeInMinutes, long retentionSizeInMB) {
         this.retentionSizeInMB = retentionSizeInMB;
         this.retentionTimeInMinutes = retentionTimeInMinutes;
     }
@@ -44,7 +44,7 @@ public int getRetentionTimeInMinutes() {
         return retentionTimeInMinutes;
     }
 
-    public int getRetentionSizeInMB() {
+    public long getRetentionSizeInMB() {
         return retentionSizeInMB;
     }
 

From ef18bab1badbe6ce537254b3ff8fd288da1e7d4d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tu=E1=BA=A5n=20V=C6=B0=C6=A1ng?= 
Date: Mon, 27 Mar 2023 11:59:50 +0700
Subject: [PATCH 148/174] [improve] Allow download link with basic auth
 (#19750)

Co-authored-by: tison 
---
 .../impl/auth/AuthenticationDataBasic.java    | 20 ++++++-----
 .../functions/utils/FunctionCommon.java       | 14 ++++++--
 .../functions/utils/FunctionCommonTest.java   | 35 ++++++++++++-------
 3 files changed, 46 insertions(+), 23 deletions(-)

diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/AuthenticationDataBasic.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/AuthenticationDataBasic.java
index 2fc89e128ec58..82de6dc198d77 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/AuthenticationDataBasic.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/AuthenticationDataBasic.java
@@ -27,16 +27,20 @@
 
 public class AuthenticationDataBasic implements AuthenticationDataProvider {
     private static final String HTTP_HEADER_NAME = "Authorization";
-    private String httpAuthToken;
-    private String commandAuthToken;
-    private Map headers = new HashMap<>();
+    private final String commandAuthToken;
+    private final Map headers;
 
     public AuthenticationDataBasic(String userId, String password) {
-        httpAuthToken = "Basic " + Base64.getEncoder().encodeToString((userId + ":" + password).getBytes());
-        commandAuthToken = userId + ":" + password;
-        headers.put(HTTP_HEADER_NAME, httpAuthToken);
-        headers.put(PULSAR_AUTH_METHOD_NAME, AuthenticationBasic.AUTH_METHOD_NAME);
-        this.headers = Collections.unmodifiableMap(this.headers);
+        this(userId + ":" + password);
+    }
+
+    public AuthenticationDataBasic(String userInfo) {
+        String httpAuthToken = "Basic " + Base64.getEncoder().encodeToString(userInfo.getBytes());
+        this.commandAuthToken = userInfo;
+        this.headers = Collections.unmodifiableMap(new HashMap(){{
+            put(HTTP_HEADER_NAME, httpAuthToken);
+            put(PULSAR_AUTH_METHOD_NAME, AuthenticationBasic.AUTH_METHOD_NAME);
+        }});
     }
 
     @Override
diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java
index d3ce9d93a2d36..28cce0fe62209 100644
--- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java
+++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java
@@ -35,9 +35,11 @@
 import java.net.ServerSocket;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.net.URLConnection;
 import java.nio.file.Files;
 import java.nio.file.StandardCopyOption;
 import java.util.Collection;
+import java.util.Map;
 import java.util.UUID;
 import lombok.AccessLevel;
 import lombok.NoArgsConstructor;
@@ -48,6 +50,7 @@
 import org.apache.pulsar.client.api.MessageId;
 import org.apache.pulsar.client.api.SubscriptionInitialPosition;
 import org.apache.pulsar.client.impl.MessageIdImpl;
+import org.apache.pulsar.client.impl.auth.AuthenticationDataBasic;
 import org.apache.pulsar.common.functions.FunctionConfig;
 import org.apache.pulsar.common.functions.Utils;
 import org.apache.pulsar.common.nar.NarClassLoader;
@@ -245,8 +248,15 @@ public static Class getSinkType(Class sinkClass) {
     }
 
     public static void downloadFromHttpUrl(String destPkgUrl, File targetFile) throws IOException {
-        URL website = new URL(destPkgUrl);
-        try (InputStream in = website.openStream()) {
+        final URL url = new URL(destPkgUrl);
+        final URLConnection connection = url.openConnection();
+        if (StringUtils.isNotEmpty(url.getUserInfo())) {
+            final AuthenticationDataBasic authBasic = new AuthenticationDataBasic(url.getUserInfo());
+            for (Map.Entry header : authBasic.getHttpHeaders()) {
+                connection.setRequestProperty(header.getKey(), header.getValue());
+            }
+        }
+        try (InputStream in = connection.getInputStream()) {
             log.info("Downloading function package from {} to {} ...", destPkgUrl, targetFile.getAbsoluteFile());
             Files.copy(in, targetFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
         }
diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionCommonTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionCommonTest.java
index aae2292752005..113824fc7c1a1 100644
--- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionCommonTest.java
+++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionCommonTest.java
@@ -18,6 +18,11 @@
  */
 package org.apache.pulsar.functions.utils;
 
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
+import java.io.File;
 import java.util.Collection;
 import org.apache.pulsar.client.impl.MessageIdImpl;
 import org.apache.pulsar.common.util.FutureUtil;
@@ -26,17 +31,11 @@
 import org.apache.pulsar.functions.api.Record;
 import org.apache.pulsar.functions.api.WindowContext;
 import org.apache.pulsar.functions.api.WindowFunction;
+import org.assertj.core.util.Files;
 import org.testng.Assert;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
-import java.io.File;
-import java.util.UUID;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-import static org.testng.Assert.assertEquals;
-
 /**
  * Unit test of {@link Exceptions}.
  */
@@ -78,12 +77,22 @@ public void testValidateHttpFileUrl() throws Exception {
 
     @Test
     public void testDownloadFile() throws Exception {
-        String jarHttpUrl = "https://repo1.maven.org/maven2/org/apache/pulsar/pulsar-common/2.4.2/pulsar-common-2.4.2.jar";
-        String testDir = FunctionCommonTest.class.getProtectionDomain().getCodeSource().getLocation().getPath();
-        File pkgFile = new File(testDir, UUID.randomUUID().toString());
-        FunctionCommon.downloadFromHttpUrl(jarHttpUrl, pkgFile);
-        Assert.assertTrue(pkgFile.exists());
-        pkgFile.delete();
+        final String jarHttpUrl = "https://repo1.maven.org/maven2/org/apache/pulsar/pulsar-common/2.4.2/pulsar-common-2.4.2.jar";
+        final File file = Files.newTemporaryFile();
+        file.deleteOnExit();
+        assertThat(file.length()).isZero();
+        FunctionCommon.downloadFromHttpUrl(jarHttpUrl, file);
+        assertThat(file.length()).isGreaterThan(0);
+    }
+
+    @Test
+    public void testDownloadFileWithBasicAuth() throws Exception {
+        final String jarHttpUrl = "https://foo:bar@httpbin.org/basic-auth/foo/bar";
+        final File file = Files.newTemporaryFile();
+        file.deleteOnExit();
+        assertThat(file.length()).isZero();
+        FunctionCommon.downloadFromHttpUrl(jarHttpUrl, file);
+        assertThat(file.length()).isGreaterThan(0);
     }
 
     @Test

From aac99c86ea39ad673bce32240056fe308f546bef Mon Sep 17 00:00:00 2001
From: Yunze Xu 
Date: Mon, 27 Mar 2023 14:24:18 +0800
Subject: [PATCH 149/174] [fix][client] Fix NPE when acknowledging multiple
 messages (#19874)

---
 .../client/api/ConsumerAckListTest.java       | 39 +++++++++++++++++++
 .../client/impl/MultiTopicsConsumerImpl.java  | 13 ++++++-
 2 files changed, 50 insertions(+), 2 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerAckListTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerAckListTest.java
index d01e5b764a7a1..baf0000be6e4f 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerAckListTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerAckListTest.java
@@ -111,4 +111,43 @@ private void sendMessagesAsyncAndWait(Producer producer, int messages) t
         latch.await();
     }
 
+    @Test(timeOut = 30000)
+    public void testAckMessageInAnotherTopic() throws Exception {
+        final String[] topics = {
+                "persistent://my-property/my-ns/test-ack-message-in-other-topic1" + UUID.randomUUID(),
+                "persistent://my-property/my-ns/test-ack-message-in-other-topic2" + UUID.randomUUID(),
+                "persistent://my-property/my-ns/test-ack-message-in-other-topic3" + UUID.randomUUID()
+        };
+        @Cleanup final Consumer allTopicsConsumer = pulsarClient.newConsumer(Schema.STRING)
+                .topic(topics)
+                .subscriptionName("sub1")
+                .subscribe();
+        Consumer partialTopicsConsumer = pulsarClient.newConsumer(Schema.STRING)
+                .topic(topics[0], topics[1])
+                .subscriptionName("sub2")
+                .subscribe();
+        for (int i = 0; i < topics.length; i++) {
+            final Producer producer = pulsarClient.newProducer(Schema.STRING)
+                    .topic(topics[i])
+                    .create();
+            producer.send("msg-" + i);
+            producer.close();
+        }
+        final List messageIdList = new ArrayList<>();
+        for (int i = 0; i < topics.length; i++) {
+            messageIdList.add(allTopicsConsumer.receive().getMessageId());
+        }
+        try {
+            partialTopicsConsumer.acknowledge(messageIdList);
+            Assert.fail();
+        } catch (PulsarClientException.NotConnectedException ignored) {
+        }
+        partialTopicsConsumer.close();
+        partialTopicsConsumer = pulsarClient.newConsumer(Schema.STRING).topic(topics[0])
+                .subscriptionName("sub2").subscribe();
+        pulsarClient.newProducer(Schema.STRING).topic(topics[0]).create().send("done");
+        final Message msg = partialTopicsConsumer.receive();
+        Assert.assertEquals(msg.getValue(), "msg-0");
+        partialTopicsConsumer.close();
+    }
 }
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java
index 341a91e97348e..f993304b0780a 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java
@@ -32,6 +32,7 @@
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.IdentityHashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -494,8 +495,16 @@ protected CompletableFuture doAcknowledge(List messageIdList,
                 topicToMessageIdMap.get(topicMessageId.getOwnerTopic())
                         .add(MessageIdImpl.convertToMessageIdImpl(topicMessageId));
             }
-            topicToMessageIdMap.forEach((topicPartitionName, messageIds) -> {
-                ConsumerImpl consumer = consumers.get(topicPartitionName);
+            final Map, List> consumerToMessageIds = new IdentityHashMap<>();
+            for (Map.Entry> entry : topicToMessageIdMap.entrySet()) {
+                ConsumerImpl consumer = consumers.get(entry.getKey());
+                if (consumer == null) {
+                    return FutureUtil.failedFuture(new PulsarClientException.NotConnectedException());
+                }
+                // Trigger the acknowledgment later to avoid sending partial acknowledgments
+                consumerToMessageIds.put(consumer, entry.getValue());
+            }
+            consumerToMessageIds.forEach((consumer, messageIds) -> {
                 resultFutures.add(consumer.doAcknowledgeWithTxn(messageIds, ackType, properties, txn)
                         .thenAccept((res) -> messageIdList.forEach(unAckedMessageTracker::remove)));
             });

From 19c84977f00033ef9601f43f1b703068a6207536 Mon Sep 17 00:00:00 2001
From: LinChen 
Date: Mon, 27 Mar 2023 15:18:01 +0800
Subject: [PATCH 150/174] [fix][broker] Fix the thread safety issue of
 BrokerData#getTimeAverageData access (#19889)

Co-authored-by: lordcheng10 
---
 .../broker/loadbalance/impl/ModularLoadManagerImpl.java       | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java
index f135840d60e59..4756b885ff217 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java
@@ -589,7 +589,9 @@ private void updateBundleData() {
             }
 
             // Using the newest data, update the aggregated time-average data for the current broker.
-            brokerData.getTimeAverageData().reset(statsMap.keySet(), bundleData, defaultStats);
+            TimeAverageBrokerData timeAverageData = new TimeAverageBrokerData();
+            timeAverageData.reset(statsMap.keySet(), bundleData, defaultStats);
+            brokerData.setTimeAverageData(timeAverageData);
             final ConcurrentOpenHashMap> namespaceToBundleRange =
                     brokerToNamespaceToBundleRange
                             .computeIfAbsent(broker, k ->

From cda2827acbe62ba6c5a4dd98dbee7c9a82548339 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Mon, 27 Mar 2023 12:32:54 +0100
Subject: [PATCH 151/174] [fix][fn] Revert change to deprecation since it broke
 the master branch (#19904)

---
 pulsar-function-go/pb/Function.pb.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/pulsar-function-go/pb/Function.pb.go b/pulsar-function-go/pb/Function.pb.go
index bad1afe6658da..3e4d49ab99022 100644
--- a/pulsar-function-go/pb/Function.pb.go
+++ b/pulsar-function-go/pb/Function.pb.go
@@ -579,7 +579,6 @@ type FunctionDetails struct {
 	Runtime              FunctionDetails_Runtime `protobuf:"varint,8,opt,name=runtime,proto3,enum=proto.FunctionDetails_Runtime" json:"runtime,omitempty"`
 	// Deprecated since, see https://github.com/apache/pulsar/issues/15560
 	//
-	// Deprecated: Do not use.
 	AutoAck              bool                          `protobuf:"varint,9,opt,name=autoAck,proto3" json:"autoAck,omitempty"`
 	Parallelism          int32                         `protobuf:"varint,10,opt,name=parallelism,proto3" json:"parallelism,omitempty"`
 	Source               *SourceSpec                   `protobuf:"bytes,11,opt,name=source,proto3" json:"source,omitempty"`

From 32ad90606062c1eda660037e6277253b35d4a1e6 Mon Sep 17 00:00:00 2001
From: Zixuan Liu 
Date: Mon, 27 Mar 2023 22:16:25 +0800
Subject: [PATCH 152/174] [fix][admin] Delete tenant local policy only if exist
 (#19925)

Signed-off-by: nodece 
---
 .../resources/LocalPoliciesResources.java     |  2 +-
 .../broker/admin/AdminApiTenantTest.java      | 76 +++++++++++++++++++
 2 files changed, 77 insertions(+), 1 deletion(-)
 create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTenantTest.java

diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/LocalPoliciesResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/LocalPoliciesResources.java
index 8ca0c121ef1b9..c6b658c3bd025 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/LocalPoliciesResources.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/LocalPoliciesResources.java
@@ -85,7 +85,7 @@ public CompletableFuture deleteLocalPoliciesAsync(NamespaceName ns) {
     public CompletableFuture deleteLocalPoliciesTenantAsync(String tenant) {
         final String localPoliciesPath = joinPath(LOCAL_POLICIES_ROOT, tenant);
         CompletableFuture future = new CompletableFuture();
-        deleteAsync(localPoliciesPath).whenComplete((ignore, ex) -> {
+        deleteIfExistsAsync(localPoliciesPath).whenComplete((ignore, ex) -> {
             if (ex != null && ex.getCause().getCause() instanceof KeeperException) {
                 future.complete(null);
             } else if (ex != null) {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTenantTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTenantTest.java
new file mode 100644
index 0000000000000..f883417614229
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTenantTest.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.admin;
+
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertThrows;
+import static org.testng.Assert.assertTrue;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
+import org.apache.pulsar.client.admin.PulsarAdminException;
+import org.apache.pulsar.common.policies.data.ClusterData;
+import org.apache.pulsar.common.policies.data.TenantInfo;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+@Test(groups = "broker-admin")
+@Slf4j
+public class AdminApiTenantTest extends MockedPulsarServiceBaseTest {
+    private final String CLUSTER = "test";
+
+    @BeforeClass
+    @Override
+    public void setup() throws Exception {
+        super.internalSetup();
+        admin.clusters()
+                .createCluster(CLUSTER, ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build());
+    }
+
+    @BeforeClass(alwaysRun = true)
+    @Override
+    public void cleanup() throws Exception {
+        super.internalCleanup();
+    }
+
+    @Test
+    public void testListTenant() throws PulsarAdminException {
+        admin.tenants().getTenants();
+    }
+
+    @Test
+    public void testCreateAndDeleteTenant() throws PulsarAdminException {
+        String tenant = "test-tenant-"+ UUID.randomUUID();
+        admin.tenants().createTenant(tenant, TenantInfo.builder().allowedClusters(Collections.singleton(CLUSTER)).build());
+        List tenants = admin.tenants().getTenants();
+        assertTrue(tenants.contains(tenant));
+        admin.tenants().deleteTenant(tenant);
+        tenants = admin.tenants().getTenants();
+        assertFalse(tenants.contains(tenant));
+    }
+
+    @Test
+    public void testDeleteNonExistTenant() {
+        String tenant = "test-non-exist-tenant-" + UUID.randomUUID();
+        assertThrows(PulsarAdminException.NotFoundException.class, () -> admin.tenants().deleteTenant(tenant));
+    }
+}

From d14e43ed9bd079b76301f10dc1c039cafd43e1cf Mon Sep 17 00:00:00 2001
From: tison 
Date: Tue, 28 Mar 2023 11:22:22 +0800
Subject: [PATCH 153/174] [improve][client] Exclude log4j-slf4j-impl from
 compile dep in pulsar-client-all (#19937)

Signed-off-by: tison 
---
 pulsar-client-all/pom.xml | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml
index b06c71616b6a5..00da6e4895097 100644
--- a/pulsar-client-all/pom.xml
+++ b/pulsar-client-all/pom.xml
@@ -54,11 +54,25 @@
       ${project.parent.version}
       true
     
+    
+      org.apache.logging.log4j
+      log4j-api
+      test
+    
+    
+      org.apache.logging.log4j
+      log4j-core
+      test
+    
+    
+      org.apache.logging.log4j
+      log4j-slf4j-impl
+      test
+    
   
 
   
     
-
       
         org.apache.maven.plugins
         maven-dependency-plugin

From 7a99e74220bdff7a0dff3dbbf3befb2316f3a6c8 Mon Sep 17 00:00:00 2001
From: SeasonPan <244014926@qq.com>
Date: Tue, 28 Mar 2023 17:58:58 +0800
Subject: [PATCH 154/174] [fix][doc] fix typo in ConsumerConfigurationData and
 ServiceConfiguration. (#19936)

---
 .../java/org/apache/pulsar/broker/ServiceConfiguration.java   | 2 +-
 .../pulsar/client/impl/conf/ConsumerConfigurationData.java    | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index ba82667690dc5..e2bcf03353b59 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -1976,7 +1976,7 @@ The delayed message index time step(in seconds) in per bucket snapshot segment,
             category = CATEGORY_STORAGE_ML,
             dynamic = true,
             doc = "The number of partitioned topics that is allowed to be automatically created"
-                    + "if allowAutoTopicCreationType is partitioned."
+                    + " if allowAutoTopicCreationType is partitioned."
     )
     private int defaultNumPartitions = 1;
     @FieldContext(
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ConsumerConfigurationData.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ConsumerConfigurationData.java
index 373d4e66c0ecf..8760926792cd7 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ConsumerConfigurationData.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ConsumerConfigurationData.java
@@ -99,7 +99,7 @@ public class ConsumerConfigurationData implements Serializable, Cloneable {
     @ApiModelProperty(
             name = "negativeAckRedeliveryBackoff",
             value = "Interface for custom message is negativeAcked policy. You can specify `RedeliveryBackoff` for a"
-                    + "consumer."
+                    + " consumer."
     )
     @JsonIgnore
     private RedeliveryBackoff negativeAckRedeliveryBackoff;
@@ -235,7 +235,7 @@ public int getMaxPendingChuckedMessage() {
 
     @ApiModelProperty(
             name = "autoAckOldestChunkedMessageOnQueueFull",
-            value = "Whether to automatically acknowledge pending chunked messages when the threashold of"
+            value = "Whether to automatically acknowledge pending chunked messages when the threshold of"
                     + " `maxPendingChunkedMessage` is reached. If set to `false`, these messages will be redelivered"
                     + " by their broker."
     )

From 83f6ea74e7a47927402d8f1de4200c4ad02e6147 Mon Sep 17 00:00:00 2001
From: Lari Hotari 
Date: Tue, 28 Mar 2023 19:07:43 +0100
Subject: [PATCH 155/174] [fix][build] Dump Jacoco coverage data to file with
 JMX interface in TestNG listener (#19947)

---
 buildtools/pom.xml                            |   2 +-
 .../pulsar/tests/JacocoDumpListener.java      | 103 ++++++++++++++++++
 pom.xml                                       |   3 +-
 3 files changed, 106 insertions(+), 2 deletions(-)
 create mode 100644 buildtools/src/main/java/org/apache/pulsar/tests/JacocoDumpListener.java

diff --git a/buildtools/pom.xml b/buildtools/pom.xml
index de52ac0930a33..b22abf286f693 100644
--- a/buildtools/pom.xml
+++ b/buildtools/pom.xml
@@ -176,7 +176,7 @@
           
             
               listener
-              org.apache.pulsar.tests.PulsarTestListener,org.apache.pulsar.tests.AnnotationListener,org.apache.pulsar.tests.FailFastNotifier
+              org.apache.pulsar.tests.PulsarTestListener,org.apache.pulsar.tests.JacocoDumpListener,org.apache.pulsar.tests.AnnotationListener,org.apache.pulsar.tests.FailFastNotifier
             
           
           
diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/JacocoDumpListener.java b/buildtools/src/main/java/org/apache/pulsar/tests/JacocoDumpListener.java
new file mode 100644
index 0000000000000..2c49d5118ae52
--- /dev/null
+++ b/buildtools/src/main/java/org/apache/pulsar/tests/JacocoDumpListener.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.tests;
+
+import java.lang.management.ManagementFactory;
+import java.util.concurrent.TimeUnit;
+import javax.management.InstanceNotFoundException;
+import javax.management.MBeanServer;
+import javax.management.MBeanServerInvocationHandler;
+import javax.management.MalformedObjectNameException;
+import javax.management.ObjectName;
+import org.testng.IExecutionListener;
+import org.testng.ISuite;
+import org.testng.ISuiteListener;
+
+/**
+ * A TestNG listener that dumps Jacoco coverage data to file using the Jacoco JMX interface.
+ *
+ * This ensures that coverage data is dumped even if the shutdown sequence of the Test JVM gets stuck. Coverage
+ * data will be dumped every 2 minutes by default and once all test suites have been run.
+ * Each test class runs in its own suite when run with maven-surefire-plugin.
+ */
+public class JacocoDumpListener implements ISuiteListener, IExecutionListener {
+    private final MBeanServer platformMBeanServer = ManagementFactory.getPlatformMBeanServer();
+    private final ObjectName jacocoObjectName;
+    private final JacocoProxy jacocoProxy;
+    private final boolean enabled;
+
+    private long lastDumpTime;
+
+    private static final long DUMP_INTERVAL_MILLIS = TimeUnit.SECONDS.toMillis(120);
+
+    public JacocoDumpListener() {
+        try {
+            jacocoObjectName = new ObjectName("org.jacoco:type=Runtime");
+        } catch (MalformedObjectNameException e) {
+            // this won't happen since the ObjectName is static and valid
+            throw new RuntimeException(e);
+        }
+        enabled = checkEnabled();
+        if (enabled) {
+            jacocoProxy = MBeanServerInvocationHandler.newProxyInstance(platformMBeanServer, jacocoObjectName,
+                    JacocoProxy.class, false);
+        } else {
+            jacocoProxy = null;
+        }
+        lastDumpTime = System.currentTimeMillis();
+    }
+
+    private boolean checkEnabled() {
+        try {
+            platformMBeanServer.getObjectInstance(jacocoObjectName);
+        } catch (InstanceNotFoundException e) {
+            // jacoco jmx is not enabled
+            return false;
+        }
+        return true;
+    }
+
+    public void onFinish(ISuite suite) {
+        // dump jacoco coverage data to file using the Jacoco JMX interface if more than DUMP_INTERVAL_MILLIS has passed
+        // since the last dump
+        if (enabled && System.currentTimeMillis() - lastDumpTime > DUMP_INTERVAL_MILLIS) {
+            // dump jacoco coverage data to file using the Jacoco JMX interface
+            triggerJacocoDump();
+        }
+    }
+    @Override
+    public void onExecutionFinish() {
+        if (enabled) {
+            // dump jacoco coverage data to file using the Jacoco JMX interface when all tests have finished
+            triggerJacocoDump();
+        }
+    }
+
+    private void triggerJacocoDump() {
+        System.out.println("Dumping Jacoco coverage data to file...");
+        long start = System.currentTimeMillis();
+        jacocoProxy.dump(true);
+        lastDumpTime = System.currentTimeMillis();
+        System.out.println("Completed in " + (lastDumpTime - start) + "ms.");
+    }
+
+    public interface JacocoProxy {
+        void dump(boolean reset);
+    }
+}
diff --git a/pom.xml b/pom.xml
index 3f33069b8c252..ac640f1a6a05f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1531,7 +1531,7 @@ flexible messaging model and an intuitive client API.
           
             
               listener
-              org.apache.pulsar.tests.PulsarTestListener,org.apache.pulsar.tests.AnnotationListener,org.apache.pulsar.tests.FailFastNotifier,org.apache.pulsar.tests.MockitoCleanupListener,org.apache.pulsar.tests.FastThreadLocalCleanupListener,org.apache.pulsar.tests.ThreadLeakDetectorListener,org.apache.pulsar.tests.SingletonCleanerListener
+              org.apache.pulsar.tests.PulsarTestListener,org.apache.pulsar.tests.JacocoDumpListener,org.apache.pulsar.tests.AnnotationListener,org.apache.pulsar.tests.FailFastNotifier,org.apache.pulsar.tests.MockitoCleanupListener,org.apache.pulsar.tests.FastThreadLocalCleanupListener,org.apache.pulsar.tests.ThreadLeakDetectorListener,org.apache.pulsar.tests.SingletonCleanerListener
             
           
         
@@ -1995,6 +1995,7 @@ flexible messaging model and an intuitive client API.
                   
                   ${project.build.directory}/jacoco_${maven.build.timestamp}_${surefire.forkNumber}.exec
                   true
+                  true
                   
                     org.apache.pulsar.*
                     org.apache.bookkeeper.mledger.*

From 9fc0b5e197aa36ff64087cd13d156bb65a097fad Mon Sep 17 00:00:00 2001
From: fengyubiao 
Date: Wed, 29 Mar 2023 09:24:12 +0800
Subject: [PATCH 156/174] [fix] [admin] fix incorrect state
 replication.connected on API partitioned-topic stat (#19942)

### Motivation

Pulsar will merge the variable `PartitionedTopicStatsImpl.replication[x].connected` by the way below when we call `pulsar-admin topics partitioned-stats`

``` java
this.connected = this.connected & other.connected
```

But the variable `connected` of `PartitionedTopicStatsImpl.replication` is initialized `false`, so the expression `this.connected & other.connected` will always be `false`.

Then we will always get the value `false` if we call `pulsar-admin topics partitioned-stats`.

### Modifications

make the variable `` of `PartitionedTopicStatsImpl` is initialized `true`
---
 .../pulsar/broker/service/ReplicatorTest.java | 25 +++++++++++++++++++
 .../data/stats/ReplicatorStatsImpl.java       |  4 ++-
 .../policies/data/stats/TopicStatsImpl.java   |  2 ++
 3 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java
index 4086c54a0ba4b..901451c022bfe 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java
@@ -97,8 +97,10 @@
 import org.apache.pulsar.common.policies.data.BacklogQuota;
 import org.apache.pulsar.common.policies.data.BacklogQuota.RetentionPolicy;
 import org.apache.pulsar.common.policies.data.ClusterData;
+import org.apache.pulsar.common.policies.data.PartitionedTopicStats;
 import org.apache.pulsar.common.policies.data.ReplicatorStats;
 import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy;
+import org.apache.pulsar.common.policies.data.TopicStats;
 import org.apache.pulsar.common.protocol.Commands;
 import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap;
 import org.apache.pulsar.schema.Schemas;
@@ -870,6 +872,29 @@ public void testReplicatorProducerClosing() throws Exception {
         assertNull(producer);
     }
 
+    @Test(priority = 5, timeOut = 30000)
+    public void testReplicatorConnected() throws Exception {
+        final String topicName = BrokerTestUtil.newUniqueName("persistent://pulsar/ns/tp_" + UUID.randomUUID());
+        final TopicName dest = TopicName.get(topicName);
+        admin1.topics().createPartitionedTopic(topicName, 1);
+
+        @Cleanup
+        MessageProducer producer1 = new MessageProducer(url1, dest);
+
+        Awaitility.await().until(() -> {
+            TopicStats topicStats = admin1.topics().getStats(topicName + "-partition-0");
+            return topicStats.getReplication().values().stream()
+                    .map(ReplicatorStats::isConnected).reduce((a, b) -> a & b).get();
+        });
+
+        PartitionedTopicStats
+                partitionedTopicStats = admin1.topics().getPartitionedStats(topicName, true);
+
+        for (ReplicatorStats replicatorStats : partitionedTopicStats.getReplication().values()){
+            assertTrue(replicatorStats.isConnected());
+        }
+    }
+
     @Test
     public void testDeleteTopicFailure() throws Exception {
         final String topicName = BrokerTestUtil.newUniqueName("persistent://pulsar/ns/tp_" + UUID.randomUUID());
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ReplicatorStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ReplicatorStatsImpl.java
index 71196e4978682..6933f5cc7ed76 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ReplicatorStatsImpl.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ReplicatorStatsImpl.java
@@ -72,7 +72,9 @@ public ReplicatorStatsImpl add(ReplicatorStatsImpl stats) {
         this.msgThroughputOut += stats.msgThroughputOut;
         this.msgRateExpired += stats.msgRateExpired;
         this.replicationBacklog += stats.replicationBacklog;
-        this.connected &= stats.connected;
+        if (this.connected) {
+            this.connected &= stats.connected;
+        }
         this.replicationDelayInSeconds = Math.max(this.replicationDelayInSeconds, stats.replicationDelayInSeconds);
         return this;
     }
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java
index 238170952f08e..12d30124f7dcd 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java
@@ -287,6 +287,7 @@ public TopicStatsImpl add(TopicStats ts) {
         if (this.replication.size() != stats.replication.size()) {
             for (String repl : stats.replication.keySet()) {
                 ReplicatorStatsImpl replStats = new ReplicatorStatsImpl();
+                replStats.setConnected(true);
                 this.replication.put(repl, replStats.add(stats.replication.get(repl)));
             }
         } else {
@@ -295,6 +296,7 @@ public TopicStatsImpl add(TopicStats ts) {
                     this.replication.get(repl).add(stats.replication.get(repl));
                 } else {
                     ReplicatorStatsImpl replStats = new ReplicatorStatsImpl();
+                    replStats.setConnected(true);
                     this.replication.put(repl, replStats.add(stats.replication.get(repl)));
                 }
             }

From eedf7026ae60f39bcf74ce67728b47d966fe237f Mon Sep 17 00:00:00 2001
From: Kai Wang 
Date: Wed, 29 Mar 2023 10:23:10 +0800
Subject: [PATCH 157/174] [improve][broker] PIP-192: Support delete namespace
 bundle admin API (#19851)

PIP: https://github.com/apache/pulsar/issues/16691

### Motivation
Raising a PR to implement https://github.com/apache/pulsar/issues/16691.

We need to support delete namespace bundle admin API.

### Modifications

* Support delete namespace bundle admin API.
* Add units test.
---
 .../broker/admin/impl/NamespacesBase.java     | 12 ++---
 .../extensions/ExtensibleLoadManagerImpl.java | 11 +++-
 .../extensions/data/BrokerLookupData.java     |  6 +++
 .../broker/namespace/NamespaceService.java    | 47 ++++++++++++++---
 .../ExtensibleLoadManagerImplTest.java        | 50 +++++++++++++++++++
 5 files changed, 111 insertions(+), 15 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
index cffc94b189225..19f8d7c437ded 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
@@ -297,11 +297,11 @@ private void internalRetryableDeleteNamespaceAsync0(boolean force, int retryTime
                 .thenCompose(ignore -> pulsar().getNamespaceService()
                         .getNamespaceBundleFactory().getBundlesAsync(namespaceName))
                 .thenCompose(bundles -> FutureUtil.waitForAll(bundles.getBundles().stream()
-                        .map(bundle -> pulsar().getNamespaceService().getOwnerAsync(bundle)
-                                .thenCompose(owner -> {
+                        .map(bundle -> pulsar().getNamespaceService().checkOwnershipPresentAsync(bundle)
+                                .thenCompose(present -> {
                                     // check if the bundle is owned by any broker,
                                     // if not then we do not need to delete the bundle
-                                    if (owner.isPresent()) {
+                                    if (present) {
                                         PulsarAdmin admin;
                                         try {
                                             admin = pulsar().getAdminClient();
@@ -1411,7 +1411,7 @@ protected void internalClearNamespaceBacklog(AsyncResponse asyncResponse, boolea
                     .getBundles(namespaceName);
             for (NamespaceBundle nsBundle : bundles.getBundles()) {
                 // check if the bundle is owned by any broker, if not then there is no backlog on this bundle to clear
-                if (pulsar().getNamespaceService().getOwner(nsBundle).isPresent()) {
+                if (pulsar().getNamespaceService().checkOwnershipPresent(nsBundle)) {
                     futures.add(pulsar().getAdminClient().namespaces()
                             .clearNamespaceBundleBacklogAsync(namespaceName.toString(), nsBundle.getBundleRange()));
                 }
@@ -1476,7 +1476,7 @@ protected void internalClearNamespaceBacklogForSubscription(AsyncResponse asyncR
                     .getBundles(namespaceName);
             for (NamespaceBundle nsBundle : bundles.getBundles()) {
                 // check if the bundle is owned by any broker, if not then there is no backlog on this bundle to clear
-                if (pulsar().getNamespaceService().getOwner(nsBundle).isPresent()) {
+                if (pulsar().getNamespaceService().checkOwnershipPresent(nsBundle)) {
                     futures.add(pulsar().getAdminClient().namespaces().clearNamespaceBundleBacklogForSubscriptionAsync(
                             namespaceName.toString(), nsBundle.getBundleRange(), subscription));
                 }
@@ -1543,7 +1543,7 @@ protected void internalUnsubscribeNamespace(AsyncResponse asyncResponse, String
                     .getBundles(namespaceName);
             for (NamespaceBundle nsBundle : bundles.getBundles()) {
                 // check if the bundle is owned by any broker, if not then there are no subscriptions
-                if (pulsar().getNamespaceService().getOwner(nsBundle).isPresent()) {
+                if (pulsar().getNamespaceService().checkOwnershipPresent(nsBundle)) {
                     futures.add(pulsar().getAdminClient().namespaces().unsubscribeNamespaceBundleAsync(
                             namespaceName.toString(), nsBundle.getBundleRange(), subscription));
                 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
index fedcad4d009b2..7aefef596e7cb 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
@@ -383,7 +383,7 @@ public CompletableFuture checkOwnershipAsync(Optional to
                 .thenApply(broker -> brokerRegistry.getBrokerId().equals(broker.orElse(null)));
     }
 
-    private CompletableFuture> getOwnershipAsync(Optional topic,
+    public CompletableFuture> getOwnershipAsync(Optional topic,
                                                                  ServiceUnitId bundleUnit) {
         final String bundle = bundleUnit.toString();
         CompletableFuture> owner;
@@ -395,6 +395,15 @@ private CompletableFuture> getOwnershipAsync(Optional> getOwnershipWithLookupDataAsync(ServiceUnitId bundleUnit) {
+        return getOwnershipAsync(Optional.empty(), bundleUnit).thenCompose(broker -> {
+            if (broker.isEmpty()) {
+                return CompletableFuture.completedFuture(Optional.empty());
+            }
+            return getBrokerRegistry().lookupAsync(broker.get());
+        });
+    }
+
     public CompletableFuture unloadNamespaceBundleAsync(ServiceUnitId bundle,
                                                               Optional destinationBroker) {
         return getOwnershipAsync(Optional.empty(), bundle)
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupData.java
index 504ae13003e04..4c9e503129e66 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupData.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLookupData.java
@@ -21,6 +21,7 @@
 import java.util.Map;
 import java.util.Optional;
 import org.apache.pulsar.broker.lookup.LookupResult;
+import org.apache.pulsar.broker.namespace.NamespaceEphemeralData;
 import org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener;
 import org.apache.pulsar.policies.data.loadbalancer.ServiceLookupData;
 
@@ -70,4 +71,9 @@ public LookupResult toLookupResult() {
         return new LookupResult(webServiceUrl, webServiceUrlTls, pulsarServiceUrl, pulsarServiceUrlTls,
                 LookupResult.Type.BrokerUrl, false);
     }
+
+    public NamespaceEphemeralData toNamespaceEphemeralData() {
+        return new NamespaceEphemeralData(pulsarServiceUrl, pulsarServiceUrlTls, webServiceUrl, webServiceUrlTls,
+                false, advertisedListeners);
+    }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
index de18d50f3e144..d092ef04018c8 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
@@ -1165,8 +1165,14 @@ public CompletableFuture checkTopicOwnership(TopicName topicName) {
     }
 
     public CompletableFuture removeOwnedServiceUnitAsync(NamespaceBundle nsBundle) {
-        return ownershipCache.removeOwnership(nsBundle)
-                .thenRun(() -> bundleFactory.invalidateBundleCache(nsBundle.getNamespaceObject()));
+        CompletableFuture future;
+        if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) {
+            ExtensibleLoadManagerImpl extensibleLoadManager = ExtensibleLoadManagerImpl.get(loadManager.get());
+            future = extensibleLoadManager.unloadNamespaceBundleAsync(nsBundle, Optional.empty());
+        } else {
+            future = ownershipCache.removeOwnership(nsBundle);
+        }
+        return future.thenRun(() -> bundleFactory.invalidateBundleCache(nsBundle.getNamespaceObject()));
     }
 
     protected void onNamespaceBundleOwned(NamespaceBundle bundle) {
@@ -1445,15 +1451,40 @@ public PulsarClientImpl getNamespaceClient(ClusterDataImpl cluster) {
         });
     }
 
-    public Optional getOwner(NamespaceBundle bundle) throws Exception {
-        // if there is no znode for the service unit, it is not owned by any broker
-        return getOwnerAsync(bundle).get(pulsar.getConfiguration().getMetadataStoreOperationTimeoutSeconds(), SECONDS);
-    }
-
     public CompletableFuture> getOwnerAsync(NamespaceBundle bundle) {
+        if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) {
+            ExtensibleLoadManagerImpl extensibleLoadManager = ExtensibleLoadManagerImpl.get(loadManager.get());
+            return extensibleLoadManager.getOwnershipWithLookupDataAsync(bundle)
+                    .thenCompose(lookupData -> {
+                        if (lookupData.isPresent()) {
+                            return CompletableFuture.completedFuture(
+                                    Optional.of(lookupData.get().toNamespaceEphemeralData()));
+                        } else {
+                            return CompletableFuture.completedFuture(Optional.empty());
+                        }
+                    });
+        }
         return ownershipCache.getOwnerAsync(bundle);
     }
 
+    public boolean checkOwnershipPresent(NamespaceBundle bundle) throws Exception {
+        return checkOwnershipPresentAsync(bundle).get(pulsar.getConfiguration()
+                        .getMetadataStoreOperationTimeoutSeconds(), SECONDS);
+    }
+
+    public CompletableFuture checkOwnershipPresentAsync(NamespaceBundle bundle) {
+        if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) {
+            if (bundle.getNamespaceObject().equals(SYSTEM_NAMESPACE)) {
+                return FutureUtil.failedFuture(new UnsupportedOperationException(
+                        "Ownership check for system namespace is not supported"));
+            }
+            ExtensibleLoadManagerImpl extensibleLoadManager = ExtensibleLoadManagerImpl.get(loadManager.get());
+            return extensibleLoadManager.getOwnershipAsync(Optional.empty(), bundle)
+                    .thenApply(Optional::isPresent);
+        }
+        return getOwnerAsync(bundle).thenApply(Optional::isPresent);
+    }
+
     public void unloadSLANamespace() throws Exception {
         PulsarAdmin adminClient = null;
         NamespaceName namespaceName = getSLAMonitorNamespace(host, config);
@@ -1461,7 +1492,7 @@ public void unloadSLANamespace() throws Exception {
         LOG.info("Checking owner for SLA namespace {}", namespaceName);
 
         NamespaceBundle nsFullBundle = getFullBundle(namespaceName);
-        if (!getOwner(nsFullBundle).isPresent()) {
+        if (!checkOwnershipPresent(nsFullBundle)) {
             // No one owns the namespace so no point trying to unload it
             // Next lookup will assign the bundle to this broker.
             return;
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
index d977e63330586..f8a7a9b629f4e 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
@@ -134,6 +134,7 @@ public class ExtensibleLoadManagerImplTest extends MockedPulsarServiceBaseTest {
     @BeforeClass
     @Override
     public void setup() throws Exception {
+        conf.setForceDeleteNamespaceAllowed(true);
         conf.setAllowAutoTopicCreation(true);
         conf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName());
         conf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName());
@@ -142,6 +143,7 @@ public void setup() throws Exception {
         pulsar1 = pulsar;
         ServiceConfiguration defaultConf = getDefaultConf();
         defaultConf.setAllowAutoTopicCreation(true);
+        defaultConf.setForceDeleteNamespaceAllowed(true);
         defaultConf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName());
         defaultConf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName());
         defaultConf.setLoadBalancerSheddingEnabled(false);
@@ -433,6 +435,54 @@ public void testSplitBundleWithSpecificPositionAdminAPI() throws Exception {
         assertTrue(bundlesData.getBoundaries().contains(midBundle));
         assertTrue(bundlesData.getBoundaries().contains(highBundle));
     }
+    @Test(timeOut = 30 * 1000)
+    public void testDeleteNamespaceBundle() throws Exception {
+        TopicName topicName = TopicName.get("test-delete-namespace-bundle");
+        NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get();
+
+        String broker = admin.lookups().lookupTopic(topicName.toString());
+        log.info("Assign the bundle {} to {}", bundle, broker);
+
+        checkOwnershipState(broker, bundle);
+
+        admin.namespaces().deleteNamespaceBundle(topicName.getNamespace(), bundle.getBundleRange());
+        assertFalse(primaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get());
+    }
+
+    @Test(timeOut = 30 * 1000)
+    public void testDeleteNamespace() throws Exception {
+        String namespace = "public/test-delete-namespace";
+        TopicName topicName = TopicName.get(namespace + "/test-delete-namespace-topic");
+        admin.namespaces().createNamespace(namespace);
+        admin.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet(this.conf.getClusterName()));
+        assertTrue(admin.namespaces().getNamespaces("public").contains(namespace));
+        admin.topics().createPartitionedTopic(topicName.toString(), 2);
+        admin.lookups().lookupTopic(topicName.toString());
+        NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get();
+        try {
+            admin.namespaces().deleteNamespaceBundle(namespace, bundle.getBundleRange());
+            fail();
+        } catch (Exception ex) {
+            assertTrue(ex.getMessage().contains("Cannot delete non empty bundle"));
+        }
+        admin.namespaces().deleteNamespaceBundle(namespace, bundle.getBundleRange(), true);
+        admin.lookups().lookupTopic(topicName.toString());
+
+        admin.namespaces().deleteNamespace(namespace, true);
+        assertFalse(admin.namespaces().getNamespaces("public").contains(namespace));
+    }
+
+    @Test(timeOut = 30 * 1000)
+    public void testCheckOwnershipPresentWithSystemNamespace() throws Exception {
+        NamespaceBundle namespaceBundle =
+                getBundleAsync(pulsar1, TopicName.get(NamespaceName.SYSTEM_NAMESPACE + "/test")).get();
+        try {
+            pulsar1.getNamespaceService().checkOwnershipPresent(namespaceBundle);
+        } catch (Exception ex) {
+            log.info("Got exception", ex);
+            assertTrue(ex.getCause() instanceof UnsupportedOperationException);
+        }
+    }
 
     @Test
     public void testMoreThenOneFilter() throws Exception {

From d8774671f5d52be4ef940ea894ba4f9b1f5717f9 Mon Sep 17 00:00:00 2001
From: StevenLuMT 
Date: Wed, 29 Mar 2023 10:36:23 +0800
Subject: [PATCH 158/174] [fix][test] Shutdown executor on PerformanceProducer
 closed (#19926)

Co-authored-by: lushiji 
---
 .../pulsar/testclient/PerformanceProducer.java      | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java
index 9aedc8c4bd327..0c56f1d5c736d 100644
--- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java
+++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java
@@ -356,6 +356,7 @@ public static void main(String[] args) throws Exception {
         long start = System.nanoTime();
 
         Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+            executorShutdownNow();
             printAggregatedThroughput(start, arguments);
             printAggregatedStats();
         }));
@@ -481,6 +482,18 @@ public static void main(String[] args) throws Exception {
         }
     }
 
+    private static void executorShutdownNow() {
+        executor.shutdownNow();
+        try {
+            if (!executor.awaitTermination(10, TimeUnit.SECONDS)) {
+                log.warn("Failed to terminate executor within timeout. The following are stack"
+                        + " traces of still running threads.");
+            }
+        } catch (InterruptedException e) {
+            log.warn("Shutdown of thread pool was interrupted");
+        }
+    }
+
     static IMessageFormatter getMessageFormatter(String formatterClass) {
         try {
             ClassLoader classLoader = PerformanceProducer.class.getClassLoader();

From 5611faf61e70ce563bb43eb499436517b54d0bb5 Mon Sep 17 00:00:00 2001
From: crossoverJie 
Date: Wed, 29 Mar 2023 13:06:02 +0800
Subject: [PATCH 159/174] [fix][client] Fix client erroneous code comments
 (#19915)

Co-authored-by: tison 
---
 .../pulsar/client/api/ClientBuilder.java      | 29 ++++++++++---------
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ClientBuilder.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ClientBuilder.java
index 8f2813ed14fff..8b959690a0363 100644
--- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ClientBuilder.java
+++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ClientBuilder.java
@@ -227,23 +227,25 @@ ClientBuilder authentication(String authPluginClassName, Map aut
     /**
      * Set lookup timeout (default: matches operation timeout)
      *
-     * Lookup operations have a different load pattern to other operations. They can be handled by any broker, are not
-     * proportional to throughput, and are harmless to retry. Given this, it makes sense to allow them to retry longer
-     * than normal operation, especially if they experience a timeout.
+     * 

+     * Lookup operations have a different load pattern to other operations.
+     * They can be handled by any broker, are not proportional to throughput,
+     * and are harmless to retry. Given this, it makes sense to allow them to
+     * retry longer than normal operation, especially if they experience a timeout.
      *
-     * By default this is set to match operation timeout. This is to maintain legacy behaviour. However, in practice
-     * it should be set to 5-10x the operation timeout.
+     * 

+     * By default, this is set to match operation timeout. This is to maintain legacy behaviour.
+     * However, in practice it should be set to 5-10x the operation timeout.
      *
-     * @param lookupTimeout
-     *            lookup timeout
-     * @param unit
-     *            time unit for {@code lookupTimeout}
+     * @param lookupTimeout lookup timeout
+     * @param unit time unit for {@code lookupTimeout}
      * @return the client builder instance
      */
     ClientBuilder lookupTimeout(int lookupTimeout, TimeUnit unit);
 
     /**
-     * Set the number of threads to be used for handling connections to brokers (default: 1 thread).
+     * Set the number of threads to be used for handling connections to brokers
+     * (default: Runtime.getRuntime().availableProcessors()).
      *
      * @param numIoThreads the number of IO threads
      * @return the client builder instance
@@ -251,11 +253,12 @@ ClientBuilder authentication(String authPluginClassName, Map aut
     ClientBuilder ioThreads(int numIoThreads);
 
     /**
-     * Set the number of threads to be used for message listeners (default: 1 thread).
+     * Set the number of threads to be used for message listeners
+     * (default: Runtime.getRuntime().availableProcessors()).
      *
      * 
The listener thread pool is shared across all the consumers and readers that are
-     * using a "listener" model to get messages. For a given consumer, the listener will be
-     * always invoked from the same thread, to ensure ordering.
+     * using a "listener" model to get messages. For a given consumer, the listener will
+     * always be invoked from the same thread, to ensure ordering.
      *
      * @param numListenerThreads the number of listener threads
      * @return the client builder instance

From bbf52736f7542423b9699387eb9521c35e187816 Mon Sep 17 00:00:00 2001
From: Cong Zhao 
Date: Wed, 29 Mar 2023 13:13:18 +0800
Subject: [PATCH 160/174] [improve][broker][PIP-195] Merge multiple buckets at
 once (#19927)

---
 .../mledger/impl/ManagedCursorImpl.java       |   5 +-
 .../bucket/BucketDelayedDeliveryTracker.java  | 199 ++++++++++--------
 .../CombinedSegmentDelayedIndexQueue.java     | 100 +++++----
 .../BucketDelayedDeliveryTrackerTest.java     |   8 +-
 .../delayed/bucket/DelayedIndexQueueTest.java |  21 +-
 5 files changed, 180 insertions(+), 153 deletions(-)

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
index 5851395b08566..05cad09b01833 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
@@ -363,8 +363,7 @@ private CompletableFuture computeCursorProperties(
                 name, copy, lastCursorLedgerStat, new MetaStoreCallback<>() {
                     @Override
                     public void operationComplete(Void result, Stat stat) {
-                        log.info("[{}] Updated ledger cursor: {} properties {}", ledger.getName(),
-                                name, cursorProperties);
+                        log.info("[{}] Updated ledger cursor: {}", ledger.getName(), name);
                         ManagedCursorImpl.this.cursorProperties = Collections.unmodifiableMap(newProperties);
                         updateCursorLedgerStat(copy, stat);
                         updateCursorPropertiesResult.complete(result);
@@ -373,7 +372,7 @@ public void operationComplete(Void result, Stat stat) {
                     @Override
                     public void operationFailed(MetaStoreException e) {
                         log.error("[{}] Error while updating ledger cursor: {} properties {}", ledger.getName(),
-                                name, cursorProperties, e);
+                                name, newProperties, e);
                         updateCursorPropertiesResult.completeExceptionally(e);
                     }
                 });
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
index 39973928abf5d..ef7be187cec3d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
@@ -32,6 +32,7 @@
 import io.netty.util.Timer;
 import java.time.Clock;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -43,6 +44,7 @@
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
 import javax.annotation.concurrent.ThreadSafe;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
@@ -93,6 +95,8 @@ public class BucketDelayedDeliveryTracker extends AbstractDelayedDeliveryTracker
 
     private static final Long INVALID_BUCKET_ID = -1L;
 
+    private static final int MAX_MERGE_NUM = 4;
+
     public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher,
                                  Timer timer, long tickTimeMillis,
                                  boolean isDelayedDeliveryDeliverAtTimeStrict,
@@ -341,115 +345,136 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver
         return true;
     }
 
-    private synchronized CompletableFuture asyncMergeBucketSnapshot() {
-        List values = immutableBuckets.asMapOfRanges().values().stream().toList();
+    private synchronized List selectMergedBuckets(final List values, int mergeNum) {
+        checkArgument(mergeNum < values.size());
         long minNumberMessages = Long.MAX_VALUE;
         long minScheduleTimestamp = Long.MAX_VALUE;
         int minIndex = -1;
-        for (int i = 0; i + 1 < values.size(); i++) {
-            ImmutableBucket bucketL = values.get(i);
-            ImmutableBucket bucketR = values.get(i + 1);
-            // We should skip the bucket which last segment already been load to memory, avoid record replicated index.
-            if (bucketL.lastSegmentEntryId > bucketL.getCurrentSegmentEntryId()
-                    && bucketR.lastSegmentEntryId > bucketR.getCurrentSegmentEntryId()
-                    // Skip the bucket that is merging
-                    && !bucketL.merging && !bucketR.merging){
-                long scheduleTimestamp =
-                        Math.min(bucketL.firstScheduleTimestamps.get(bucketL.currentSegmentEntryId + 1),
-                                bucketR.firstScheduleTimestamps.get(bucketR.currentSegmentEntryId + 1));
-                long numberMessages = bucketL.numberBucketDelayedMessages + bucketR.numberBucketDelayedMessages;
-                if (scheduleTimestamp <= minScheduleTimestamp) {
-                    minScheduleTimestamp = scheduleTimestamp;
-                    if (numberMessages < minNumberMessages) {
-                        minNumberMessages = numberMessages;
+        for (int i = 0; i + (mergeNum - 1) < values.size(); i++) {
+            List immutableBuckets = values.subList(i, i + mergeNum);
+            if (immutableBuckets.stream().allMatch(bucket -> {
+                // We should skip the bucket which last segment already been load to memory,
+                // avoid record replicated index.
+                return bucket.lastSegmentEntryId > bucket.currentSegmentEntryId && !bucket.merging;
+            })) {
+                long numberMessages = immutableBuckets.stream()
+                        .mapToLong(bucket -> bucket.numberBucketDelayedMessages)
+                        .sum();
+                if (numberMessages <= minNumberMessages) {
+                    minNumberMessages = numberMessages;
+                    long scheduleTimestamp = immutableBuckets.stream()
+                            .mapToLong(bucket -> bucket.firstScheduleTimestamps.get(bucket.currentSegmentEntryId + 1))
+                            .min().getAsLong();
+                    if (scheduleTimestamp < minScheduleTimestamp) {
+                        minScheduleTimestamp = scheduleTimestamp;
                         minIndex = i;
                     }
                 }
             }
         }
 
-        if (minIndex == -1) {
-            log.warn("[{}] Can't find able merged bucket", dispatcher.getName());
-            return CompletableFuture.completedFuture(null);
+        if (minIndex >= 0) {
+            return values.subList(minIndex, minIndex + MAX_MERGE_NUM);
+        } else if (mergeNum > 2){
+            return selectMergedBuckets(values, mergeNum - 1);
+        } else {
+            return Collections.emptyList();
         }
+    }
 
-        ImmutableBucket immutableBucketA = values.get(minIndex);
-        ImmutableBucket immutableBucketB = values.get(minIndex + 1);
+    private synchronized CompletableFuture asyncMergeBucketSnapshot() {
+        List immutableBucketList = immutableBuckets.asMapOfRanges().values().stream().toList();
+        List toBeMergeImmutableBuckets = selectMergedBuckets(immutableBucketList, MAX_MERGE_NUM);
 
+        if (toBeMergeImmutableBuckets.isEmpty()) {
+            log.warn("[{}] Can't find able merged buckets", dispatcher.getName());
+            return CompletableFuture.completedFuture(null);
+        }
+
+        final String bucketsStr = toBeMergeImmutableBuckets.stream().map(Bucket::bucketKey).collect(
+                Collectors.joining(",")).replaceAll(DELAYED_BUCKET_KEY_PREFIX + "_", "");
         if (log.isDebugEnabled()) {
-            log.info("[{}] Merging bucket snapshot, bucketAKey: {}, bucketBKey: {}", dispatcher.getName(),
-                    immutableBucketA.bucketKey(), immutableBucketB.bucketKey());
+            log.info("[{}] Merging bucket snapshot, bucketKeys: {}", dispatcher.getName(), bucketsStr);
         }
 
-        immutableBucketA.merging = true;
-        immutableBucketB.merging = true;
-        return asyncMergeBucketSnapshot(immutableBucketA, immutableBucketB).whenComplete((__, ex) -> {
+        for (ImmutableBucket immutableBucket : toBeMergeImmutableBuckets) {
+            immutableBucket.merging = true;
+        }
+        return asyncMergeBucketSnapshot(toBeMergeImmutableBuckets).whenComplete((__, ex) -> {
             synchronized (this) {
-                immutableBucketA.merging = false;
-                immutableBucketB.merging = false;
+                for (ImmutableBucket immutableBucket : toBeMergeImmutableBuckets) {
+                    immutableBucket.merging = false;
+                }
             }
             if (ex != null) {
-                log.error("[{}] Failed to merge bucket snapshot, bucketAKey: {}, bucketBKey: {}",
-                        dispatcher.getName(), immutableBucketA.bucketKey(), immutableBucketB.bucketKey(), ex);
+                log.error("[{}] Failed to merge bucket snapshot, bucketKeys: {}",
+                        dispatcher.getName(), bucketsStr, ex);
             } else {
-                log.info("[{}] Merge bucket snapshot finish, bucketAKey: {}, bucketBKey: {}",
-                        dispatcher.getName(), immutableBucketA.bucketKey(), immutableBucketB.bucketKey());
+                log.info("[{}] Merge bucket snapshot finish, bucketKeys: {}, bucketNum: {}",
+                        dispatcher.getName(), bucketsStr, immutableBuckets.asMapOfRanges().size());
             }
         });
     }
 
-    private synchronized CompletableFuture asyncMergeBucketSnapshot(ImmutableBucket bucketA,
-                                                                          ImmutableBucket bucketB) {
-        CompletableFuture createAFuture = bucketA.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE);
-        CompletableFuture createBFuture = bucketB.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE);
+    private synchronized CompletableFuture asyncMergeBucketSnapshot(List buckets) {
+        List> createFutures =
+                buckets.stream().map(bucket -> bucket.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE))
+                        .toList();
 
-        return CompletableFuture.allOf(createAFuture, createBFuture).thenCompose(bucketId -> {
-            if (INVALID_BUCKET_ID.equals(createAFuture.join()) || INVALID_BUCKET_ID.equals(createBFuture.join())) {
+        return FutureUtil.waitForAll(createFutures).thenCompose(bucketId -> {
+            if (createFutures.stream().anyMatch(future -> INVALID_BUCKET_ID.equals(future.join()))) {
                 return FutureUtil.failedFuture(new RuntimeException("Can't merge buckets due to bucket create failed"));
             }
 
-            CompletableFuture> futureA =
-                    bucketA.getRemainSnapshotSegment();
-            CompletableFuture> futureB =
-                    bucketB.getRemainSnapshotSegment();
-            return futureA.thenCombine(futureB, CombinedSegmentDelayedIndexQueue::wrap)
+            List>> getRemainFutures =
+                    buckets.stream().map(ImmutableBucket::getRemainSnapshotSegment).toList();
+
+            return FutureUtil.waitForAll(getRemainFutures)
+                    .thenApply(__ -> {
+                        return CombinedSegmentDelayedIndexQueue.wrap(
+                                getRemainFutures.stream().map(CompletableFuture::join).toList());
+                    })
                     .thenAccept(combinedDelayedIndexQueue -> {
                         synchronized (BucketDelayedDeliveryTracker.this) {
                             Pair immutableBucketDelayedIndexPair =
                                     lastMutableBucket.createImmutableBucketAndAsyncPersistent(
                                             timeStepPerBucketSnapshotSegmentInMillis,
                                             maxIndexesPerBucketSnapshotSegment,
-                                            sharedBucketPriorityQueue, combinedDelayedIndexQueue, bucketA.startLedgerId,
-                                            bucketB.endLedgerId);
+                                            sharedBucketPriorityQueue, combinedDelayedIndexQueue,
+                                            buckets.get(0).startLedgerId,
+                                            buckets.get(buckets.size() - 1).endLedgerId);
 
                             // Merge bit map to new bucket
-                            Map delayedIndexBitMapA = bucketA.getDelayedIndexBitMap();
-                            Map delayedIndexBitMapB = bucketB.getDelayedIndexBitMap();
-                            Map delayedIndexBitMap = new HashMap<>(delayedIndexBitMapA);
-                            delayedIndexBitMapB.forEach((ledgerId, bitMapB) -> {
-                                delayedIndexBitMap.compute(ledgerId, (k, bitMapA) -> {
-                                    if (bitMapA == null) {
-                                        return bitMapB;
-                                    }
-
-                                    bitMapA.or(bitMapB);
-                                    return bitMapA;
+                            Map delayedIndexBitMap =
+                                    new HashMap<>(buckets.get(0).getDelayedIndexBitMap());
+                            for (int i = 1; i < buckets.size(); i++) {
+                                buckets.get(i).delayedIndexBitMap.forEach((ledgerId, bitMapB) -> {
+                                    delayedIndexBitMap.compute(ledgerId, (k, bitMap) -> {
+                                        if (bitMap == null) {
+                                            return bitMapB;
+                                        }
+
+                                        bitMap.or(bitMapB);
+                                        return bitMap;
+                                    });
                                 });
-                            });
+                            }
                             immutableBucketDelayedIndexPair.getLeft().setDelayedIndexBitMap(delayedIndexBitMap);
 
                             afterCreateImmutableBucket(immutableBucketDelayedIndexPair);
 
                             immutableBucketDelayedIndexPair.getLeft().getSnapshotCreateFuture()
                                     .orElse(NULL_LONG_PROMISE).thenCompose(___ -> {
-                                        CompletableFuture removeAFuture = bucketA.asyncDeleteBucketSnapshot();
-                                        CompletableFuture removeBFuture = bucketB.asyncDeleteBucketSnapshot();
-                                        return CompletableFuture.allOf(removeAFuture, removeBFuture);
+                                        List> removeFutures =
+                                                buckets.stream().map(ImmutableBucket::asyncDeleteBucketSnapshot)
+                                                        .toList();
+                                        return FutureUtil.waitForAll(removeFutures);
                                     });
 
-                            Map, ImmutableBucket> immutableBucketMap = immutableBuckets.asMapOfRanges();
-                            immutableBucketMap.remove(Range.closed(bucketA.startLedgerId, bucketA.endLedgerId));
-                            immutableBucketMap.remove(Range.closed(bucketB.startLedgerId, bucketB.endLedgerId));
+                            for (ImmutableBucket bucket : buckets) {
+                                immutableBuckets.asMapOfRanges()
+                                        .remove(Range.closed(bucket.startLedgerId, bucket.endLedgerId));
+                            }
                         }
                     });
         });
@@ -593,14 +618,12 @@ public boolean shouldPauseAllDeliveries() {
 
     @Override
     public synchronized CompletableFuture clear() {
-        return cleanImmutableBuckets(true).thenRun(() -> {
-            synchronized (this) {
-                sharedBucketPriorityQueue.clear();
-                lastMutableBucket.clear();
-                snapshotSegmentLastIndexTable.clear();
-                numberDelayedMessages = 0;
-            }
-        });
+        CompletableFuture future = cleanImmutableBuckets();
+        sharedBucketPriorityQueue.clear();
+        lastMutableBucket.clear();
+        snapshotSegmentLastIndexTable.clear();
+        numberDelayedMessages = 0;
+        return future;
     }
 
     @Override
@@ -609,30 +632,24 @@ public synchronized void close() {
         lastMutableBucket.close();
         sharedBucketPriorityQueue.close();
         try {
-            cleanImmutableBuckets(false).get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS);
+            List> completableFutures = immutableBuckets.asMapOfRanges().values().stream()
+                    .map(bucket -> bucket.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE)).toList();
+            FutureUtil.waitForAll(completableFutures).get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS);
         } catch (Exception e) {
             log.warn("[{}] Failed wait to snapshot generate", dispatcher.getName(), e);
         }
     }
 
-    private CompletableFuture cleanImmutableBuckets(boolean delete) {
-        if (immutableBuckets != null) {
-            List> futures = new ArrayList<>();
-            Iterator iterator = immutableBuckets.asMapOfRanges().values().iterator();
-            while (iterator.hasNext()) {
-                ImmutableBucket bucket = iterator.next();
-                if (delete) {
-                    futures.add(bucket.clear());
-                } else {
-                    bucket.getSnapshotCreateFuture().ifPresent(future -> futures.add(future.thenApply(x -> null)));
-                }
-                numberDelayedMessages -= bucket.getNumberBucketDelayedMessages();
-                iterator.remove();
-            }
-            return FutureUtil.waitForAll(futures);
-        } else {
-            return CompletableFuture.completedFuture(null);
+    private CompletableFuture cleanImmutableBuckets() {
+        List> futures = new ArrayList<>();
+        Iterator iterator = immutableBuckets.asMapOfRanges().values().iterator();
+        while (iterator.hasNext()) {
+            ImmutableBucket bucket = iterator.next();
+            futures.add(bucket.clear());
+            numberDelayedMessages -= bucket.getNumberBucketDelayedMessages();
+            iterator.remove();
         }
+        return FutureUtil.waitForAll(futures);
     }
 
     private boolean removeIndexBit(long ledgerId, long entryId) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/CombinedSegmentDelayedIndexQueue.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/CombinedSegmentDelayedIndexQueue.java
index 3f89cc9fdfb15..5655a26878296 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/CombinedSegmentDelayedIndexQueue.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/CombinedSegmentDelayedIndexQueue.java
@@ -18,37 +18,48 @@
  */
 package org.apache.pulsar.broker.delayed.bucket;
 
+import java.util.Comparator;
 import java.util.List;
+import java.util.Objects;
+import java.util.PriorityQueue;
 import javax.annotation.concurrent.NotThreadSafe;
+import lombok.AllArgsConstructor;
 import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex;
 import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment;
 
 @NotThreadSafe
 class CombinedSegmentDelayedIndexQueue implements DelayedIndexQueue {
 
-    private final List segmentListA;
-    private final List segmentListB;
+    @AllArgsConstructor
+    static class Node {
+        List segmentList;
 
-    private int segmentListACursor = 0;
-    private int segmentListBCursor = 0;
-    private int segmentACursor = 0;
-    private int segmentBCursor = 0;
+        int segmentListCursor;
 
-    private CombinedSegmentDelayedIndexQueue(List segmentListA,
-                                             List segmentListB) {
-        this.segmentListA = segmentListA;
-        this.segmentListB = segmentListB;
+        int segmentCursor;
     }
 
-    public static CombinedSegmentDelayedIndexQueue wrap(
-            List segmentListA,
-            List segmentListB) {
-        return new CombinedSegmentDelayedIndexQueue(segmentListA, segmentListB);
+    private static final Comparator COMPARATOR_NODE = (node1, node2) -> DelayedIndexQueue.COMPARATOR.compare(
+            node1.segmentList.get(node1.segmentListCursor).getIndexes(node1.segmentCursor),
+            node2.segmentList.get(node2.segmentListCursor).getIndexes(node2.segmentCursor));
+
+    private final PriorityQueue kpq;
+
+    private CombinedSegmentDelayedIndexQueue(List> segmentLists) {
+        this.kpq = new PriorityQueue<>(segmentLists.size(), COMPARATOR_NODE);
+        for (List segmentList : segmentLists) {
+            Node node = new Node(segmentList, 0, 0);
+            kpq.offer(node);
+        }
+    }
+
+    public static CombinedSegmentDelayedIndexQueue wrap(List> segmentLists) {
+        return new CombinedSegmentDelayedIndexQueue(segmentLists);
     }
 
     @Override
     public boolean isEmpty() {
-        return segmentListACursor >= segmentListA.size() && segmentListBCursor >= segmentListB.size();
+        return kpq.isEmpty();
     }
 
     @Override
@@ -62,48 +73,35 @@ public DelayedIndex pop() {
     }
 
     private DelayedIndex getValue(boolean needAdvanceCursor) {
-        // skip empty segment
-        while (segmentListACursor < segmentListA.size()
-                && segmentListA.get(segmentListACursor).getIndexesCount() == 0) {
-            segmentListACursor++;
-        }
-        while (segmentListBCursor < segmentListB.size()
-                && segmentListB.get(segmentListBCursor).getIndexesCount() == 0) {
-            segmentListBCursor++;
-        }
+        Node node = kpq.peek();
+        Objects.requireNonNull(node);
 
-        DelayedIndex delayedIndexA = null;
-        DelayedIndex delayedIndexB = null;
-        if (segmentListACursor >= segmentListA.size()) {
-            delayedIndexB = segmentListB.get(segmentListBCursor).getIndexes(segmentBCursor);
-        } else if (segmentListBCursor >= segmentListB.size()) {
-            delayedIndexA = segmentListA.get(segmentListACursor).getIndexes(segmentACursor);
-        } else {
-            delayedIndexA = segmentListA.get(segmentListACursor).getIndexes(segmentACursor);
-            delayedIndexB = segmentListB.get(segmentListBCursor).getIndexes(segmentBCursor);
+        SnapshotSegment snapshotSegment = node.segmentList.get(node.segmentListCursor);
+        DelayedIndex delayedIndex = snapshotSegment.getIndexes(node.segmentCursor);
+        if (!needAdvanceCursor) {
+            return delayedIndex;
         }
 
-        DelayedIndex resultValue;
-        if (delayedIndexB == null || (delayedIndexA != null && COMPARATOR.compare(delayedIndexA, delayedIndexB) < 0)) {
-            resultValue = delayedIndexA;
-            if (needAdvanceCursor) {
-                if (++segmentACursor >= segmentListA.get(segmentListACursor).getIndexesCount()) {
-                    segmentListA.set(segmentListACursor, null);
-                    ++segmentListACursor;
-                    segmentACursor = 0;
-                }
-            }
-        } else {
-            resultValue = delayedIndexB;
-            if (needAdvanceCursor) {
-                if (++segmentBCursor >= segmentListB.get(segmentListBCursor).getIndexesCount()) {
-                    segmentListB.set(segmentListBCursor, null);
-                    ++segmentListBCursor;
-                    segmentBCursor = 0;
+        kpq.poll();
+
+        if (node.segmentCursor + 1 < snapshotSegment.getIndexesCount()) {
+            node.segmentCursor++;
+            kpq.offer(node);
+        } else  {
+            // help GC
+            node.segmentList.set(node.segmentListCursor, null);
+            while (node.segmentListCursor + 1 < node.segmentList.size()) {
+                node.segmentListCursor++;
+                node.segmentCursor = 0;
+
+                // skip empty segment
+                if (node.segmentList.get(node.segmentListCursor).getIndexesCount() > 0) {
+                    kpq.offer(node);
+                    break;
                 }
             }
         }
 
-        return resultValue;
+        return delayedIndex;
     }
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
index fbb866d48ed69..717bada7705dc 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
@@ -267,9 +267,7 @@ public void testMergeSnapshot(final BucketDelayedDeliveryTracker tracker) {
 
         int size = tracker.getImmutableBuckets().asMapOfRanges().size();
 
-        Awaitility.await().untilAsserted(() -> {
-            assertEquals(10, size);
-        });
+        assertTrue(size <= 10);
 
         tracker.addMessage(111, 1011, 111 * 10);
         Awaitility.await().untilAsserted(() -> {
@@ -333,9 +331,7 @@ public void testWithBkException(final BucketDelayedDeliveryTracker tracker) {
 
         int size = tracker.getImmutableBuckets().asMapOfRanges().size();
 
-        Awaitility.await().untilAsserted(() -> {
-            assertEquals(10, size);
-        });
+        assertTrue(size <= 10);
 
         tracker.addMessage(111, 1011, 111 * 10);
         Awaitility.await().untilAsserted(() -> {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueueTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueueTest.java
index 865ccb6934ab7..8f87f0d49a20c 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueueTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueueTest.java
@@ -101,8 +101,25 @@ public void testCombinedSegmentDelayedIndexQueue() {
         segmentListB.add(snapshotSegmentB);
         segmentListB.add(SnapshotSegment.newBuilder().build());
 
+        List listC = new ArrayList<>();
+        for (int i = 10; i < 30; i+=2) {
+            DelayedIndex delayedIndex =
+                    DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(2L).setEntryId(1L)
+                            .build();
+
+            DelayedIndex delayedIndex2 =
+                    DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(2L).setEntryId(2L)
+                            .build();
+            listC.add(delayedIndex);
+            listC.add(delayedIndex2);
+        }
+
+        SnapshotSegment snapshotSegmentC = SnapshotSegment.newBuilder().addAllIndexes(listC).build();
+        List segmentListC = new ArrayList<>();
+        segmentListC.add(snapshotSegmentC);
+
         CombinedSegmentDelayedIndexQueue delayedIndexQueue =
-                CombinedSegmentDelayedIndexQueue.wrap(segmentListA, segmentListB);
+                CombinedSegmentDelayedIndexQueue.wrap(List.of(segmentListA, segmentListB, segmentListC));
 
         int count = 0;
         while (!delayedIndexQueue.isEmpty()) {
@@ -114,7 +131,7 @@ public void testCombinedSegmentDelayedIndexQueue() {
                 Assert.assertTrue(COMPARATOR.compare(peek, pop) >= 0);
             }
         }
-        Assert.assertEquals(38, count);
+        Assert.assertEquals(58, count);
     }
 
     @Test

From 7cb48fd9d41f8606d86a0503b0130aec6b9d00d5 Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com>
Date: Tue, 28 Mar 2023 23:54:05 -0700
Subject: [PATCH 161/174] [improve][io] KCA: option to collapse partitioned
 topics (#19923)

---
 pulsar-io/kafka-connect-adaptor/pom.xml       |  7 ++
 .../io/kafka/connect/KafkaConnectSink.java    | 20 ++++-
 .../connect/PulsarKafkaConnectSinkConfig.java |  5 ++
 .../kafka/connect/KafkaConnectSinkTest.java   | 88 +++++++++++++++++++
 4 files changed, 118 insertions(+), 2 deletions(-)

diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml
index 5e192be95ecf4..12ebda087eb3d 100644
--- a/pulsar-io/kafka-connect-adaptor/pom.xml
+++ b/pulsar-io/kafka-connect-adaptor/pom.xml
@@ -39,6 +39,13 @@
       provided
     
 
+    
+      ${project.groupId}
+      pulsar-common
+      ${project.version}
+      compile
+    
+
     
       com.fasterxml.jackson.core
       jackson-databind
diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java
index 06f66f60380d9..475724cc4e545 100644
--- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java
+++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java
@@ -54,6 +54,7 @@
 import org.apache.pulsar.client.api.SubscriptionType;
 import org.apache.pulsar.client.api.schema.GenericObject;
 import org.apache.pulsar.client.api.schema.KeyValueSchema;
+import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.schema.KeyValue;
 import org.apache.pulsar.common.schema.SchemaType;
 import org.apache.pulsar.functions.api.Record;
@@ -91,6 +92,9 @@ public class KafkaConnectSink implements Sink {
     protected String topicName;
 
     private boolean sanitizeTopicName = false;
+    // Thi is a workaround for https://github.com/apache/pulsar/issues/19922
+    private boolean collapsePartitionedTopics = false;
+
     private final Cache sanitizedTopicCache =
             CacheBuilder.newBuilder().maximumSize(1000)
                     .expireAfterAccess(30, TimeUnit.MINUTES).build();
@@ -159,6 +163,7 @@ public void open(Map config, SinkContext ctx) throws Exception {
         topicName = kafkaSinkConfig.getTopic();
         unwrapKeyValueIfAvailable = kafkaSinkConfig.isUnwrapKeyValueIfAvailable();
         sanitizeTopicName = kafkaSinkConfig.isSanitizeTopicName();
+        collapsePartitionedTopics = kafkaSinkConfig.isCollapsePartitionedTopics();
 
         useIndexAsOffset = kafkaSinkConfig.isUseIndexAsOffset();
         maxBatchBitsForOffset = kafkaSinkConfig.getMaxBatchBitsForOffset();
@@ -417,8 +422,19 @@ static BatchMessageSequenceRef getMessageSequenceRefForBatchMessage(MessageId me
 
     @SuppressWarnings("rawtypes")
     protected SinkRecord toSinkRecord(Record sourceRecord) {
-        final int partition = sourceRecord.getPartitionIndex().orElse(0);
-        final String topic = sanitizeNameIfNeeded(sourceRecord.getTopicName().orElse(topicName), sanitizeTopicName);
+        final int partition;
+        final String topic;
+
+        if (collapsePartitionedTopics
+                && sourceRecord.getTopicName().isPresent()
+                && TopicName.get(sourceRecord.getTopicName().get()).isPartitioned()) {
+            TopicName tn = TopicName.get(sourceRecord.getTopicName().get());
+            partition = tn.getPartitionIndex();
+            topic = sanitizeNameIfNeeded(tn.getPartitionedTopicName(), sanitizeTopicName);
+        } else {
+            partition = sourceRecord.getPartitionIndex().orElse(0);
+            topic = sanitizeNameIfNeeded(sourceRecord.getTopicName().orElse(topicName), sanitizeTopicName);
+        }
         final Object key;
         final Object value;
         final Schema keySchema;
diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaConnectSinkConfig.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaConnectSinkConfig.java
index 19dd784578915..2525081a41ebb 100644
--- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaConnectSinkConfig.java
+++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaConnectSinkConfig.java
@@ -94,6 +94,11 @@ public class PulsarKafkaConnectSinkConfig implements Serializable {
                     + "In some cases it may result in topic name collisions (topic_a and topic.a will become the same)")
     private boolean sanitizeTopicName = false;
 
+    @FieldDoc(
+            defaultValue = "false",
+            help = "Supply kafka record with topic name without -partition- suffix for partitioned topics.")
+    private boolean collapsePartitionedTopics = false;
+
     public static PulsarKafkaConnectSinkConfig load(String yamlFile) throws IOException {
         ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
         return mapper.readValue(new File(yamlFile), PulsarKafkaConnectSinkConfig.class);
diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java
index e9d454ed2fd5a..6ccfa3b71a2f7 100644
--- a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java
+++ b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java
@@ -1571,6 +1571,94 @@ public void testGetMessageSequenceRefForBatchMessage() throws Exception {
         assertEquals(ref.getBatchIdx(), batchIdx);
     }
 
+    @Test
+    public void collapsePartitionedTopicEnabledTest() throws Exception {
+        testCollapsePartitionedTopic(true,
+                "persistent://a/b/fake-topic-partition-0",
+                "persistent://a/b/fake-topic",
+                0);
+
+        testCollapsePartitionedTopic(true,
+                "persistent://a/b/fake-topic-partition-1",
+                "persistent://a/b/fake-topic",
+                1);
+
+        testCollapsePartitionedTopic(true,
+                "persistent://a/b/fake-topic",
+                "persistent://a/b/fake-topic",
+                0);
+
+        testCollapsePartitionedTopic(true,
+                "fake-topic-partition-5",
+                "persistent://public/default/fake-topic",
+                5);
+    }
+
+    @Test
+    public void collapsePartitionedTopicDisabledTest() throws Exception {
+        testCollapsePartitionedTopic(false,
+                "persistent://a/b/fake-topic-partition-0",
+                "persistent://a/b/fake-topic-partition-0",
+                0);
+
+        testCollapsePartitionedTopic(false,
+                "persistent://a/b/fake-topic-partition-1",
+                "persistent://a/b/fake-topic-partition-1",
+                0);
+
+        testCollapsePartitionedTopic(false,
+                "persistent://a/b/fake-topic",
+                "persistent://a/b/fake-topic",
+                0);
+
+        testCollapsePartitionedTopic(false,
+                "fake-topic-partition-5",
+                "fake-topic-partition-5",
+                0);
+    }
+
+    private void testCollapsePartitionedTopic(boolean isEnabled,
+                                              String pulsarTopic,
+                                              String expectedKafkaTopic,
+                                              int expectedPartition) throws Exception {
+        props.put("kafkaConnectorSinkClass", SchemaedFileStreamSinkConnector.class.getCanonicalName());
+        props.put("collapsePartitionedTopics", Boolean.toString(isEnabled));
+
+        KafkaConnectSink sink = new KafkaConnectSink();
+            sink.open(props, context);
+
+        AvroSchema pulsarAvroSchema
+                = AvroSchema.of(PulsarSchemaToKafkaSchemaTest.StructWithAnnotations.class);
+
+        final GenericData.Record obj = new GenericData.Record(pulsarAvroSchema.getAvroSchema());
+        obj.put("field1", (byte) 10);
+        obj.put("field2", "test");
+        obj.put("field3", (short) 100);
+
+        final GenericRecord rec = getGenericRecord(obj, pulsarAvroSchema);
+        Message msg = mock(MessageImpl.class);
+        when(msg.getValue()).thenReturn(rec);
+        when(msg.getKey()).thenReturn("key");
+        when(msg.hasKey()).thenReturn(true);
+        when(msg.getMessageId()).thenReturn(new MessageIdImpl(1, 0, 0));
+
+        final AtomicInteger status = new AtomicInteger(0);
+        Record record = PulsarRecord.builder()
+                .topicName(pulsarTopic)
+                .message(msg)
+                .schema(pulsarAvroSchema)
+                .ackFunction(status::incrementAndGet)
+                .failFunction(status::decrementAndGet)
+                .build();
+
+        SinkRecord sinkRecord = sink.toSinkRecord(record);
+
+        Assert.assertEquals(sinkRecord.topic(), expectedKafkaTopic);
+        Assert.assertEquals(sinkRecord.kafkaPartition(), expectedPartition);
+
+        sink.close();
+    }
+
     @SneakyThrows
     private java.util.Date getDateFromString(String dateInString) {
         SimpleDateFormat formatter = new SimpleDateFormat("dd/MM/yyyy hh:mm:ss");

From ee5ac8607ebd7f90843789d4eead48de8600c1f0 Mon Sep 17 00:00:00 2001
From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com>
Date: Wed, 29 Mar 2023 02:30:24 -0700
Subject: [PATCH 162/174] [improve][broker] PIP-192 Improved Auto Unload Logic
 (#19813)

Master Issue: Master Issue: https://github.com/apache/pulsar/issues/16691, https://github.com/apache/pulsar/issues/18099

### Motivation

Raising a PR to implement Master Issue: https://github.com/apache/pulsar/issues/16691, https://github.com/apache/pulsar/issues/18099

We want to reduce unload frequencies from flaky traffic.

### Modifications
This PR
- Introduced a config `loadBalancerSheddingConditionHitCountThreshold` to further restrict shedding conditions based on the hit count.
- Normalized offload traffic
- Lowered the default `loadBalanceSheddingDelayInSeconds` value from 600 to 180, as 10 mins are too long. 3 mins can be long enough to catch the new load after unloads.
- Changed the config `loadBalancerBundleLoadReportPercentage` to `loadBalancerMaxNumberOfBundlesInBundleLoadReport` to make the topk bundle count absolute instead of relative.
- Renamed `loadBalancerNamespaceBundleSplitConditionThreshold` to `loadBalancerNamespaceBundleSplitConditionHitCountThreshold` to be consistent with `*ConditionHitCountThreshold`.
- Renamed `loadBalancerMaxNumberOfBrokerTransfersPerCycle ` to `loadBalancerMaxNumberOfBrokerSheddingPerCycle`.
- Added LoadDataStore cleanup logic in BSC monitor.
- Added `msgThroughputEMA` in BrokerLoadData to smooth the broker throughput info.
- Updated Topk bundles sorted in a ascending order (instead of descending)
- Update some info logs to only show in the debug mode.
- Added load data tombstone upon Own, Releasing, Splitting
- Added the bundle ownership(isOwned) check upon split and unload.
- Added swap unload logic
---
 conf/broker.conf                              |  85 +++
 .../pulsar/broker/ServiceConfiguration.java   |  23 +-
 .../extensions/ExtensibleLoadManagerImpl.java |  67 ++-
 .../channel/ServiceUnitStateChannel.java      |  25 +
 .../channel/ServiceUnitStateChannelImpl.java  |  85 ++-
 .../extensions/data/BrokerLoadData.java       |  34 +-
 .../extensions/models/TopKBundles.java        |   2 +-
 .../extensions/models/UnloadCounter.java      |   4 +-
 .../extensions/models/UnloadDecision.java     |   2 +-
 .../reporter/BrokerLoadDataReporter.java      |  90 ++-
 .../reporter/TopBundleLoadDataReporter.java   |  63 ++-
 .../scheduler/NamespaceUnloadStrategy.java    |   8 +
 .../extensions/scheduler/SplitScheduler.java  |  16 +-
 .../extensions/scheduler/TransferShedder.java | 469 +++++++++++-----
 .../extensions/scheduler/UnloadScheduler.java |  19 +-
 ...faultNamespaceBundleSplitStrategyImpl.java | 144 +++--
 .../LeastResourceUsageWithWeight.java         |   8 +-
 .../ExtensibleLoadManagerImplTest.java        |   6 +-
 .../channel/ServiceUnitStateChannelTest.java  |  66 +++
 .../extensions/data/BrokerLoadDataTest.java   |  10 +-
 .../extensions/models/TopKBundlesTest.java    |  14 +-
 .../reporter/BrokerLoadDataReporterTest.java  |  93 +++-
 .../TopBundleLoadDataReporterTest.java        |  90 ++-
 .../scheduler/TransferShedderTest.java        | 515 +++++++++++++++---
 .../scheduler/UnloadSchedulerTest.java        |  12 +-
 ...faultNamespaceBundleSplitStrategyTest.java |  49 +-
 .../loadbalancer/NamespaceBundleStats.java    |   2 +
 .../data/loadbalancer/ResourceUsage.java      |   2 +
 28 files changed, 1648 insertions(+), 355 deletions(-)

diff --git a/conf/broker.conf b/conf/broker.conf
index d52adb254563d..1bbb12313f9a6 100644
--- a/conf/broker.conf
+++ b/conf/broker.conf
@@ -1373,6 +1373,91 @@ loadBalancerBundleUnloadMinThroughputThreshold=10
 # Time to wait for the unloading of a namespace bundle
 namespaceBundleUnloadingTimeoutMs=60000
 
+### --- Load balancer extension --- ###
+
+# Option to enable the debug mode for the load balancer logics.
+# The debug mode prints more logs to provide more information such as load balance states and decisions.
+# (only used in load balancer extension logics)
+loadBalancerDebugModeEnabled=false
+
+# The target standard deviation of the resource usage across brokers
+# (100% resource usage is 1.0 load).
+# The shedder logic tries to distribute bundle load across brokers to meet this target std.
+# The smaller value will incur load balancing more frequently.
+# (only used in load balancer extension TransferSheddeer)
+loadBalancerBrokerLoadTargetStd=0.25
+
+# Threshold to the consecutive count of fulfilled shedding(unload) conditions.
+# If the unload scheduler consecutively finds bundles that meet unload conditions
+# many times bigger than this threshold, the scheduler will shed the bundles.
+# The bigger value will incur less bundle unloading/transfers.
+# (only used in load balancer extension TransferSheddeer)
+loadBalancerSheddingConditionHitCountThreshold=3
+
+# Option to enable the bundle transfer mode when distributing bundle loads.
+# On: transfer bundles from overloaded brokers to underloaded
+# -- pre-assigns the destination broker upon unloading).
+# Off: unload bundles from overloaded brokers
+# -- post-assigns the destination broker upon lookups).
+# (only used in load balancer extension TransferSheddeer)
+loadBalancerTransferEnabled=true
+
+# Maximum number of brokers to unload bundle load for each unloading cycle.
+# The bigger value will incur more unloading/transfers for each unloading cycle.
+# (only used in load balancer extension TransferSheddeer)
+loadBalancerMaxNumberOfBrokerSheddingPerCycle=3
+
+# Delay (in seconds) to the next unloading cycle after unloading.
+# The logic tries to give enough time for brokers to recompute load after unloading.
+# The bigger value will delay the next unloading cycle longer.
+# (only used in load balancer extension TransferSheddeer)
+loadBalanceSheddingDelayInSeconds=180
+
+# Broker load data time to live (TTL in seconds).
+# The logic tries to avoid (possibly unavailable) brokers with out-dated load data,
+# and those brokers will be ignored in the load computation.
+# When tuning this value, please consider loadBalancerReportUpdateMaxIntervalMinutes.
+#The current default is loadBalancerReportUpdateMaxIntervalMinutes * 2.
+# (only used in load balancer extension TransferSheddeer)
+loadBalancerBrokerLoadDataTTLInSeconds=1800
+
+# Max number of bundles in bundle load report from each broker.
+# The load balancer distributes bundles across brokers,
+# based on topK bundle load data and other broker load data.
+# The bigger value will increase the overhead of reporting many bundles in load data.
+# (only used in load balancer extension logics)
+loadBalancerMaxNumberOfBundlesInBundleLoadReport=10
+
+# Service units'(bundles) split interval. Broker periodically checks whether
+# some service units(e.g. bundles) should split if they become hot-spots.
+# (only used in load balancer extension logics)
+loadBalancerSplitIntervalMinutes=1
+
+# Max number of bundles to split to per cycle.
+# (only used in load balancer extension logics)
+loadBalancerMaxNumberOfBundlesToSplitPerCycle=10
+
+# Threshold to the consecutive count of fulfilled split conditions.
+# If the split scheduler consecutively finds bundles that meet split conditions
+# many times bigger than this threshold, the scheduler will trigger splits on the bundles
+# (if the number of bundles is less than loadBalancerNamespaceMaximumBundles).
+# (only used in load balancer extension logics)
+loadBalancerNamespaceBundleSplitConditionHitCountThreshold=3
+
+# After this delay, the service-unit state channel tombstones any service units (e.g., bundles)
+# in semi-terminal states. For example, after splits, parent bundles will be `deleted`,
+# and then after this delay, the parent bundles' state will be `tombstoned`
+# in the service-unit state channel.
+# Pulsar does not immediately remove such semi-terminal states
+# to avoid unnecessary system confusion,
+# as the bundles in the `tombstoned` state might temporarily look available to reassign.
+# Rarely, one could lower this delay in order to aggressively clean
+# the service-unit state channel when there are a large number of bundles.
+# minimum value = 30 secs
+# (only used in load balancer extension logics)
+loadBalancerServiceUnitStateCleanUpDelayTimeInSeconds=604800
+
+
 ### --- Replication --- ###
 
 # Enable replication metrics
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index e2bcf03353b59..03e0fec0a5346 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -2506,6 +2506,17 @@ The delayed message index time step(in seconds) in per bucket snapshot segment,
     )
     private double loadBalancerBrokerLoadTargetStd = 0.25;
 
+    @FieldContext(
+            category = CATEGORY_LOAD_BALANCER,
+            dynamic = true,
+            doc = "Threshold to the consecutive count of fulfilled shedding(unload) conditions. "
+                    + "If the unload scheduler consecutively finds bundles that meet unload conditions "
+                    + "many times bigger than this threshold, the scheduler will shed the bundles. "
+                    + "The bigger value will incur less bundle unloading/transfers. "
+                    + "(only used in load balancer extension TransferSheddeer)"
+    )
+    private int loadBalancerSheddingConditionHitCountThreshold = 3;
+
     @FieldContext(
             category = CATEGORY_LOAD_BALANCER,
             dynamic = true,
@@ -2521,11 +2532,11 @@ The delayed message index time step(in seconds) in per bucket snapshot segment,
     @FieldContext(
             category = CATEGORY_LOAD_BALANCER,
             dynamic = true,
-            doc = "Maximum number of brokers to transfer bundle load for each unloading cycle. "
+            doc = "Maximum number of brokers to unload bundle load for each unloading cycle. "
                     + "The bigger value will incur more unloading/transfers for each unloading cycle. "
                     + "(only used in load balancer extension TransferSheddeer)"
     )
-    private int loadBalancerMaxNumberOfBrokerTransfersPerCycle = 3;
+    private int loadBalancerMaxNumberOfBrokerSheddingPerCycle = 3;
 
     @FieldContext(
             category = CATEGORY_LOAD_BALANCER,
@@ -2535,7 +2546,7 @@ The delayed message index time step(in seconds) in per bucket snapshot segment,
                     + "The bigger value will delay the next unloading cycle longer. "
                     + "(only used in load balancer extension TransferSheddeer)"
     )
-    private long loadBalanceUnloadDelayInSeconds = 600;
+    private long loadBalanceSheddingDelayInSeconds = 180;
 
     @FieldContext(
             category = CATEGORY_LOAD_BALANCER,
@@ -2552,13 +2563,13 @@ The delayed message index time step(in seconds) in per bucket snapshot segment,
     @FieldContext(
             dynamic = true,
             category = CATEGORY_LOAD_BALANCER,
-            doc = "Percentage of bundles to compute topK bundle load data from each broker. "
+            doc = "Max number of bundles in bundle load report from each broker. "
                     + "The load balancer distributes bundles across brokers, "
                     + "based on topK bundle load data and other broker load data."
                     + "The bigger value will increase the overhead of reporting many bundles in load data. "
                     + "(only used in load balancer extension logics)"
     )
-    private double loadBalancerBundleLoadReportPercentage = 10;
+    private int loadBalancerMaxNumberOfBundlesInBundleLoadReport = 10;
     @FieldContext(
             category = CATEGORY_LOAD_BALANCER,
             doc = "Service units'(bundles) split interval. Broker periodically checks whether "
@@ -2582,7 +2593,7 @@ The delayed message index time step(in seconds) in per bucket snapshot segment,
                     + "(if the number of bundles is less than loadBalancerNamespaceMaximumBundles). "
                     + "(only used in load balancer extension logics)"
     )
-    private int loadBalancerNamespaceBundleSplitConditionThreshold = 5;
+    private int loadBalancerNamespaceBundleSplitConditionHitCountThreshold = 3;
 
     @FieldContext(
             category = CATEGORY_LOAD_BALANCER,
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
index 7aefef596e7cb..3e078d0a5eb20 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java
@@ -85,6 +85,7 @@
 import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap;
 import org.apache.pulsar.metadata.api.coordination.LeaderElectionState;
+import org.slf4j.Logger;
 
 @Slf4j
 public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
@@ -101,6 +102,8 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
 
     private static final long MAX_ROLE_CHANGE_RETRY_DELAY_IN_MILLIS = 200;
 
+    private static final long MONITOR_INTERVAL_IN_MILLIS = 120_000;
+
     private PulsarService pulsar;
 
     private ServiceConfiguration conf;
@@ -108,10 +111,12 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
     @Getter
     private BrokerRegistry brokerRegistry;
 
+    @Getter
     private ServiceUnitStateChannel serviceUnitStateChannel;
 
     private AntiAffinityGroupPolicyFilter antiAffinityGroupPolicyFilter;
 
+    @Getter
     private AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper;
 
     private LoadDataStore brokerLoadDataStore;
@@ -139,6 +144,8 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
 
     private ScheduledFuture brokerLoadDataReportTask;
     private ScheduledFuture topBundlesLoadDataReportTask;
+
+    private ScheduledFuture monitorTask;
     private SplitScheduler splitScheduler;
 
     private UnloadManager unloadManager;
@@ -148,6 +155,7 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
     private boolean started = false;
 
     private final AssignCounter assignCounter = new AssignCounter();
+    @Getter
     private final UnloadCounter unloadCounter = new UnloadCounter();
     private final SplitCounter splitCounter = new SplitCounter();
 
@@ -162,14 +170,14 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager {
             lookupRequests = ConcurrentOpenHashMap.>>newBuilder()
             .build();
-    private final CountDownLatch loadStoreInitWaiter = new CountDownLatch(1);
+    private final CountDownLatch initWaiter = new CountDownLatch(1);
 
     public enum Role {
         Leader,
         Follower
     }
 
-    private Role role;
+    private volatile Role role;
 
     /**
      * Life cycle: Constructor -> initialize -> start -> close.
@@ -194,6 +202,10 @@ public static ExtensibleLoadManagerImpl get(LoadManager loadManager) {
         return loadManagerWrapper.get();
     }
 
+    public static boolean debug(ServiceConfiguration config, Logger log) {
+        return config.isLoadBalancerDebugModeEnabled() || log.isDebugEnabled();
+    }
+
     @Override
     public void start() throws PulsarServerException {
         if (this.started) {
@@ -231,7 +243,6 @@ public void start() throws PulsarServerException {
             this.brokerLoadDataStore.startTableView();
             this.topBundlesLoadDataStore = LoadDataStoreFactory
                     .create(pulsar.getClient(), TOP_BUNDLES_LOAD_DATA_STORE_TOPIC, TopBundlesLoadData.class);
-            this.loadStoreInitWaiter.countDown();
         } catch (LoadDataStoreException e) {
             throw new PulsarServerException(e);
         }
@@ -247,7 +258,8 @@ public void start() throws PulsarServerException {
 
         this.topBundleLoadDataReporter =
                 new TopBundleLoadDataReporter(pulsar, brokerRegistry.getBrokerId(), topBundlesLoadDataStore);
-
+        this.serviceUnitStateChannel.listen(brokerLoadDataReporter);
+        this.serviceUnitStateChannel.listen(topBundleLoadDataReporter);
         var interval = conf.getLoadBalancerReportUpdateMinIntervalMillis();
         this.brokerLoadDataReportTask = this.pulsar.getLoadManagerExecutor()
                 .scheduleAtFixedRate(() -> {
@@ -273,13 +285,21 @@ public void start() throws PulsarServerException {
                         interval,
                         interval, TimeUnit.MILLISECONDS);
 
+        this.monitorTask = this.pulsar.getLoadManagerExecutor()
+                .scheduleAtFixedRate(() -> {
+                            monitor();
+                        },
+                        MONITOR_INTERVAL_IN_MILLIS,
+                        MONITOR_INTERVAL_IN_MILLIS, TimeUnit.MILLISECONDS);
+
         this.unloadScheduler = new UnloadScheduler(
-                pulsar, pulsar.getLoadManagerExecutor(), unloadManager,
-                context, serviceUnitStateChannel, antiAffinityGroupPolicyHelper, unloadCounter, unloadMetrics);
+                pulsar, pulsar.getLoadManagerExecutor(), unloadManager, context, serviceUnitStateChannel,
+                unloadCounter, unloadMetrics);
         this.unloadScheduler.start();
         this.splitScheduler = new SplitScheduler(
                 pulsar, serviceUnitStateChannel, splitManager, splitCounter, splitMetrics, context);
         this.splitScheduler.start();
+        this.initWaiter.countDown();
         this.started = true;
     }
 
@@ -327,7 +347,7 @@ public CompletableFuture> assign(Optional {
                 if (broker.isEmpty()) {
                     String errorMsg = String.format(
-                            "Failed to look up a broker registry:%s for bundle:%s", broker, bundle);
+                            "Failed to get or assign the owner for bundle:%s", bundle);
                     log.error(errorMsg);
                     throw new IllegalStateException(errorMsg);
                 }
@@ -498,6 +518,10 @@ public void close() throws PulsarServerException {
                 topBundlesLoadDataReportTask.cancel(true);
             }
 
+            if (monitorTask != null) {
+                monitorTask.cancel(true);
+            }
+
             this.brokerLoadDataStore.close();
             this.topBundlesLoadDataStore.close();
             this.unloadScheduler.close();
@@ -539,8 +563,8 @@ void playLeader() {
             int retry = 0;
             while (true) {
                 try {
+                    initWaiter.await();
                     serviceUnitStateChannel.scheduleOwnershipMonitor();
-                    loadStoreInitWaiter.await();
                     topBundlesLoadDataStore.startTableView();
                     unloadScheduler.start();
                     break;
@@ -575,8 +599,8 @@ void playFollower() {
             int retry = 0;
             while (true) {
                 try {
+                    initWaiter.await();
                     serviceUnitStateChannel.cancelOwnershipMonitor();
-                    loadStoreInitWaiter.await();
                     topBundlesLoadDataStore.closeTableView();
                     unloadScheduler.close();
                     break;
@@ -631,4 +655,29 @@ public List getMetrics() {
 
         return metricsCollection;
     }
+
+    private void monitor() {
+        try {
+            initWaiter.await();
+
+            // Monitor role
+            // Periodically check the role in case ZK watcher fails.
+            var isChannelOwner = serviceUnitStateChannel.isChannelOwner();
+            if (isChannelOwner) {
+                if (role != Leader) {
+                    log.warn("Current role:{} does not match with the channel ownership:{}. "
+                            + "Playing the leader role.", role, isChannelOwner);
+                    playLeader();
+                }
+            } else {
+                if (role != Follower) {
+                    log.warn("Current role:{} does not match with the channel ownership:{}. "
+                            + "Playing the follower role.", role, isChannelOwner);
+                    playFollower();
+                }
+            }
+        } catch (Throwable e) {
+            log.error("Failed to get the channel ownership.", e);
+        }
+    }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java
index 719c72a67b4ea..4782a31fe0c56 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java
@@ -64,6 +64,12 @@ public interface ServiceUnitStateChannel extends Closeable {
      */
     CompletableFuture isChannelOwnerAsync();
 
+    /**
+     * Checks if the current broker is the owner broker of the system topic in this channel.
+     * @return True if the current broker is the owner. Otherwise, false.
+     */
+    boolean isChannelOwner();
+
     /**
      * Handles the metadata session events to track
      * if the connection between the broker and metadata store is stable or not.
@@ -125,6 +131,25 @@ public interface ServiceUnitStateChannel extends Closeable {
      */
     CompletableFuture> getOwnerAsync(String serviceUnit);
 
+    /**
+     * Checks if the target broker is the owner of the service unit.
+     *
+     *
+     * @param serviceUnit (e.g. bundle)
+     * @param targetBroker
+     * @return true if the target broker is the owner. false if unknown.
+     */
+    boolean isOwner(String serviceUnit, String targetBroker);
+
+    /**
+     * Checks if the current broker is the owner of the service unit.
+     *
+     *
+     * @param serviceUnit (e.g. bundle))
+     * @return true if the current broker is the owner. false if unknown.
+     */
+    boolean isOwner(String serviceUnit);
+
     /**
      * Asynchronously publishes the service unit assignment event to the system topic in this channel.
      * It de-duplicates assignment events if there is any ongoing assignment event for the same service unit.
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
index ca9e55923e799..e7fb59450b25c 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java
@@ -70,6 +70,7 @@
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.loadbalance.LeaderElectionService;
 import org.apache.pulsar.broker.loadbalance.extensions.BrokerRegistry;
+import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl;
 import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerWrapper;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
 import org.apache.pulsar.broker.loadbalance.extensions.manager.StateChangeListener;
@@ -233,7 +234,7 @@ public void scheduleOwnershipMonitor() {
                                     log.info("Failed to monitor the ownerships. will retry..", e);
                                 }
                             },
-                            ownershipMonitorDelayTimeInSecs, ownershipMonitorDelayTimeInSecs, SECONDS);
+                            0, ownershipMonitorDelayTimeInSecs, SECONDS);
             log.info("This leader broker:{} started the ownership monitor.",
                     lookupServiceAddress);
         }
@@ -390,7 +391,7 @@ private boolean validateChannelState(ChannelState targetState, boolean checkLowe
     }
 
     private boolean debug() {
-        return config.isLoadBalancerDebugModeEnabled() || log.isDebugEnabled();
+        return ExtensibleLoadManagerImpl.debug(config, log);
     }
 
     public CompletableFuture> getChannelOwnerAsync() {
@@ -426,7 +427,7 @@ public CompletableFuture isChannelOwnerAsync() {
         });
     }
 
-    private boolean isChannelOwner() {
+    public boolean isChannelOwner() {
         try {
             return isChannelOwnerAsync().get(
                     MAX_CHANNEL_OWNER_ELECTION_WAITING_TIME_IN_SECS, TimeUnit.SECONDS);
@@ -437,6 +438,25 @@ private boolean isChannelOwner() {
         }
     }
 
+    public boolean isOwner(String serviceUnit, String targetBroker) {
+        if (!validateChannelState(Started, true)) {
+            throw new IllegalStateException("Invalid channel state:" + channelState.name());
+        }
+        var ownerFuture = getOwnerAsync(serviceUnit);
+        if (!ownerFuture.isDone() || ownerFuture.isCompletedExceptionally() || ownerFuture.isCancelled()) {
+            return false;
+        }
+        var owner = ownerFuture.join();
+        if (owner.isPresent() && StringUtils.equals(targetBroker, owner.get())) {
+            return true;
+        }
+        return false;
+    }
+
+    public boolean isOwner(String serviceUnit) {
+        return isOwner(serviceUnit, lookupServiceAddress);
+    }
+
     public CompletableFuture> getOwnerAsync(String serviceUnit) {
         if (!validateChannelState(Started, true)) {
             return CompletableFuture.failedFuture(
@@ -951,9 +971,11 @@ private void handleBrokerCreationEvent(String broker) {
                             + " Active cleanup job count:{}",
                     broker, cleanupJobs.size());
         } else {
-            log.info("Failed to cancel the ownership cleanup for broker:{}."
-                            + " There was no scheduled cleanup job. Active cleanup job count:{}",
-                    broker, cleanupJobs.size());
+            if (debug()) {
+                log.info("No needs to cancel the ownership cleanup for broker:{}."
+                                + " There was no scheduled cleanup job. Active cleanup job count:{}",
+                        broker, cleanupJobs.size());
+            }
         }
     }
 
@@ -989,6 +1011,8 @@ private void scheduleCleanup(String broker, long delayInSecs) {
                                     log.error("Failed to run the cleanup job for the broker {}, "
                                                     + "totalCleanupErrorCnt:{}.",
                                             broker, totalCleanupErrorCnt.incrementAndGet(), e);
+                                } finally {
+                                    cleanupJobs.remove(broker);
                                 }
                             }
                             , delayed);
@@ -1057,7 +1081,11 @@ private void doCleanup(String broker) throws ExecutionException, InterruptedExce
 
         double cleanupTime = TimeUnit.NANOSECONDS
                 .toMillis((System.nanoTime() - startTime));
-        // TODO: clean load data stores
+
+        // clean load data stores
+        getContext().topBundleLoadDataStore().removeAsync(broker);
+        getContext().brokerLoadDataStore().removeAsync(broker);
+
         log.info("Completed a cleanup for the inactive broker:{} in {} ms. "
                         + "Cleaned up orphan service units: orphanServiceUnitCleanupCnt:{}, "
                         + "approximate cleanupErrorCnt:{}, metrics:{} ",
@@ -1066,7 +1094,7 @@ private void doCleanup(String broker) throws ExecutionException, InterruptedExce
                 orphanServiceUnitCleanupCnt,
                 totalCleanupErrorCntStart - totalCleanupErrorCnt.get(),
                 printCleanupMetrics());
-        cleanupJobs.remove(broker);
+
     }
 
     private Optional getRollForwardStateData(
@@ -1123,7 +1151,11 @@ protected void monitorOwnerships(List brokers) {
             return;
         }
 
-        log.info("Started the ownership monitor run for activeBrokerCount:{}", brokers.size());
+        var debug = debug();
+        if (debug) {
+            log.info("Started the ownership monitor run for activeBrokerCount:{}", brokers.size());
+        }
+
         long startTime = System.nanoTime();
         Set inactiveBrokers = new HashSet<>();
         Set activeBrokers = new HashSet<>(brokers);
@@ -1199,27 +1231,32 @@ protected void monitorOwnerships(List brokers) {
             log.error("Failed to flush the in-flight messages.", e);
         }
 
+        boolean cleaned = false;
         if (serviceUnitTombstoneCleanupCnt > 0) {
             this.totalServiceUnitTombstoneCleanupCnt += serviceUnitTombstoneCleanupCnt;
+            cleaned = true;
         }
 
         if (orphanServiceUnitCleanupCnt > 0) {
             this.totalOrphanServiceUnitCleanupCnt += orphanServiceUnitCleanupCnt;
+            cleaned = true;
         }
 
-        double monitorTime = TimeUnit.NANOSECONDS
-                .toMillis((System.nanoTime() - startTime));
-        log.info("Completed the ownership monitor run in {} ms. "
-                        + "Scheduled cleanups for inactive brokers:{}. inactiveBrokerCount:{}. "
-                        + "Published cleanups for orphan service units, orphanServiceUnitCleanupCnt:{}. "
-                        + "Tombstoned semi-terminal state service units, serviceUnitTombstoneCleanupCnt:{}. "
-                        + "Approximate cleanupErrorCnt:{}, metrics:{}. ",
-                monitorTime,
-                inactiveBrokers, inactiveBrokers.size(),
-                orphanServiceUnitCleanupCnt,
-                serviceUnitTombstoneCleanupCnt,
-                totalCleanupErrorCntStart - totalCleanupErrorCnt.get(),
-                printCleanupMetrics());
+        if (debug || cleaned) {
+            double monitorTime = TimeUnit.NANOSECONDS
+                    .toMillis((System.nanoTime() - startTime));
+            log.info("Completed the ownership monitor run in {} ms. "
+                            + "Scheduled cleanups for inactive brokers:{}. inactiveBrokerCount:{}. "
+                            + "Published cleanups for orphan service units, orphanServiceUnitCleanupCnt:{}. "
+                            + "Tombstoned semi-terminal state service units, serviceUnitTombstoneCleanupCnt:{}. "
+                            + "Approximate cleanupErrorCnt:{}, metrics:{}. ",
+                    monitorTime,
+                    inactiveBrokers, inactiveBrokers.size(),
+                    orphanServiceUnitCleanupCnt,
+                    serviceUnitTombstoneCleanupCnt,
+                    totalCleanupErrorCntStart - totalCleanupErrorCnt.get(),
+                    printCleanupMetrics());
+        }
 
     }
 
@@ -1353,4 +1390,8 @@ public void listen(StateChangeListener listener) {
     public Set> getOwnershipEntrySet() {
         return tableview.entrySet();
     }
+
+    public static ServiceUnitStateChannel get(PulsarService pulsar) {
+        return ExtensibleLoadManagerImpl.get(pulsar.getLoadManager().get()).getServiceUnitStateChannel();
+    }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadData.java
index 8b373bc5954b2..48665d39a0d3e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadData.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadData.java
@@ -75,6 +75,7 @@ public class BrokerLoadData {
      * The historical resource percentage is configured by loadBalancerHistoryResourcePercentage.
      */
     private double weightedMaxEMA;
+    private double msgThroughputEMA;
     private long updatedAt;
 
     @Setter
@@ -88,6 +89,7 @@ public BrokerLoadData() {
         bandwidthOut = new ResourceUsage();
         maxResourceUsage = DEFAULT_RESOURCE_USAGE;
         weightedMaxEMA = DEFAULT_RESOURCE_USAGE;
+        msgThroughputEMA = 0;
     }
 
     /**
@@ -142,6 +144,7 @@ public void update(final BrokerLoadData other) {
         bundleCount = other.bundleCount;
         topics = other.topics;
         weightedMaxEMA = other.weightedMaxEMA;
+        msgThroughputEMA = other.msgThroughputEMA;
         maxResourceUsage = other.maxResourceUsage;
         updatedAt = other.updatedAt;
         reportedAt = other.reportedAt;
@@ -161,6 +164,7 @@ private void updateSystemResourceUsage(final ResourceUsage cpu, final ResourceUs
     private void updateFeatures(ServiceConfiguration conf) {
         updateMaxResourceUsage();
         updateWeightedMaxEMA(conf);
+        updateMsgThroughputEMA(conf);
     }
 
     private void updateMaxResourceUsage() {
@@ -188,6 +192,32 @@ private void updateWeightedMaxEMA(ServiceConfiguration conf) {
                 weightedMaxEMA * historyPercentage + (1 - historyPercentage) * weightedMax;
     }
 
+    private void updateMsgThroughputEMA(ServiceConfiguration conf) {
+        var historyPercentage = conf.getLoadBalancerHistoryResourcePercentage();
+        double msgThroughput = msgThroughputIn + msgThroughputOut;
+        msgThroughputEMA = updatedAt == 0 ? msgThroughput :
+                msgThroughputEMA * historyPercentage + (1 - historyPercentage) * msgThroughput;
+    }
+
+    public void clear() {
+        cpu = new ResourceUsage();
+        memory = new ResourceUsage();
+        directMemory = new ResourceUsage();
+        bandwidthIn = new ResourceUsage();
+        bandwidthOut = new ResourceUsage();
+        maxResourceUsage = DEFAULT_RESOURCE_USAGE;
+        weightedMaxEMA = DEFAULT_RESOURCE_USAGE;
+        msgThroughputEMA = 0;
+        msgThroughputIn = 0;
+        msgThroughputOut = 0;
+        msgRateIn = 0.0;
+        msgRateOut = 0.0;
+        bundleCount = 0;
+        topics = 0;
+        updatedAt = 0;
+        reportedAt = 0;
+    }
+
     public String toString(ServiceConfiguration conf) {
         return String.format("cpu= %.2f%%, memory= %.2f%%, directMemory= %.2f%%, "
                         + "bandwithIn= %.2f%%, bandwithOut= %.2f%%, "
@@ -195,7 +225,7 @@ public String toString(ServiceConfiguration conf) {
                         + "bandwithInResourceWeight= %f, bandwithOutResourceWeight= %f, "
                         + "msgThroughputIn= %.2f, msgThroughputOut= %.2f, msgRateIn= %.2f, msgRateOut= %.2f, "
                         + "bundleCount= %d, "
-                        + "maxResourceUsage= %.2f%%, weightedMaxEMA= %.2f%%, "
+                        + "maxResourceUsage= %.2f%%, weightedMaxEMA= %.2f%%, msgThroughputEMA= %.2f, "
                         + "updatedAt= %d, reportedAt= %d",
 
                 cpu.percentUsage(), memory.percentUsage(), directMemory.percentUsage(),
@@ -207,7 +237,7 @@ public String toString(ServiceConfiguration conf) {
                 conf.getLoadBalancerBandwithOutResourceWeight(),
                 msgThroughputIn, msgThroughputOut, msgRateIn, msgRateOut,
                 bundleCount,
-                maxResourceUsage * 100, weightedMaxEMA * 100,
+                maxResourceUsage * 100, weightedMaxEMA * 100, msgThroughputEMA,
                 updatedAt, reportedAt
         );
     }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java
index d49361741bec4..2f5c32197c1fd 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java
@@ -86,7 +86,7 @@ public void update(Map bundleStats, int topk) {
             topk = Math.min(topk, arr.size());
             partitionSort(arr, topk);
 
-            for (int i = 0; i < topk; i++) {
+            for (int i = topk - 1; i >= 0; i--) {
                 var etr = arr.get(i);
                 topKBundlesLoadData.add(
                         new TopBundlesLoadData.BundleLoadData(etr.getKey(), (NamespaceBundleStats) etr.getValue()));
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadCounter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadCounter.java
index 37483b58b53e1..4a5d41f7576ba 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadCounter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadCounter.java
@@ -22,8 +22,8 @@
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Skip;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Success;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Admin;
-import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Balanced;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.CoolDown;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.HitCount;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBrokers;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBundles;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoLoadData;
@@ -68,7 +68,7 @@ Overloaded, new AtomicLong(),
                         Underloaded, new AtomicLong(),
                         Admin, new AtomicLong()),
                 Skip, Map.of(
-                        Balanced, new AtomicLong(),
+                        HitCount, new AtomicLong(),
                         NoBundles, new AtomicLong(),
                         CoolDown, new AtomicLong(),
                         OutDatedData, new AtomicLong(),
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadDecision.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadDecision.java
index e1087ab6e53ba..1301b0708a5bd 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadDecision.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/UnloadDecision.java
@@ -41,7 +41,7 @@ public enum Label {
     public enum Reason {
         Overloaded,
         Underloaded,
-        Balanced,
+        HitCount,
         NoBundles,
         CoolDown,
         OutDatedData,
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporter.java
index 256e52c4554d1..b07acfda7f77d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporter.java
@@ -18,15 +18,21 @@
  */
 package org.apache.pulsar.broker.loadbalance.extensions.reporter;
 
+import com.google.common.annotations.VisibleForTesting;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.SystemUtils;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.loadbalance.BrokerHostUsage;
+import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData;
+import org.apache.pulsar.broker.loadbalance.extensions.manager.StateChangeListener;
 import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore;
 import org.apache.pulsar.broker.loadbalance.impl.GenericBrokerHostUsageImpl;
 import org.apache.pulsar.broker.loadbalance.impl.LinuxBrokerHostUsageImpl;
@@ -37,7 +43,9 @@
  * The broker load data reporter.
  */
 @Slf4j
-public class BrokerLoadDataReporter implements LoadDataReporter {
+public class BrokerLoadDataReporter implements LoadDataReporter, StateChangeListener {
+
+    private static final long TOMBSTONE_DELAY_IN_MILLIS = 1000 * 10;
 
     private final PulsarService pulsar;
 
@@ -54,6 +62,10 @@ public class BrokerLoadDataReporter implements LoadDataReporter
 
     private final BrokerLoadData lastData;
 
+    private volatile long lastTombstonedAt;
+
+    private long tombstoneDelayInMillis;
+
     public BrokerLoadDataReporter(PulsarService pulsar,
                                   String lookupServiceAddress,
                                   LoadDataStore brokerLoadDataStore) {
@@ -68,6 +80,7 @@ public BrokerLoadDataReporter(PulsarService pulsar,
         }
         this.localData = new BrokerLoadData();
         this.lastData = new BrokerLoadData();
+        this.tombstoneDelayInMillis = TOMBSTONE_DELAY_IN_MILLIS;
 
     }
 
@@ -92,8 +105,11 @@ public BrokerLoadData generateLoadData() {
     @Override
     public CompletableFuture reportAsync(boolean force) {
         BrokerLoadData newLoadData = this.generateLoadData();
+        boolean debug = ExtensibleLoadManagerImpl.debug(conf, log);
         if (force || needBrokerDataUpdate()) {
-            log.info("publishing load report:{}", localData.toString(conf));
+            if (debug) {
+                log.info("publishing load report:{}", localData.toString(conf));
+            }
             CompletableFuture future =
                     this.brokerLoadDataStore.pushAsync(this.lookupServiceAddress, newLoadData);
             future.whenComplete((__, ex) -> {
@@ -106,7 +122,9 @@ public CompletableFuture reportAsync(boolean force) {
             });
             return future;
         } else {
-            log.info("skipping load report:{}", localData.toString(conf));
+            if (debug) {
+                log.info("skipping load report:{}", localData.toString(conf));
+            }
         }
         return CompletableFuture.completedFuture(null);
     }
@@ -117,10 +135,13 @@ private boolean needBrokerDataUpdate() {
         final long updateMaxIntervalMillis = TimeUnit.MINUTES
                 .toMillis(loadBalancerReportUpdateMaxIntervalMinutes);
         long timeSinceLastReportWrittenToStore = System.currentTimeMillis() - localData.getReportedAt();
+        boolean debug = ExtensibleLoadManagerImpl.debug(conf, log);
         if (timeSinceLastReportWrittenToStore > updateMaxIntervalMillis) {
-            log.info("Writing local data to metadata store because time since last"
-                            + " update exceeded threshold of {} minutes",
-                    loadBalancerReportUpdateMaxIntervalMinutes);
+            if (debug) {
+                log.info("Writing local data to metadata store because time since last"
+                                + " update exceeded threshold of {} minutes",
+                        loadBalancerReportUpdateMaxIntervalMinutes);
+            }
             // Always update after surpassing the maximum interval.
             return true;
         }
@@ -133,10 +154,13 @@ private boolean needBrokerDataUpdate() {
                                         localData.getMsgThroughputIn() + localData.getMsgThroughputOut()),
                                 percentChange(lastData.getBundleCount(), localData.getBundleCount()))));
         if (maxChange > loadBalancerReportUpdateThresholdPercentage) {
-            log.info("Writing local data to metadata store because maximum change {}% exceeded threshold {}%; "
-                            + "time since last report written is {} seconds", maxChange,
-                    loadBalancerReportUpdateThresholdPercentage,
-                    timeSinceLastReportWrittenToStore / 1000.0);
+            if (debug) {
+                log.info(String.format("Writing local data to metadata store "
+                                + "because maximum change %.2f%% exceeded threshold %d%%. "
+                                + "Time since last report written is %.2f%% seconds", maxChange,
+                        loadBalancerReportUpdateThresholdPercentage,
+                        timeSinceLastReportWrittenToStore / 1000.0));
+            }
             return true;
         }
         return false;
@@ -152,4 +176,50 @@ protected double percentChange(final double oldValue, final double newValue) {
         }
         return 100 * Math.abs((oldValue - newValue) / oldValue);
     }
+
+    @VisibleForTesting
+    protected void tombstone() {
+        var now = System.currentTimeMillis();
+        if (now - lastTombstonedAt < tombstoneDelayInMillis) {
+            return;
+        }
+        var lastSuccessfulTombstonedAt = lastTombstonedAt;
+        lastTombstonedAt = now; // dedup first
+        brokerLoadDataStore.removeAsync(lookupServiceAddress)
+                .whenComplete((__, e) -> {
+                            if (e != null) {
+                                log.error("Failed to clean broker load data.", e);
+                                lastTombstonedAt = lastSuccessfulTombstonedAt;
+                            } else {
+                                boolean debug = ExtensibleLoadManagerImpl.debug(conf, log);
+                                if (debug) {
+                                    log.info("Cleaned broker load data.");
+                                }
+                            }
+                        }
+                );
+
+    }
+
+    @Override
+    public void handleEvent(String serviceUnit, ServiceUnitStateData data, Throwable t) {
+        if (t != null) {
+            return;
+        }
+        ServiceUnitState state = ServiceUnitStateData.state(data);
+        switch (state) {
+            case Releasing, Splitting -> {
+                if (StringUtils.equals(data.sourceBroker(), lookupServiceAddress)) {
+                    localData.clear();
+                    tombstone();
+                }
+            }
+            case Owned -> {
+                if (StringUtils.equals(data.dstBroker(), lookupServiceAddress)) {
+                    localData.clear();
+                    tombstone();
+                }
+            }
+        }
+    }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java
index cc1a39add5d4e..0fa37d3687c20 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporter.java
@@ -18,10 +18,16 @@
  */
 package org.apache.pulsar.broker.loadbalance.extensions.reporter;
 
+import com.google.common.annotations.VisibleForTesting;
 import java.util.concurrent.CompletableFuture;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData;
+import org.apache.pulsar.broker.loadbalance.extensions.manager.StateChangeListener;
 import org.apache.pulsar.broker.loadbalance.extensions.models.TopKBundles;
 import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore;
 
@@ -29,7 +35,9 @@
  * The top k highest-loaded bundles' load data reporter.
  */
 @Slf4j
-public class TopBundleLoadDataReporter implements LoadDataReporter {
+public class TopBundleLoadDataReporter implements LoadDataReporter, StateChangeListener {
+
+    private static final long TOMBSTONE_DELAY_IN_MILLIS = 1000 * 10;
 
     private final PulsarService pulsar;
 
@@ -41,6 +49,9 @@ public class TopBundleLoadDataReporter implements LoadDataReporter bundleLoadDataStore) {
@@ -49,6 +60,7 @@ public TopBundleLoadDataReporter(PulsarService pulsar,
         this.bundleLoadDataStore = bundleLoadDataStore;
         this.lastBundleStatsUpdatedAt = 0;
         this.topKBundles = new TopKBundles(pulsar);
+        this.tombstoneDelayInMillis = TOMBSTONE_DELAY_IN_MILLIS;
     }
 
     @Override
@@ -60,8 +72,7 @@ public TopBundlesLoadData generateLoadData() {
             var pulsarStatsUpdatedAt = pulsarStats.getUpdatedAt();
             if (pulsarStatsUpdatedAt > lastBundleStatsUpdatedAt) {
                 var bundleStats = pulsar.getBrokerService().getBundleStats();
-                double percentage = pulsar.getConfiguration().getLoadBalancerBundleLoadReportPercentage();
-                int topk = Math.max(1, (int) (bundleStats.size() * percentage / 100.0));
+                int topk = pulsar.getConfiguration().getLoadBalancerMaxNumberOfBundlesInBundleLoadReport();
                 topKBundles.update(bundleStats, topk);
                 lastBundleStatsUpdatedAt = pulsarStatsUpdatedAt;
                 result = topKBundles.getLoadData();
@@ -74,6 +85,9 @@ public TopBundlesLoadData generateLoadData() {
     public CompletableFuture reportAsync(boolean force) {
         var topBundlesLoadData = generateLoadData();
         if (topBundlesLoadData != null || force) {
+            if (ExtensibleLoadManagerImpl.debug(pulsar.getConfiguration(), log)) {
+                log.info("Reporting TopBundlesLoadData:{}", topKBundles.getLoadData());
+            }
             return this.bundleLoadDataStore.pushAsync(lookupServiceAddress, topKBundles.getLoadData())
                     .exceptionally(e -> {
                         log.error("Failed to report top-bundles load data.", e);
@@ -83,4 +97,47 @@ public CompletableFuture reportAsync(boolean force) {
             return CompletableFuture.completedFuture(null);
         }
     }
+
+    @VisibleForTesting
+    protected void tombstone() {
+        var now = System.currentTimeMillis();
+        if (now - lastTombstonedAt < tombstoneDelayInMillis) {
+            return;
+        }
+        var lastSuccessfulTombstonedAt = lastTombstonedAt;
+        lastTombstonedAt = now; // dedup first
+        bundleLoadDataStore.removeAsync(lookupServiceAddress)
+                .whenComplete((__, e) -> {
+                            if (e != null) {
+                                log.error("Failed to clean broker load data.", e);
+                                lastTombstonedAt = lastSuccessfulTombstonedAt;
+                            } else {
+                                boolean debug = ExtensibleLoadManagerImpl.debug(pulsar.getConfiguration(), log);
+                                if (debug) {
+                                    log.info("Cleaned broker load data.");
+                                }
+                            }
+                        }
+                );
+    }
+
+    @Override
+    public void handleEvent(String serviceUnit, ServiceUnitStateData data, Throwable t) {
+        if (t != null) {
+            return;
+        }
+        ServiceUnitState state = ServiceUnitStateData.state(data);
+        switch (state) {
+            case Releasing, Splitting -> {
+                if (StringUtils.equals(data.sourceBroker(), lookupServiceAddress)) {
+                    tombstone();
+                }
+            }
+            case Owned -> {
+                if (StringUtils.equals(data.dstBroker(), lookupServiceAddress)) {
+                    tombstone();
+                }
+            }
+        }
+    }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/NamespaceUnloadStrategy.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/NamespaceUnloadStrategy.java
index 42af396abcad4..421e6240c8f0d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/NamespaceUnloadStrategy.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/NamespaceUnloadStrategy.java
@@ -20,6 +20,7 @@
 
 import java.util.Map;
 import java.util.Set;
+import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision;
 
@@ -43,4 +44,11 @@ Set findBundlesForUnloading(LoadManagerContext context,
                                                 Map recentlyUnloadedBundles,
                                                 Map recentlyUnloadedBrokers);
 
+    /**
+     * Initializes the internals.
+     *
+     * @param pulsar The pulsar service instance.
+     */
+    void initialize(PulsarService pulsar);
+
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitScheduler.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitScheduler.java
index 589df80fc5c14..816fde0038a58 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitScheduler.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/SplitScheduler.java
@@ -22,6 +22,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
+import java.util.StringJoiner;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
@@ -30,6 +31,7 @@
 import lombok.extern.slf4j.Slf4j;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
 import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel;
 import org.apache.pulsar.broker.loadbalance.extensions.manager.SplitManager;
@@ -98,7 +100,7 @@ public SplitScheduler(PulsarService pulsar,
 
     @Override
     public void execute() {
-        boolean debugMode = conf.isLoadBalancerDebugModeEnabled() || log.isDebugEnabled();
+        boolean debugMode = ExtensibleLoadManagerImpl.debug(conf, log);
         if (debugMode) {
             log.info("Load balancer enabled: {}, Split enabled: {}.",
                     conf.isLoadBalancerEnabled(), conf.isLoadBalancerAutoBundleSplitEnabled());
@@ -113,8 +115,10 @@ public void execute() {
 
         synchronized (bundleSplitStrategy) {
             final Set decisions = bundleSplitStrategy.findBundlesToSplit(context, pulsar);
+            if (debugMode) {
+                log.info("Split Decisions:", decisions);
+            }
             if (!decisions.isEmpty()) {
-
                 // currently following the unloading timeout
                 var asyncOpTimeoutMs = conf.getNamespaceBundleUnloadingTimeoutMs();
                 List> futures = new ArrayList<>();
@@ -156,6 +160,14 @@ public void start() {
         task = loadManagerExecutor.scheduleAtFixedRate(() -> {
             try {
                 execute();
+                var debugMode = ExtensibleLoadManagerImpl.debug(conf, log);
+                if (debugMode) {
+                    StringJoiner joiner = new StringJoiner("\n");
+                    joiner.add("### OwnershipEntrySet start ###");
+                    serviceUnitStateChannel.getOwnershipEntrySet().forEach(e -> joiner.add(e.toString()));
+                    joiner.add("### OwnershipEntrySet end ###");
+                    log.info(joiner.toString());
+                }
             } catch (Throwable e) {
                 log.error("Failed to run the split job.", e);
             }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java
index effddaf9c4159..669219daba273 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java
@@ -20,8 +20,8 @@
 
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Failure;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Skip;
-import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Balanced;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.CoolDown;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.HitCount;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBrokers;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBundles;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoLoadData;
@@ -30,19 +30,26 @@
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Underloaded;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Unknown;
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.MinMaxPriorityQueue;
+import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
 import lombok.Getter;
+import lombok.NoArgsConstructor;
 import lombok.experimental.Accessors;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData;
@@ -80,17 +87,23 @@
  * (loadBalancerMaxNumberOfBrokerTransfersPerCycle).
  * 9. Print more logs with a debug option(loadBalancerDebugModeEnabled=true).
  */
+@NoArgsConstructor
 public class TransferShedder implements NamespaceUnloadStrategy {
     private static final Logger log = LoggerFactory.getLogger(TransferShedder.class);
     private static final double KB = 1024;
+    private static final String CANNOT_CONTINUE_UNLOAD_MSG = "Can't continue the unload cycle.";
+    private static final String CANNOT_UNLOAD_BROKER_MSG = "Can't unload broker:%s.";
+    private static final String CANNOT_UNLOAD_BUNDLE_MSG = "Can't unload bundle:%s.";
     private final LoadStats stats = new LoadStats();
-    private final PulsarService pulsar;
-    private final SimpleResourceAllocationPolicies allocationPolicies;
-    private final IsolationPoliciesHelper isolationPoliciesHelper;
-    private final AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper;
-
-    private final Set decisionCache;
-    private final UnloadCounter counter;
+    private PulsarService pulsar;
+    private SimpleResourceAllocationPolicies allocationPolicies;
+    private IsolationPoliciesHelper isolationPoliciesHelper;
+    private AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper;
+    private Set decisionCache;
+    @Getter
+    private UnloadCounter counter;
+    private ServiceUnitStateChannel channel;
+    private int unloadConditionHitCount = 0;
 
     @VisibleForTesting
     public TransferShedder(UnloadCounter counter){
@@ -110,9 +123,22 @@ public TransferShedder(PulsarService pulsar,
         this.allocationPolicies = new SimpleResourceAllocationPolicies(pulsar);
         this.counter = counter;
         this.isolationPoliciesHelper = new IsolationPoliciesHelper(allocationPolicies);
+        this.channel = ServiceUnitStateChannelImpl.get(pulsar);
         this.antiAffinityGroupPolicyHelper = antiAffinityGroupPolicyHelper;
     }
 
+    @Override
+    public void initialize(PulsarService pulsar){
+        this.pulsar = pulsar;
+        this.decisionCache = new HashSet<>();
+        this.allocationPolicies = new SimpleResourceAllocationPolicies(pulsar);
+        var manager = ExtensibleLoadManagerImpl.get(pulsar.getLoadManager().get());
+        this.counter = manager.getUnloadCounter();
+        this.isolationPoliciesHelper = new IsolationPoliciesHelper(allocationPolicies);
+        this.channel = ServiceUnitStateChannelImpl.get(pulsar);
+        this.antiAffinityGroupPolicyHelper = manager.getAntiAffinityGroupPolicyHelper();
+    }
+
 
     @Getter
     @Accessors(fluent = true)
@@ -122,17 +148,14 @@ static class LoadStats {
         private int totalBrokers;
         private double avg;
         private double std;
-        private MinMaxPriorityQueue minBrokers;
-        private MinMaxPriorityQueue maxBrokers;
         private LoadDataStore loadDataStore;
-
+        private List> brokersSortedByLoad;
+        int maxBrokerIndex;
+        int minBrokerIndex;
+        int numberOfBrokerSheddingPerCycle;
+        int maxNumberOfBrokerSheddingPerCycle;
         LoadStats() {
-            this.minBrokers = MinMaxPriorityQueue.orderedBy((a, b) -> Double.compare(
-                    loadDataStore.get((String) b).get().getWeightedMaxEMA(),
-                    loadDataStore.get((String) a).get().getWeightedMaxEMA())).create();
-            this.maxBrokers = MinMaxPriorityQueue.orderedBy((a, b) -> Double.compare(
-                    loadDataStore.get((String) a).get().getWeightedMaxEMA(),
-                    loadDataStore.get((String) b).get().getWeightedMaxEMA())).create();
+            brokersSortedByLoad = new ArrayList<>();
         }
 
         private void update(double sum, double sqSum, int totalBrokers) {
@@ -143,8 +166,7 @@ private void update(double sum, double sqSum, int totalBrokers) {
             if (totalBrokers == 0) {
                 this.avg = 0;
                 this.std = 0;
-                minBrokers.clear();
-                maxBrokers.clear();
+
             } else {
                 this.avg = sum / totalBrokers;
                 this.std = Math.sqrt(sqSum / totalBrokers - avg * avg);
@@ -156,34 +178,42 @@ void offload(double max, double min, double offload) {
             double maxd = Math.max(0, max - offload);
             double mind = min + offload;
             sqSum += maxd * maxd + mind * mind;
-            std = Math.sqrt(sqSum / totalBrokers - avg * avg);
+            std = Math.sqrt(Math.abs(sqSum / totalBrokers - avg * avg));
+            numberOfBrokerSheddingPerCycle++;
+            minBrokerIndex++;
         }
 
-        void clear(){
+        void clear() {
             sum = 0.0;
             sqSum = 0.0;
             totalBrokers = 0;
             avg = 0.0;
             std = 0.0;
-            minBrokers.clear();
-            maxBrokers.clear();
+            maxBrokerIndex = 0;
+            minBrokerIndex = 0;
+            numberOfBrokerSheddingPerCycle = 0;
+            maxNumberOfBrokerSheddingPerCycle = 0;
+            brokersSortedByLoad.clear();
+            loadDataStore = null;
         }
 
         Optional update(final LoadDataStore loadStore,
+                                               final Map availableBrokers,
                                                Map recentlyUnloadedBrokers,
                                                final ServiceConfiguration conf) {
 
-
+            maxNumberOfBrokerSheddingPerCycle = conf.getLoadBalancerMaxNumberOfBrokerSheddingPerCycle();
+            var debug = ExtensibleLoadManagerImpl.debug(conf, log);
             UnloadDecision.Reason decisionReason = null;
             double sum = 0.0;
             double sqSum = 0.0;
             int totalBrokers = 0;
-            int maxTransfers = conf.getLoadBalancerMaxNumberOfBrokerTransfersPerCycle();
             long now = System.currentTimeMillis();
+            var missingLoadDataBrokers = new HashSet<>(availableBrokers.keySet());
             for (Map.Entry entry : loadStore.entrySet()) {
                 BrokerLoadData localBrokerData = entry.getValue();
                 String broker = entry.getKey();
-
+                missingLoadDataBrokers.remove(broker);
                 // We don't want to use the outdated load data.
                 if (now - localBrokerData.getUpdatedAt()
                         > conf.getLoadBalancerBrokerLoadDataTTLInSeconds() * 1000) {
@@ -196,12 +226,16 @@ Optional update(final LoadDataStore loadS
 
                 // Also, we should give enough time for each broker to recompute its load after transfers.
                 if (recentlyUnloadedBrokers.containsKey(broker)) {
-                    if (localBrokerData.getUpdatedAt() - recentlyUnloadedBrokers.get(broker)
-                            < conf.getLoadBalanceUnloadDelayInSeconds() * 1000) {
-                        log.warn(
-                                "Broker:{} load data timestamp:{} is too early since "
-                                        + "the last transfer timestamp:{}. Stop unloading.",
-                                broker, localBrokerData.getUpdatedAt(), recentlyUnloadedBrokers.get(broker));
+                    var elapsed = localBrokerData.getUpdatedAt() - recentlyUnloadedBrokers.get(broker);
+                    if (elapsed < conf.getLoadBalanceSheddingDelayInSeconds() * 1000) {
+                        if (debug) {
+                            log.warn(
+                                    "Broker:{} load data is too early since "
+                                            + "the last transfer. elapsed {} secs < threshold {} secs",
+                                    broker,
+                                    TimeUnit.MILLISECONDS.toSeconds(elapsed),
+                                    conf.getLoadBalanceSheddingDelayInSeconds());
+                        }
                         update(0.0, 0.0, 0);
                         return Optional.of(CoolDown);
                     } else {
@@ -211,25 +245,28 @@ Optional update(final LoadDataStore loadS
 
                 double load = localBrokerData.getWeightedMaxEMA();
 
-                minBrokers.offer(broker);
-                if (minBrokers.size() > maxTransfers) {
-                    minBrokers.poll();
-                }
-                maxBrokers.offer(broker);
-                if (maxBrokers.size() > maxTransfers) {
-                    maxBrokers.poll();
-                }
                 sum += load;
                 sqSum += load * load;
                 totalBrokers++;
             }
 
-
             if (totalBrokers == 0) {
                 if (decisionReason == null) {
                     decisionReason = NoBrokers;
                 }
                 update(0.0, 0.0, 0);
+                if (debug) {
+                    log.info("There is no broker load data.");
+                }
+                return Optional.of(decisionReason);
+            }
+
+            if (!missingLoadDataBrokers.isEmpty()) {
+                decisionReason = NoLoadData;
+                update(0.0, 0.0, 0);
+                if (debug) {
+                    log.info("There is missing load data from brokers:{}", missingLoadDataBrokers);
+                }
                 return Optional.of(decisionReason);
             }
 
@@ -237,21 +274,35 @@ Optional update(final LoadDataStore loadS
             return Optional.empty();
         }
 
-        boolean hasTransferableBrokers() {
-            return !(maxBrokers.isEmpty() || minBrokers.isEmpty()
-                    || maxBrokers.peekLast().equals(minBrokers().peekLast()));
-        }
-
         void setLoadDataStore(LoadDataStore loadDataStore) {
             this.loadDataStore = loadDataStore;
+            brokersSortedByLoad.addAll(loadDataStore.entrySet());
+            brokersSortedByLoad.sort(Comparator.comparingDouble(
+                    a -> a.getValue().getWeightedMaxEMA()));
+            maxBrokerIndex = brokersSortedByLoad.size() - 1;
+            minBrokerIndex = 0;
+        }
+
+        String peekMinBroker() {
+            return brokersSortedByLoad.get(minBrokerIndex).getKey();
+        }
+
+        String pollMaxBroker() {
+            return brokersSortedByLoad.get(maxBrokerIndex--).getKey();
         }
 
         @Override
         public String toString() {
             return String.format(
-                    "sum:%.2f, sqSum:%.2f, avg:%.2f, std:%.2f, totalBrokers:%d, "
-                            + "minBrokers:%s, maxBrokers:%s",
-                    sum, sqSum, avg, std, totalBrokers, minBrokers, maxBrokers);
+                    "sum:%.2f, sqSum:%.2f, avg:%.2f, std:%.2f, totalBrokers:%d, brokersSortedByLoad:%s",
+                    sum, sqSum, avg, std, totalBrokers,
+                    brokersSortedByLoad.stream().map(v->v.getKey()).collect(Collectors.toList()));
+        }
+
+
+        boolean hasTransferableBrokers() {
+            return numberOfBrokerSheddingPerCycle < maxNumberOfBrokerSheddingPerCycle
+                    && minBrokerIndex < maxBrokerIndex;
         }
     }
 
@@ -263,20 +314,36 @@ public Set findBundlesForUnloading(LoadManagerContext context,
         final var conf = context.brokerConfiguration();
         decisionCache.clear();
         stats.clear();
+        Map availableBrokers;
+        try {
+            availableBrokers = context.brokerRegistry().getAvailableBrokerLookupDataAsync()
+                    .get(context.brokerConfiguration().getMetadataStoreOperationTimeoutSeconds(), TimeUnit.SECONDS);
+        } catch (ExecutionException | InterruptedException | TimeoutException e) {
+            counter.update(Failure, Unknown);
+            log.warn("Failed to fetch available brokers. Stop unloading.", e);
+            return decisionCache;
+        }
 
         try {
             final var loadStore = context.brokerLoadDataStore();
             stats.setLoadDataStore(loadStore);
-            boolean debugMode = conf.isLoadBalancerDebugModeEnabled() || log.isDebugEnabled();
+            boolean debugMode = ExtensibleLoadManagerImpl.debug(conf, log);
 
-            var skipReason = stats.update(context.brokerLoadDataStore(), recentlyUnloadedBrokers, conf);
+            var skipReason = stats.update(
+                    context.brokerLoadDataStore(), availableBrokers, recentlyUnloadedBrokers, conf);
             if (skipReason.isPresent()) {
-                log.warn("Failed to update load stat. Reason:{}. Stop unloading.", skipReason.get());
+                if (debugMode) {
+                    log.warn(CANNOT_CONTINUE_UNLOAD_MSG
+                                    + " Skipped the load stat update. Reason:{}.",
+                            skipReason.get());
+                }
                 counter.update(Skip, skipReason.get());
                 return decisionCache;
             }
             counter.updateLoadData(stats.avg, stats.std);
 
+
+
             if (debugMode) {
                 log.info("brokers' load stats:{}", stats);
             }
@@ -287,138 +354,282 @@ public Set findBundlesForUnloading(LoadManagerContext context,
 
             final double targetStd = conf.getLoadBalancerBrokerLoadTargetStd();
             boolean transfer = conf.isLoadBalancerTransferEnabled();
+            if (stats.std() > targetStd || isUnderLoaded(context, stats.peekMinBroker(), stats.avg)) {
+                unloadConditionHitCount++;
+            } else {
+                unloadConditionHitCount = 0;
+            }
 
-            Map availableBrokers;
-            try {
-                availableBrokers = context.brokerRegistry().getAvailableBrokerLookupDataAsync()
-                        .get(context.brokerConfiguration().getMetadataStoreOperationTimeoutSeconds(), TimeUnit.SECONDS);
-            } catch (ExecutionException | InterruptedException | TimeoutException e) {
-                counter.update(Skip, Unknown);
-                log.warn("Failed to fetch available brokers. Reason: Unknown. Stop unloading.", e);
+            if (unloadConditionHitCount <= conf.getLoadBalancerSheddingConditionHitCountThreshold()) {
+                if (debugMode) {
+                    log.info(CANNOT_CONTINUE_UNLOAD_MSG
+                                    + " Shedding condition hit count:{} is less than or equal to the threshold:{}.",
+                            unloadConditionHitCount, conf.getLoadBalancerSheddingConditionHitCountThreshold());
+                }
+                counter.update(Skip, HitCount);
                 return decisionCache;
             }
 
             while (true) {
                 if (!stats.hasTransferableBrokers()) {
                     if (debugMode) {
-                        log.info("Exhausted target transfer brokers. Stop unloading");
+                        log.info(CANNOT_CONTINUE_UNLOAD_MSG
+                                + " Exhausted target transfer brokers.");
                     }
                     break;
                 }
                 UnloadDecision.Reason reason;
                 if (stats.std() <= targetStd) {
-                    if (hasMsgThroughput(context, stats.minBrokers.peekLast())) {
+                    if (!isUnderLoaded(context, stats.peekMinBroker(), stats.avg)) {
                         if (debugMode) {
-                            log.info("std:{} <= targetStd:{} and minBroker:{} has msg throughput. Stop unloading.",
-                                    stats.std, targetStd, stats.minBrokers.peekLast());
+                            log.info(CANNOT_CONTINUE_UNLOAD_MSG
+                                            + "The overall cluster load meets the target, std:{} <= targetStd:{},"
+                                            + " and minBroker:{} is not underloaded.",
+                                    stats.std(), targetStd, stats.peekMinBroker());
                         }
                         break;
                     } else {
                         reason = Underloaded;
+                        if (debugMode) {
+                            log.info(String.format("broker:%s is underloaded:%s although "
+                                            + "load std:%.2f <= targetStd:%.2f. "
+                                            + "Continuing unload for this underloaded broker.",
+                                    stats.peekMinBroker(),
+                                    context.brokerLoadDataStore().get(stats.peekMinBroker()).get(),
+                                    stats.std(), targetStd));
+                        }
                     }
                 } else {
                     reason = Overloaded;
                 }
 
-                String maxBroker = stats.maxBrokers().pollLast();
-                String minBroker = stats.minBrokers().pollLast();
+                String maxBroker = stats.pollMaxBroker();
+                String minBroker = stats.peekMinBroker();
                 Optional maxBrokerLoadData = context.brokerLoadDataStore().get(maxBroker);
                 Optional minBrokerLoadData = context.brokerLoadDataStore().get(minBroker);
                 if (maxBrokerLoadData.isEmpty()) {
-                    log.error("maxBroker:{} maxBrokerLoadData is empty. Skip unloading from this max broker.",
-                            maxBroker);
+                    log.error(String.format(CANNOT_UNLOAD_BROKER_MSG
+                            + " MaxBrokerLoadData is empty.", maxBroker));
                     numOfBrokersWithEmptyLoadData++;
                     continue;
                 }
                 if (minBrokerLoadData.isEmpty()) {
-                    log.error("minBroker:{} minBrokerLoadData is empty. Skip unloading to this min broker.", minBroker);
+                    log.error("Can't transfer load to broker:{}. MinBrokerLoadData is empty.", minBroker);
                     numOfBrokersWithEmptyLoadData++;
                     continue;
                 }
-
-                double max = maxBrokerLoadData.get().getWeightedMaxEMA();
-                double min = minBrokerLoadData.get().getWeightedMaxEMA();
-                double offload = (max - min) / 2;
+                double maxLoad = maxBrokerLoadData.get().getWeightedMaxEMA();
+                double minLoad = minBrokerLoadData.get().getWeightedMaxEMA();
+                double offload = (maxLoad - minLoad) / 2;
                 BrokerLoadData brokerLoadData = maxBrokerLoadData.get();
-                double brokerThroughput = brokerLoadData.getMsgThroughputIn() + brokerLoadData.getMsgThroughputOut();
-                double offloadThroughput = brokerThroughput * offload;
+                double maxBrokerThroughput = brokerLoadData.getMsgThroughputIn()
+                        + brokerLoadData.getMsgThroughputOut();
+                double minBrokerThroughput = minBrokerLoadData.get().getMsgThroughputIn()
+                        + minBrokerLoadData.get().getMsgThroughputOut();
+                double offloadThroughput = maxBrokerThroughput * offload / maxLoad;
 
                 if (debugMode) {
-                    log.info(
-                            "Attempting to shed load from broker:{}{}, which has the max resource "
-                                    + "usage {}%, targetStd:{},"
-                                    + " -- Offloading {}%, at least {} KByte/s of traffic, left throughput {} KByte/s",
+                    log.info(String.format(
+                            "Attempting to shed load from broker:%s%s, which has the max resource "
+                                    + "usage:%.2f%%, targetStd:%.2f,"
+                                    + " -- Trying to offload %.2f%%, %.2f KByte/s of traffic.",
                             maxBroker, transfer ? " to broker:" + minBroker : "",
-                            100 * max, targetStd,
-                            offload * 100, offloadThroughput / KB, (brokerThroughput - offloadThroughput) / KB);
+                            maxLoad * 100,
+                            targetStd,
+                            offload * 100,
+                            offloadThroughput / KB
+                    ));
                 }
 
                 double trafficMarkedToOffload = 0;
-                boolean atLeastOneBundleSelected = false;
+                double trafficMarkedToGain = 0;
 
                 Optional bundlesLoadData = context.topBundleLoadDataStore().get(maxBroker);
                 if (bundlesLoadData.isEmpty() || bundlesLoadData.get().getTopBundlesLoadData().isEmpty()) {
-                    log.error("maxBroker:{} topBundlesLoadData is empty. Skip unloading from this broker.", maxBroker);
+                    log.error(String.format(CANNOT_UNLOAD_BROKER_MSG
+                            + " TopBundlesLoadData is empty.", maxBroker));
                     numOfBrokersWithEmptyLoadData++;
                     continue;
                 }
 
-                var topBundlesLoadData = bundlesLoadData.get().getTopBundlesLoadData();
-                if (topBundlesLoadData.size() > 1) {
-                    int remainingTopBundles = topBundlesLoadData.size();
-                    for (var e : topBundlesLoadData) {
-                        String bundle = e.bundleName();
-                        if (!recentlyUnloadedBundles.containsKey(bundle)
-                                && isTransferable(context, availableBrokers,
-                                bundle, maxBroker, Optional.of(minBroker))) {
-                            var bundleData = e.stats();
-                            double throughput = bundleData.msgThroughputIn + bundleData.msgThroughputOut;
-                            if (remainingTopBundles > 1
-                                    && (trafficMarkedToOffload < offloadThroughput
-                                    || !atLeastOneBundleSelected)) {
-                                Unload unload;
-                                if (transfer) {
-                                    unload = new Unload(maxBroker, bundle, Optional.of(minBroker));
-                                } else {
-                                    unload = new Unload(maxBroker, bundle);
+                var maxBrokerTopBundlesLoadData = bundlesLoadData.get().getTopBundlesLoadData();
+                if (maxBrokerTopBundlesLoadData.size() == 1) {
+                    numOfBrokersWithFewBundles++;
+                    log.warn(String.format(CANNOT_UNLOAD_BROKER_MSG
+                                    + " Sole namespace bundle:%s is overloading the broker. ",
+                            maxBroker, maxBrokerTopBundlesLoadData.iterator().next()));
+                    continue;
+                }
+                Optional minBundlesLoadData = context.topBundleLoadDataStore().get(minBroker);
+                var minBrokerTopBundlesLoadDataIter =
+                        minBundlesLoadData.isPresent() ? minBundlesLoadData.get().getTopBundlesLoadData().iterator() :
+                                null;
+
+
+                if (maxBrokerTopBundlesLoadData.isEmpty()) {
+                    numOfBrokersWithFewBundles++;
+                    log.warn(String.format(CANNOT_UNLOAD_BROKER_MSG
+                            + " Broker overloaded despite having no bundles", maxBroker));
+                    continue;
+                }
+
+                int remainingTopBundles = maxBrokerTopBundlesLoadData.size();
+                for (var e : maxBrokerTopBundlesLoadData) {
+                    String bundle = e.bundleName();
+                    if (channel != null && !channel.isOwner(bundle, maxBroker)) {
+                        if (debugMode) {
+                            log.warn(String.format(CANNOT_UNLOAD_BUNDLE_MSG
+                                    + " MaxBroker:%s is not the owner.", bundle, maxBroker));
+                        }
+                        continue;
+                    }
+                    if (recentlyUnloadedBundles.containsKey(bundle)) {
+                        if (debugMode) {
+                            log.info(String.format(CANNOT_UNLOAD_BUNDLE_MSG
+                                            + " Bundle has been recently unloaded at ts:%d.",
+                                    bundle, recentlyUnloadedBundles.get(bundle)));
+                        }
+                        continue;
+                    }
+                    if (!isTransferable(context, availableBrokers, bundle, maxBroker, Optional.of(minBroker))) {
+                        if (debugMode) {
+                            log.info(String.format(CANNOT_UNLOAD_BUNDLE_MSG
+                                    + " This unload can't meet "
+                                    + "affinity(isolation) or anti-affinity group policies.", bundle));
+                        }
+                        continue;
+                    }
+                    if (remainingTopBundles <= 1) {
+                        if (debugMode) {
+                            log.info(String.format(CANNOT_UNLOAD_BUNDLE_MSG
+                                            + " The remaining bundles in TopBundlesLoadData from the maxBroker:%s is"
+                                            + " less than or equal to 1.",
+                                    bundle, maxBroker));
+                        }
+                        break;
+                    }
+
+                    var bundleData = e.stats();
+                    double maxBrokerBundleThroughput = bundleData.msgThroughputIn + bundleData.msgThroughputOut;
+                    boolean swap = false;
+                    List minToMaxUnloads = new ArrayList<>();
+                    double minBrokerBundleSwapThroughput = 0.0;
+                    if (trafficMarkedToOffload - trafficMarkedToGain + maxBrokerBundleThroughput > offloadThroughput) {
+                        // see if we can swap bundles from min to max broker to balance better.
+                        if (transfer && minBrokerTopBundlesLoadDataIter != null) {
+                            var maxBrokerNewThroughput =
+                                    maxBrokerThroughput - trafficMarkedToOffload + trafficMarkedToGain
+                                            - maxBrokerBundleThroughput;
+                            var minBrokerNewThroughput =
+                                    minBrokerThroughput + trafficMarkedToOffload - trafficMarkedToGain
+                                            + maxBrokerBundleThroughput;
+                            while (minBrokerTopBundlesLoadDataIter.hasNext()) {
+                                var minBrokerBundleData = minBrokerTopBundlesLoadDataIter.next();
+                                if (!isTransferable(context, availableBrokers,
+                                        minBrokerBundleData.bundleName(), minBroker, Optional.of(maxBroker))) {
+                                    continue;
+                                }
+                                var minBrokerBundleThroughput =
+                                        minBrokerBundleData.stats().msgThroughputIn
+                                                + minBrokerBundleData.stats().msgThroughputOut;
+                                var maxBrokerNewThroughputTmp = maxBrokerNewThroughput + minBrokerBundleThroughput;
+                                var minBrokerNewThroughputTmp = minBrokerNewThroughput - minBrokerBundleThroughput;
+                                if (maxBrokerNewThroughputTmp < maxBrokerThroughput
+                                        && minBrokerNewThroughputTmp < maxBrokerThroughput) {
+                                    minToMaxUnloads.add(new Unload(minBroker,
+                                            minBrokerBundleData.bundleName(), Optional.of(maxBroker)));
+                                    maxBrokerNewThroughput = maxBrokerNewThroughputTmp;
+                                    minBrokerNewThroughput = minBrokerNewThroughputTmp;
+                                    minBrokerBundleSwapThroughput += minBrokerBundleThroughput;
+                                    if (minBrokerNewThroughput <= maxBrokerNewThroughput
+                                            && maxBrokerNewThroughput < maxBrokerThroughput * 0.75) {
+                                        swap = true;
+                                        break;
+                                    }
+                                }
+                            }
+                        }
+                        if (!swap) {
+                            if (debugMode) {
+                                log.info(String.format(CANNOT_UNLOAD_BUNDLE_MSG
+                                                + " The traffic to unload:%.2f - gain:%.2f = %.2f KByte/s is "
+                                                + "greater than the target :%.2f KByte/s.",
+                                        bundle,
+                                        (trafficMarkedToOffload + maxBrokerBundleThroughput) / KB,
+                                        trafficMarkedToGain / KB,
+                                        (trafficMarkedToOffload - trafficMarkedToGain + maxBrokerBundleThroughput) / KB,
+                                        offloadThroughput / KB));
+                            }
+                            break;
+                        }
+                    }
+                    Unload unload;
+                    if (transfer) {
+                        if (swap) {
+                            minToMaxUnloads.forEach(minToMaxUnload -> {
+                                if (debugMode) {
+                                    log.info("Decided to gain bundle:{} from min broker:{}",
+                                            minToMaxUnload.serviceUnit(), minToMaxUnload.sourceBroker());
                                 }
                                 var decision = new UnloadDecision();
-                                decision.setUnload(unload);
+                                decision.setUnload(minToMaxUnload);
                                 decision.succeed(reason);
                                 decisionCache.add(decision);
-                                trafficMarkedToOffload += throughput;
-                                atLeastOneBundleSelected = true;
-                                remainingTopBundles--;
+                            });
+                            if (debugMode) {
+                                log.info(String.format(
+                                        "Total traffic %.2f KByte/s to transfer from min broker:%s to max broker:%s.",
+                                        minBrokerBundleSwapThroughput / KB, minBroker, maxBroker));
+                                trafficMarkedToGain += minBrokerBundleSwapThroughput;
                             }
                         }
+                        unload = new Unload(maxBroker, bundle, Optional.of(minBroker));
+                    } else {
+                        unload = new Unload(maxBroker, bundle);
                     }
-                    if (!atLeastOneBundleSelected) {
-                        numOfBrokersWithFewBundles++;
+                    var decision = new UnloadDecision();
+                    decision.setUnload(unload);
+                    decision.succeed(reason);
+                    decisionCache.add(decision);
+                    trafficMarkedToOffload += maxBrokerBundleThroughput;
+                    remainingTopBundles--;
+
+                    if (debugMode) {
+                        log.info(String.format("Decided to unload bundle:%s, throughput:%.2f KByte/s."
+                                        + " The traffic marked to unload:%.2f - gain:%.2f = %.2f KByte/s."
+                                        + " Target:%.2f KByte/s.",
+                                bundle, maxBrokerBundleThroughput / KB,
+                                trafficMarkedToOffload / KB,
+                                trafficMarkedToGain / KB,
+                                (trafficMarkedToOffload - trafficMarkedToGain) / KB,
+                                offloadThroughput / KB));
                     }
-                } else if (topBundlesLoadData.size() == 1) {
-                    numOfBrokersWithFewBundles++;
-                    log.warn(
-                            "HIGH USAGE WARNING : Sole namespace bundle {} is overloading broker {}. "
-                                    + "No Load Shedding will be done on this broker",
-                            topBundlesLoadData.iterator().next(), maxBroker);
-                } else {
-                    numOfBrokersWithFewBundles++;
-                    log.warn("Broker {} is overloaded despite having no bundles", maxBroker);
                 }
-
                 if (trafficMarkedToOffload > 0) {
-                    stats.offload(max, min, offload);
+                    var adjustedOffload =
+                            (trafficMarkedToOffload - trafficMarkedToGain) * maxLoad / maxBrokerThroughput;
+                    stats.offload(maxLoad, minLoad, adjustedOffload);
                     if (debugMode) {
                         log.info(
                                 String.format("brokers' load stats:%s, after offload{max:%.2f, min:%.2f, offload:%.2f}",
-                                        stats, max, min, offload));
+                                        stats, maxLoad, minLoad, adjustedOffload));
                     }
+                } else {
+                    numOfBrokersWithFewBundles++;
+                    log.warn(String.format(CANNOT_UNLOAD_BROKER_MSG
+                            + " There is no bundle that can be unloaded in top bundles load data. "
+                            + "Consider splitting bundles owned by the broker "
+                            + "to make each bundle serve less traffic "
+                            + "or increasing loadBalancerMaxNumberOfBundlesInBundleLoadReport"
+                            + " to report more bundles in the top bundles load data.", maxBroker));
                 }
-            }
+
+            } // while end
 
             if (debugMode) {
                 log.info("decisionCache:{}", decisionCache);
             }
+
             if (decisionCache.isEmpty()) {
                 UnloadDecision.Reason reason;
                 if (numOfBrokersWithEmptyLoadData > 0) {
@@ -426,10 +637,13 @@ && isTransferable(context, availableBrokers,
                 } else if (numOfBrokersWithFewBundles > 0) {
                     reason = NoBundles;
                 } else {
-                    reason = Balanced;
+                    reason = HitCount;
                 }
                 counter.update(Skip, reason);
+            } else {
+                unloadConditionHitCount = 0;
             }
+
         } catch (Throwable e) {
             log.error("Failed to process unloading. ", e);
             this.counter.update(Failure, Unknown);
@@ -438,13 +652,19 @@ && isTransferable(context, availableBrokers,
     }
 
 
-    private boolean hasMsgThroughput(LoadManagerContext context, String broker) {
+    private boolean isUnderLoaded(LoadManagerContext context, String broker, double avgLoad) {
         var brokerLoadDataOptional = context.brokerLoadDataStore().get(broker);
         if (brokerLoadDataOptional.isEmpty()) {
             return false;
         }
         var brokerLoadData = brokerLoadDataOptional.get();
-        return brokerLoadData.getMsgThroughputIn() + brokerLoadData.getMsgThroughputOut() > 0.0;
+        if (brokerLoadData.getMsgThroughputEMA() < 1) {
+            return true;
+        }
+
+        return brokerLoadData.getWeightedMaxEMA()
+                < avgLoad * Math.min(0.5, Math.max(0.0,
+                context.brokerConfiguration().getLoadBalancerBrokerLoadTargetStd() / 2));
     }
 
 
@@ -467,7 +687,8 @@ private boolean isTransferable(LoadManagerContext context,
             return false;
         }
 
-        if (!antiAffinityGroupPolicyHelper.canUnload(availableBrokers, bundle, srcBroker, dstBroker)) {
+        if (antiAffinityGroupPolicyHelper != null
+                && !antiAffinityGroupPolicyHelper.canUnload(availableBrokers, bundle, srcBroker, dstBroker)) {
             return false;
         }
         return true;
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java
index 31310c5c9cc80..d6c754c90fcf6 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadScheduler.java
@@ -40,7 +40,6 @@
 import org.apache.pulsar.broker.loadbalance.extensions.models.Unload;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadCounter;
 import org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision;
-import org.apache.pulsar.broker.loadbalance.extensions.policies.AntiAffinityGroupPolicyHelper;
 import org.apache.pulsar.common.stats.Metrics;
 import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.common.util.Reflections;
@@ -81,11 +80,10 @@ public UnloadScheduler(PulsarService pulsar,
                            UnloadManager unloadManager,
                            LoadManagerContext context,
                            ServiceUnitStateChannel channel,
-                           AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper,
                            UnloadCounter counter,
                            AtomicReference> unloadMetrics) {
         this(pulsar, loadManagerExecutor, unloadManager, context, channel,
-                createNamespaceUnloadStrategy(pulsar, antiAffinityGroupPolicyHelper, counter), counter, unloadMetrics);
+                createNamespaceUnloadStrategy(pulsar), counter, unloadMetrics);
     }
 
     @VisibleForTesting
@@ -212,19 +210,22 @@ public void close() {
         this.recentlyUnloadedBrokers.clear();
     }
 
-    private static NamespaceUnloadStrategy createNamespaceUnloadStrategy(PulsarService pulsar,
-                                                                         AntiAffinityGroupPolicyHelper helper,
-                                                                         UnloadCounter counter) {
+    private static NamespaceUnloadStrategy createNamespaceUnloadStrategy(PulsarService pulsar) {
         ServiceConfiguration conf = pulsar.getConfiguration();
+        NamespaceUnloadStrategy unloadStrategy;
         try {
-            return Reflections.createInstance(conf.getLoadBalancerLoadSheddingStrategy(), NamespaceUnloadStrategy.class,
+            unloadStrategy = Reflections.createInstance(conf.getLoadBalancerLoadSheddingStrategy(),
+                    NamespaceUnloadStrategy.class,
                     Thread.currentThread().getContextClassLoader());
+            log.info("Created namespace unload strategy:{}", unloadStrategy.getClass().getCanonicalName());
         } catch (Exception e) {
             log.error("Error when trying to create namespace unload strategy: {}",
                     conf.getLoadBalancerLoadPlacementStrategy(), e);
+            log.error("create namespace unload strategy failed. using TransferShedder instead.");
+            unloadStrategy = new TransferShedder();
         }
-        log.error("create namespace unload strategy failed. using TransferShedder instead.");
-        return new TransferShedder(pulsar, counter, helper);
+        unloadStrategy.initialize(pulsar);
+        return unloadStrategy;
     }
 
     private boolean isLoadBalancerSheddingEnabled() {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java
index da04a287f1ac5..15bfdc747f1fc 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyImpl.java
@@ -32,6 +32,7 @@
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl;
 import org.apache.pulsar.broker.loadbalance.extensions.models.Split;
 import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter;
 import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision;
@@ -47,15 +48,17 @@
  */
 @Slf4j
 public class DefaultNamespaceBundleSplitStrategyImpl implements NamespaceBundleSplitStrategy {
+    private static final String CANNOT_CONTINUE_SPLIT_MSG = "Can't continue the split cycle.";
+    private static final String CANNOT_SPLIT_BUNDLE_MSG = "Can't split broker:%s.";
     private final Set decisionCache;
     private final Map namespaceBundleCount;
-    private final Map bundleHighTrafficFrequency;
+    private final Map splitConditionHitCounts;
     private final SplitCounter counter;
 
     public DefaultNamespaceBundleSplitStrategyImpl(SplitCounter counter) {
         decisionCache = new HashSet<>();
         namespaceBundleCount = new HashMap<>();
-        bundleHighTrafficFrequency = new HashMap<>();
+        splitConditionHitCounts = new HashMap<>();
         this.counter = counter;
 
     }
@@ -71,22 +74,33 @@ public Set findBundlesToSplit(LoadManagerContext context, PulsarS
         long maxBundleMsgRate = conf.getLoadBalancerNamespaceBundleMaxMsgRate();
         long maxBundleBandwidth = conf.getLoadBalancerNamespaceBundleMaxBandwidthMbytes() * LoadManagerShared.MIBI;
         long maxSplitCount = conf.getLoadBalancerMaxNumberOfBundlesToSplitPerCycle();
-        long splitConditionThreshold = conf.getLoadBalancerNamespaceBundleSplitConditionThreshold();
+        long splitConditionHitCountThreshold = conf.getLoadBalancerNamespaceBundleSplitConditionHitCountThreshold();
         boolean debug = log.isDebugEnabled() || conf.isLoadBalancerDebugModeEnabled();
+        var channel = ServiceUnitStateChannelImpl.get(pulsar);
 
         Map bundleStatsMap = pulsar.getBrokerService().getBundleStats();
         NamespaceBundleFactory namespaceBundleFactory =
                 pulsar.getNamespaceService().getNamespaceBundleFactory();
 
-        // clean bundleHighTrafficFrequency
-        bundleHighTrafficFrequency.keySet().retainAll(bundleStatsMap.keySet());
+        // clean splitConditionHitCounts
+        splitConditionHitCounts.keySet().retainAll(bundleStatsMap.keySet());
 
         for (var entry : bundleStatsMap.entrySet()) {
             final String bundle = entry.getKey();
             final NamespaceBundleStats stats = entry.getValue();
             if (stats.topics < 2) {
                 if (debug) {
-                    log.info("The count of topics on the bundle {} is less than 2, skip split!", bundle);
+                    log.info(String.format(CANNOT_SPLIT_BUNDLE_MSG
+                            + " The topic count is less than 2.", bundle));
+                }
+                continue;
+            }
+
+            if (!channel.isOwner(bundle)) {
+                if (debug) {
+                    log.error(String.format(CANNOT_SPLIT_BUNDLE_MSG
+                            + " This broker is not the owner.", bundle));
+                    counter.update(Failure, Unknown);
                 }
                 continue;
             }
@@ -96,7 +110,8 @@ public Set findBundlesToSplit(LoadManagerContext context, PulsarS
             if (!namespaceBundleFactory
                     .canSplitBundle(namespaceBundleFactory.getBundle(namespaceName, bundleRange))) {
                 if (debug) {
-                    log.info("Can't split the bundle:{}. invalid bundle range:{}. ", bundle, bundleRange);
+                    log.info(String.format(CANNOT_SPLIT_BUNDLE_MSG
+                            + " Invalid bundle range:%s.", bundle, bundleRange));
                 }
                 counter.update(Failure, Unknown);
                 continue;
@@ -117,54 +132,85 @@ public Set findBundlesToSplit(LoadManagerContext context, PulsarS
             }
 
             if (reason != Unknown) {
-                bundleHighTrafficFrequency.put(bundle, bundleHighTrafficFrequency.getOrDefault(bundle, 0) + 1);
+                splitConditionHitCounts.put(bundle, splitConditionHitCounts.getOrDefault(bundle, 0) + 1);
             } else {
-                bundleHighTrafficFrequency.remove(bundle);
+                splitConditionHitCounts.remove(bundle);
+            }
+
+            if (splitConditionHitCounts.getOrDefault(bundle, 0) <= splitConditionHitCountThreshold) {
+                if (debug) {
+                    log.info(String.format(
+                            CANNOT_SPLIT_BUNDLE_MSG
+                                    + " Split condition hit count: %d is"
+                                    + " less than or equal to threshold: %d. "
+                                    + "Topics: %d/%d, "
+                                    + "Sessions: (%d+%d)/%d, "
+                                    + "Message Rate: %.2f/%d (msgs/s), "
+                                    + "Message Throughput: %.2f/%d (MB/s).",
+                            bundle,
+                            splitConditionHitCounts.getOrDefault(bundle, 0),
+                            splitConditionHitCountThreshold,
+                            stats.topics, maxBundleTopics,
+                            stats.producerCount, stats.consumerCount, maxBundleSessions,
+                            totalMessageRate, maxBundleMsgRate,
+                            totalMessageThroughput / LoadManagerShared.MIBI,
+                            maxBundleBandwidth / LoadManagerShared.MIBI
+                    ));
+                }
+                continue;
             }
 
-            if (bundleHighTrafficFrequency.getOrDefault(bundle, 0) > splitConditionThreshold) {
-                final String namespace = LoadManagerShared.getNamespaceNameFromBundleName(bundle);
-                try {
-                    final int bundleCount = pulsar.getNamespaceService()
-                            .getBundleCount(NamespaceName.get(namespace));
-                    if ((bundleCount + namespaceBundleCount.getOrDefault(namespace, 0))
-                            < maxBundleCount) {
-                        if (debug) {
-                            log.info("The bundle {} is considered to split. Topics: {}/{}, Sessions: ({}+{})/{}, "
-                                            + "Message Rate: {}/{} (msgs/s), Message Throughput: {}/{} (MB/s)",
-                                    bundle, stats.topics, maxBundleTopics, stats.producerCount, stats.consumerCount,
-                                    maxBundleSessions, totalMessageRate, maxBundleMsgRate,
-                                    totalMessageThroughput / LoadManagerShared.MIBI,
-                                    maxBundleBandwidth / LoadManagerShared.MIBI);
-                        }
-                        var decision = new SplitDecision();
-                        decision.setSplit(new Split(bundle, context.brokerRegistry().getBrokerId()));
-                        decision.succeed(reason);
-                        decisionCache.add(decision);
-                        int bundleNum = namespaceBundleCount.getOrDefault(namespace, 0);
-                        namespaceBundleCount.put(namespace, bundleNum + 1);
-                        bundleHighTrafficFrequency.remove(bundle);
-                        // Clear namespace bundle-cache
-                        namespaceBundleFactory.invalidateBundleCache(NamespaceName.get(namespaceName));
-                        if (decisionCache.size() == maxSplitCount) {
-                            if (debug) {
-                                log.info("Too many bundles to split in this split cycle {} / {}. Stop.",
-                                        decisionCache.size(), maxSplitCount);
-                            }
-                            break;
-                        }
-                    } else {
-                        if (debug) {
-                            log.info(
-                                    "Could not split namespace bundle {} because namespace {} has too many bundles:"
-                                            + "{}", bundle, namespace, bundleCount);
-                        }
+            final String namespace = LoadManagerShared.getNamespaceNameFromBundleName(bundle);
+            try {
+                final int bundleCount = pulsar.getNamespaceService()
+                        .getBundleCount(NamespaceName.get(namespace));
+                if ((bundleCount + namespaceBundleCount.getOrDefault(namespace, 0))
+                        >= maxBundleCount) {
+                    if (debug) {
+                        log.info(String.format(CANNOT_SPLIT_BUNDLE_MSG + " Namespace:%s has too many bundles:%d",
+                                bundle, namespace, bundleCount));
                     }
-                } catch (Exception e) {
-                    counter.update(Failure, Unknown);
-                    log.warn("Error while computing bundle splits for namespace {}", namespace, e);
+                    continue;
+                }
+            } catch (Exception e) {
+                counter.update(Failure, Unknown);
+                log.warn("Failed to get bundle count in namespace:{}", namespace, e);
+                continue;
+            }
+
+            if (debug) {
+                log.info(String.format(
+                        "Splitting bundle: %s. "
+                                + "Topics: %d/%d, "
+                                + "Sessions: (%d+%d)/%d, "
+                                + "Message Rate: %.2f/%d (msgs/s), "
+                                + "Message Throughput: %.2f/%d (MB/s)",
+                        bundle,
+                        stats.topics, maxBundleTopics,
+                        stats.producerCount, stats.consumerCount, maxBundleSessions,
+                        totalMessageRate, maxBundleMsgRate,
+                        totalMessageThroughput / LoadManagerShared.MIBI,
+                        maxBundleBandwidth / LoadManagerShared.MIBI
+                ));
+            }
+            var decision = new SplitDecision();
+            decision.setSplit(new Split(bundle, context.brokerRegistry().getBrokerId()));
+            decision.succeed(reason);
+            decisionCache.add(decision);
+            int bundleNum = namespaceBundleCount.getOrDefault(namespace, 0);
+            namespaceBundleCount.put(namespace, bundleNum + 1);
+            splitConditionHitCounts.remove(bundle);
+            // Clear namespace bundle-cache
+            namespaceBundleFactory.invalidateBundleCache(NamespaceName.get(namespaceName));
+            if (decisionCache.size() == maxSplitCount) {
+                if (debug) {
+                    log.info(CANNOT_CONTINUE_SPLIT_MSG
+                                    + "Too many bundles split in this cycle {} / {}.",
+                            decisionCache.size(), maxSplitCount);
                 }
+                break;
             }
+
         }
         return decisionCache;
     }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java
index 678927dac9293..902cfdaf73f5f 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java
@@ -82,7 +82,7 @@ public Optional select(
             Set candidates, ServiceUnitId bundleToAssign, LoadManagerContext context) {
         var conf = context.brokerConfiguration();
         if (candidates.isEmpty()) {
-            log.info("There are no available brokers as candidates at this point for bundle: {}", bundleToAssign);
+            log.warn("There are no available brokers as candidates at this point for bundle: {}", bundleToAssign);
             return Optional.empty();
         }
 
@@ -131,8 +131,10 @@ public Optional select(
 
         if (bestBrokers.isEmpty()) {
             // Assign randomly as all brokers are overloaded.
-            log.warn("Assign randomly as none of the brokers are underloaded. candidatesSize:{}, "
-                    + "noLoadDataBrokersSize:{}", candidates.size(), noLoadDataBrokers.size());
+            if (debugMode) {
+                log.info("Assign randomly as none of the brokers are underloaded. candidatesSize:{}, "
+                        + "noLoadDataBrokersSize:{}", candidates.size(), noLoadDataBrokers.size());
+            }
             for (String broker : candidates) {
                 bestBrokers.add(broker);
             }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
index f8a7a9b629f4e..4c57e6b93fa3c 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java
@@ -26,8 +26,8 @@
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Failure;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Skip;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Success;
-import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Balanced;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.CoolDown;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.HitCount;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBrokers;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBundles;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoLoadData;
@@ -671,7 +671,7 @@ public void testGetMetrics() throws Exception {
                         put(Underloaded, new AtomicLong(2));
                     }},
                     Skip, new LinkedHashMap<>() {{
-                        put(Balanced, new AtomicLong(3));
+                        put(HitCount, new AtomicLong(3));
                         put(NoBundles, new AtomicLong(4));
                         put(CoolDown, new AtomicLong(5));
                         put(OutDatedData, new AtomicLong(6));
@@ -756,7 +756,7 @@ SplitDecision.Reason.Unknown, new AtomicLong(6))
                         dimensions=[{broker=localhost, feature=max, metric=loadBalancing}], metrics=[{brk_lb_resource_usage=0.04}]
                         dimensions=[{broker=localhost, metric=bundleUnloading}], metrics=[{brk_lb_unload_broker_total=2, brk_lb_unload_bundle_total=3}]
                         dimensions=[{broker=localhost, metric=bundleUnloading, reason=Unknown, result=Failure}], metrics=[{brk_lb_unload_broker_breakdown_total=10}]
-                        dimensions=[{broker=localhost, metric=bundleUnloading, reason=Balanced, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=3}]
+                        dimensions=[{broker=localhost, metric=bundleUnloading, reason=HitCount, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=3}]
                         dimensions=[{broker=localhost, metric=bundleUnloading, reason=NoBundles, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=4}]
                         dimensions=[{broker=localhost, metric=bundleUnloading, reason=CoolDown, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=5}]
                         dimensions=[{broker=localhost, metric=bundleUnloading, reason=OutDatedData, result=Skip}], metrics=[{brk_lb_unload_broker_breakdown_total=6}]
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
index c05c8ac741565..9eda98e5d842d 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java
@@ -78,6 +78,7 @@
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
 import org.apache.pulsar.broker.loadbalance.extensions.models.Split;
 import org.apache.pulsar.broker.loadbalance.extensions.models.Unload;
+import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore;
 import org.apache.pulsar.broker.loadbalance.extensions.strategy.BrokerSelectionStrategy;
 import org.apache.pulsar.broker.namespace.NamespaceService;
 import org.apache.pulsar.broker.testcontext.PulsarTestContext;
@@ -132,6 +133,8 @@ protected void setup() throws Exception {
         pulsar1 = pulsar;
         registry = new BrokerRegistryImpl(pulsar);
         loadManagerContext = mock(LoadManagerContext.class);
+        doReturn(mock(LoadDataStore.class)).when(loadManagerContext).brokerLoadDataStore();
+        doReturn(mock(LoadDataStore.class)).when(loadManagerContext).topBundleLoadDataStore();
         brokerSelector = mock(BrokerSelectionStrategy.class);
         additionalPulsarTestContext = createAdditionalPulsarTestContext(getDefaultConf());
         pulsar2 = additionalPulsarTestContext.getPulsarService();
@@ -1208,6 +1211,69 @@ public void splitTestWhenDestBrokerFails()
 
     }
 
+    @Test(priority = 15)
+    public void testIsOwner() throws IllegalAccessException {
+
+        var owner1 = channel1.isOwner(bundle);
+        var owner2 = channel2.isOwner(bundle);
+
+        assertFalse(owner1);
+        assertFalse(owner2);
+
+        owner1 = channel1.isOwner(bundle, lookupServiceAddress2);
+        owner2 = channel2.isOwner(bundle, lookupServiceAddress1);
+
+        assertFalse(owner1);
+        assertFalse(owner2);
+
+        channel1.publishAssignEventAsync(bundle, lookupServiceAddress1);
+        owner2 = channel2.isOwner(bundle);
+        assertFalse(owner2);
+
+        waitUntilOwnerChanges(channel1, bundle, null);
+        waitUntilOwnerChanges(channel2, bundle, null);
+
+        owner1 = channel1.isOwner(bundle);
+        owner2 = channel2.isOwner(bundle);
+
+        assertTrue(owner1);
+        assertFalse(owner2);
+
+        owner1 = channel1.isOwner(bundle, lookupServiceAddress1);
+        owner2 = channel2.isOwner(bundle, lookupServiceAddress2);
+
+        assertTrue(owner1);
+        assertFalse(owner2);
+
+        owner1 = channel2.isOwner(bundle, lookupServiceAddress1);
+        owner2 = channel1.isOwner(bundle, lookupServiceAddress2);
+
+        assertTrue(owner1);
+        assertFalse(owner2);
+
+        overrideTableView(channel1, bundle, new ServiceUnitStateData(Assigning, lookupServiceAddress1, 1));
+        assertFalse(channel1.isOwner(bundle));
+
+        overrideTableView(channel1, bundle, new ServiceUnitStateData(Owned, lookupServiceAddress1, 1));
+        assertTrue(channel1.isOwner(bundle));
+
+        overrideTableView(channel1, bundle, new ServiceUnitStateData(Releasing, null, lookupServiceAddress1, 1));
+        assertFalse(channel1.isOwner(bundle));
+
+        overrideTableView(channel1, bundle, new ServiceUnitStateData(Splitting, null, lookupServiceAddress1, 1));
+        assertTrue(channel1.isOwner(bundle));
+
+        overrideTableView(channel1, bundle, new ServiceUnitStateData(Free, null, lookupServiceAddress1, 1));
+        assertFalse(channel1.isOwner(bundle));
+
+        overrideTableView(channel1, bundle, new ServiceUnitStateData(Deleted, null, lookupServiceAddress1, 1));
+        assertFalse(channel1.isOwner(bundle));
+
+        overrideTableView(channel1, bundle, null);
+        assertFalse(channel1.isOwner(bundle));
+    }
+
+
     private static ConcurrentOpenHashMap>> getOwnerRequests(
             ServiceUnitStateChannel channel) throws IllegalAccessException {
         return (ConcurrentOpenHashMap>>)
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadDataTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadDataTest.java
index 5ba7629dd1132..85792a7ba9387 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadDataTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/data/BrokerLoadDataTest.java
@@ -74,6 +74,7 @@ public void testUpdateBySystemResourceUsage() {
         assertEquals(data.getTopics(), 6);
         assertEquals(data.getMaxResourceUsage(), 0.04); // skips memory usage
         assertEquals(data.getWeightedMaxEMA(), 2);
+        assertEquals(data.getMsgThroughputEMA(), 3);
         assertThat(data.getUpdatedAt(), greaterThanOrEqualTo(now));
 
         now = System.currentTimeMillis();
@@ -103,6 +104,7 @@ public void testUpdateBySystemResourceUsage() {
         assertEquals(data.getTopics(), 10);
         assertEquals(data.getMaxResourceUsage(), 3.0);
         assertEquals(data.getWeightedMaxEMA(), 1.875);
+        assertEquals(data.getMsgThroughputEMA(), 5);
         assertThat(data.getUpdatedAt(), greaterThanOrEqualTo(now));
         assertEquals(data.getReportedAt(), 0l);
         assertEquals(data.toString(conf), "cpu= 300.00%, memory= 100.00%, directMemory= 2.00%, "
@@ -111,8 +113,11 @@ public void testUpdateBySystemResourceUsage() {
                 + "bandwithInResourceWeight= 0.500000, bandwithOutResourceWeight= 0.500000, "
                 + "msgThroughputIn= 5.00, msgThroughputOut= 6.00, "
                 + "msgRateIn= 7.00, msgRateOut= 8.00, bundleCount= 9, "
-                + "maxResourceUsage= 300.00%, weightedMaxEMA= 187.50%, "
+                + "maxResourceUsage= 300.00%, weightedMaxEMA= 187.50%, msgThroughputEMA= 5.00, "
                 + "updatedAt= " + data.getUpdatedAt() + ", reportedAt= " + data.getReportedAt());
+
+        data.clear();
+        assertEquals(data, new BrokerLoadData());
     }
 
     @Test
@@ -143,6 +148,9 @@ public void testUpdateByBrokerLoadData() {
         data.update(other);
 
         assertEquals(data, other);
+
+        data.clear();
+        assertEquals(data, new BrokerLoadData());
     }
 
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundlesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundlesTest.java
index 9b42163bd664b..472d44df8906d 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundlesTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundlesTest.java
@@ -87,15 +87,15 @@ public void testTopBundlesLoadData() {
         Map bundleStats = new HashMap<>();
         var topKBundles = new TopKBundles(pulsar);
         NamespaceBundleStats stats1 = new NamespaceBundleStats();
-        stats1.msgRateIn = 500;
+        stats1.msgRateIn = 100000;
         bundleStats.put(bundle1, stats1);
 
         NamespaceBundleStats stats2 = new NamespaceBundleStats();
-        stats2.msgRateIn = 10000;
+        stats2.msgRateIn = 500;
         bundleStats.put(bundle2, stats2);
 
         NamespaceBundleStats stats3 = new NamespaceBundleStats();
-        stats3.msgRateIn = 100000;
+        stats3.msgRateIn = 10000;
         bundleStats.put(bundle3, stats3);
 
         NamespaceBundleStats stats4 = new NamespaceBundleStats();
@@ -107,8 +107,8 @@ public void testTopBundlesLoadData() {
         var top1 = topKBundles.getLoadData().getTopBundlesLoadData().get(1);
         var top2 = topKBundles.getLoadData().getTopBundlesLoadData().get(2);
 
-        assertEquals(top0.bundleName(), bundle3);
-        assertEquals(top1.bundleName(), bundle2);
+        assertEquals(top0.bundleName(), bundle2);
+        assertEquals(top1.bundleName(), bundle3);
         assertEquals(top2.bundleName(), bundle1);
     }
 
@@ -225,8 +225,8 @@ public void testLoadBalancerSheddingBundlesWithPoliciesEnabledConfig() throws Me
         var top0 = topKBundles.getLoadData().getTopBundlesLoadData().get(0);
         var top1 = topKBundles.getLoadData().getTopBundlesLoadData().get(1);
 
-        assertEquals(top0.bundleName(), bundle2);
-        assertEquals(top1.bundleName(), bundle1);
+        assertEquals(top0.bundleName(), bundle1);
+        assertEquals(top1.bundleName(), bundle2);
 
         configuration.setLoadBalancerSheddingBundlesWithPoliciesEnabled(false);
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java
index ee7e708667a32..93ab35981e1c4 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/BrokerLoadDataReporterTest.java
@@ -18,28 +18,37 @@
  */
 package org.apache.pulsar.broker.loadbalance.extensions.reporter;
 
+import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.VERSION_ID_INIT;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.testng.Assert.assertEquals;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
 import org.apache.commons.lang.reflect.FieldUtils;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData;
 import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore;
 import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared;
 import org.apache.pulsar.broker.service.BrokerService;
 import org.apache.pulsar.broker.service.PulsarStats;
 import org.apache.pulsar.broker.stats.BrokerStats;
+import org.apache.pulsar.client.util.ExecutorProvider;
 import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage;
 import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage;
 import org.mockito.MockedStatic;
 import org.mockito.Mockito;
+import org.testcontainers.shaded.org.awaitility.Awaitility;
+import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
@@ -52,17 +61,24 @@ public class BrokerLoadDataReporterTest {
     ServiceConfiguration config;
     BrokerStats brokerStats;
     SystemResourceUsage usage;
+    String broker = "broker1";
+    String bundle = "bundle1";
+    ScheduledExecutorService executor;
 
     @BeforeMethod
     void setup() {
         config = new ServiceConfiguration();
+        config.setLoadBalancerDebugModeEnabled(true);
         pulsar = mock(PulsarService.class);
         store = mock(LoadDataStore.class);
         brokerService = mock(BrokerService.class);
         pulsarStats = mock(PulsarStats.class);
         doReturn(brokerService).when(pulsar).getBrokerService();
         doReturn(config).when(pulsar).getConfiguration();
-        doReturn(Executors.newSingleThreadScheduledExecutor()).when(pulsar).getLoadManagerExecutor();
+        executor = Executors
+                .newSingleThreadScheduledExecutor(new
+                        ExecutorProvider.ExtendedThreadFactory("pulsar-load-manager"));
+        doReturn(executor).when(pulsar).getLoadManagerExecutor();
         doReturn(pulsarStats).when(brokerService).getPulsarStats();
         brokerStats = new BrokerStats(0);
         brokerStats.topics = 6;
@@ -74,6 +90,7 @@ void setup() {
         doReturn(pulsarStats).when(brokerService).getPulsarStats();
         doReturn(brokerStats).when(pulsarStats).getBrokerStats();
         doReturn(CompletableFuture.completedFuture(null)).when(store).pushAsync(any(), any());
+        doReturn(CompletableFuture.completedFuture(null)).when(store).removeAsync(any());
 
         usage = new SystemResourceUsage();
         usage.setCpu(new ResourceUsage(1.0, 100.0));
@@ -83,6 +100,11 @@ void setup() {
         usage.setBandwidthOut(new ResourceUsage(4.0, 100.0));
     }
 
+    @AfterMethod
+    void shutdown(){
+        executor.shutdown();
+    }
+
     public void testGenerate() throws IllegalAccessException {
         try (MockedStatic mockLoadManagerShared = Mockito.mockStatic(LoadManagerShared.class)) {
             mockLoadManagerShared.when(() -> LoadManagerShared.getSystemResourceUsage(any())).thenReturn(usage);
@@ -124,4 +146,73 @@ public void testReport() throws IllegalAccessException {
         }
     }
 
+    @Test
+    public void testTombstone() throws IllegalAccessException, InterruptedException {
+
+        var target = spy(new BrokerLoadDataReporter(pulsar, broker, store));
+
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Assigning, broker, VERSION_ID_INIT), null);
+        verify(store, times(0)).removeAsync(eq(broker));
+        verify(target, times(0)).tombstone();
+
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Deleted, broker, VERSION_ID_INIT), null);
+        verify(store, times(0)).removeAsync(eq(broker));
+        verify(target, times(0)).tombstone();
+
+
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Init, broker, VERSION_ID_INIT), null);
+        verify(store, times(0)).removeAsync(eq(broker));
+        verify(target, times(0)).tombstone();
+
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Free, broker, VERSION_ID_INIT), null);
+        verify(store, times(0)).removeAsync(eq(broker));
+        verify(target, times(0)).tombstone();
+
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Releasing, "broker-2", broker, VERSION_ID_INIT),
+                new RuntimeException());
+        verify(store, times(0)).removeAsync(eq(broker));
+        verify(target, times(0)).tombstone();
+
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Releasing, "broker-2", broker, VERSION_ID_INIT), null);
+        Awaitility.waitAtMost(3, TimeUnit.SECONDS).untilAsserted(() -> {
+            verify(target, times(1)).tombstone();
+            verify(store, times(1)).removeAsync(eq(broker));
+            var localData = (BrokerLoadData) FieldUtils.readDeclaredField(target, "localData", true);
+            assertEquals(localData, new BrokerLoadData());
+        });
+
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Releasing, "broker-2", broker, VERSION_ID_INIT), null);
+        Awaitility.waitAtMost(3, TimeUnit.SECONDS).untilAsserted(() -> {
+            verify(target, times(2)).tombstone();
+            verify(store, times(1)).removeAsync(eq(broker));
+            var localData = (BrokerLoadData) FieldUtils.readDeclaredField(target, "localData", true);
+            assertEquals(localData, new BrokerLoadData());
+        });
+
+        FieldUtils.writeDeclaredField(target, "tombstoneDelayInMillis", 0, true);
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Splitting, "broker-2", broker, VERSION_ID_INIT), null);
+        Awaitility.waitAtMost(3, TimeUnit.SECONDS).untilAsserted(() -> {
+            verify(target, times(3)).tombstone();
+            verify(store, times(2)).removeAsync(eq(broker));
+            var localData = (BrokerLoadData) FieldUtils.readDeclaredField(target, "localData", true);
+            assertEquals(localData, new BrokerLoadData());
+        });
+
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Owned, broker, VERSION_ID_INIT), null);
+        Awaitility.waitAtMost(3, TimeUnit.SECONDS).untilAsserted(() -> {
+            verify(target, times(4)).tombstone();
+            verify(store, times(3)).removeAsync(eq(broker));
+            var localData = (BrokerLoadData) FieldUtils.readDeclaredField(target, "localData", true);
+            assertEquals(localData, new BrokerLoadData());
+        });
+    }
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java
index b5c415c405fbd..be8c6af2b0404 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/reporter/TopBundleLoadDataReporterTest.java
@@ -18,9 +18,12 @@
  */
 package org.apache.pulsar.broker.loadbalance.extensions.reporter;
 
+import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.VERSION_ID_INIT;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.testng.Assert.assertEquals;
@@ -29,9 +32,12 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
 import org.apache.commons.lang.reflect.FieldUtils;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData;
 import org.apache.pulsar.broker.loadbalance.extensions.models.TopKBundles;
 import org.apache.pulsar.broker.loadbalance.extensions.store.LoadDataStore;
@@ -42,6 +48,7 @@
 import org.apache.pulsar.broker.service.PulsarStats;
 import org.apache.pulsar.metadata.api.MetadataStoreException;
 import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats;
+import org.testcontainers.shaded.org.awaitility.Awaitility;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
@@ -58,10 +65,13 @@ public class TopBundleLoadDataReporterTest {
     private LocalPoliciesResources localPoliciesResources;
     String bundle1 = "my-tenant/my-namespace1/0x00000000_0x0FFFFFFF";
     String bundle2 = "my-tenant/my-namespace2/0x00000000_0x0FFFFFFF";
+    String bundle = bundle1;
+    String broker = "broker-1";
 
     @BeforeMethod
     void setup() throws MetadataStoreException {
         config = new ServiceConfiguration();
+        config.setLoadBalancerDebugModeEnabled(true);
         pulsar = mock(PulsarService.class);
         store = mock(LoadDataStore.class);
         brokerService = mock(BrokerService.class);
@@ -75,6 +85,7 @@ void setup() throws MetadataStoreException {
         doReturn(config).when(pulsar).getConfiguration();
         doReturn(pulsarStats).when(brokerService).getPulsarStats();
         doReturn(CompletableFuture.completedFuture(null)).when(store).pushAsync(any(), any());
+        doReturn(CompletableFuture.completedFuture(null)).when(store).removeAsync(any());
 
         doReturn(pulsarResources).when(pulsar).getPulsarResources();
         doReturn(namespaceResources).when(pulsarResources).getNamespaceResources();
@@ -102,22 +113,23 @@ public void testZeroUpdatedAt() {
 
     public void testGenerateLoadData() throws IllegalAccessException {
         doReturn(1l).when(pulsarStats).getUpdatedAt();
-        config.setLoadBalancerBundleLoadReportPercentage(100);
+        config.setLoadBalancerMaxNumberOfBundlesInBundleLoadReport(2);
         var target = new TopBundleLoadDataReporter(pulsar, "", store);
         var expected = new TopKBundles(pulsar);
         expected.update(bundleStats, 2);
         assertEquals(target.generateLoadData(), expected.getLoadData());
 
-        config.setLoadBalancerBundleLoadReportPercentage(50);
+        config.setLoadBalancerMaxNumberOfBundlesInBundleLoadReport(1);
         FieldUtils.writeDeclaredField(target, "lastBundleStatsUpdatedAt", 0l, true);
         expected = new TopKBundles(pulsar);
         expected.update(bundleStats, 1);
         assertEquals(target.generateLoadData(), expected.getLoadData());
 
-        config.setLoadBalancerBundleLoadReportPercentage(1);
+        config.setLoadBalancerMaxNumberOfBundlesInBundleLoadReport(0);
         FieldUtils.writeDeclaredField(target, "lastBundleStatsUpdatedAt", 0l, true);
+
         expected = new TopKBundles(pulsar);
-        expected.update(bundleStats, 1);
+        expected.update(bundleStats, 0);
         assertEquals(target.generateLoadData(), expected.getLoadData());
 
         doReturn(new HashMap()).when(brokerService).getBundleStats();
@@ -128,21 +140,83 @@ public void testGenerateLoadData() throws IllegalAccessException {
 
 
     public void testReportForce()  {
-        var target = new TopBundleLoadDataReporter(pulsar, "broker-1", store);
+        var target = new TopBundleLoadDataReporter(pulsar, broker, store);
         target.reportAsync(false);
         verify(store, times(0)).pushAsync(any(), any());
         target.reportAsync(true);
-        verify(store, times(1)).pushAsync("broker-1", new TopBundlesLoadData());
+        verify(store, times(1)).pushAsync(broker, new TopBundlesLoadData());
 
     }
 
     public void testReport(){
-        var target = new TopBundleLoadDataReporter(pulsar, "broker-1", store);
+        pulsar.getConfiguration().setLoadBalancerMaxNumberOfBundlesInBundleLoadReport(1);
+        var target = new TopBundleLoadDataReporter(pulsar, broker, store);
         doReturn(1l).when(pulsarStats).getUpdatedAt();
         var expected = new TopKBundles(pulsar);
         expected.update(bundleStats, 1);
         target.reportAsync(false);
-        verify(store, times(1)).pushAsync("broker-1", expected.getLoadData());
+        verify(store, times(1)).pushAsync(broker, expected.getLoadData());
     }
 
+    @Test
+    public void testTombstone() throws IllegalAccessException {
+
+        var target = spy(new TopBundleLoadDataReporter(pulsar, broker, store));
+
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Assigning, broker, VERSION_ID_INIT), null);
+        verify(store, times(0)).removeAsync(eq(broker));
+        verify(target, times(0)).tombstone();
+
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Deleted, broker, VERSION_ID_INIT), null);
+        verify(store, times(0)).removeAsync(eq(broker));
+        verify(target, times(0)).tombstone();
+
+
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Init, broker, VERSION_ID_INIT), null);
+        verify(store, times(0)).removeAsync(eq(broker));
+        verify(target, times(0)).tombstone();
+
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Free, broker, VERSION_ID_INIT), null);
+        verify(store, times(0)).removeAsync(eq(broker));
+        verify(target, times(0)).tombstone();
+
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Releasing, "broker-2", broker, VERSION_ID_INIT),
+                new RuntimeException());
+        verify(store, times(0)).removeAsync(eq(broker));
+        verify(target, times(0)).tombstone();
+
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Releasing, "broker-2", broker, VERSION_ID_INIT), null);
+        Awaitility.waitAtMost(3, TimeUnit.SECONDS).untilAsserted(() -> {
+            verify(target, times(1)).tombstone();
+            verify(store, times(1)).removeAsync(eq(broker));
+        });
+
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Releasing, "broker-2", broker, VERSION_ID_INIT), null);
+        Awaitility.waitAtMost(3, TimeUnit.SECONDS).untilAsserted(() -> {
+            verify(target, times(2)).tombstone();
+            verify(store, times(1)).removeAsync(eq(broker));
+        });
+
+        FieldUtils.writeDeclaredField(target, "tombstoneDelayInMillis", 0, true);
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Splitting, "broker-2", broker, VERSION_ID_INIT), null);
+        Awaitility.waitAtMost(3, TimeUnit.SECONDS).untilAsserted(() -> {
+            verify(target, times(3)).tombstone();
+            verify(store, times(2)).removeAsync(eq(broker));
+        });
+
+        target.handleEvent(bundle,
+                new ServiceUnitStateData(ServiceUnitState.Owned, broker, VERSION_ID_INIT), null);
+        Awaitility.waitAtMost(3, TimeUnit.SECONDS).untilAsserted(() -> {
+            verify(target, times(4)).tombstone();
+            verify(store, times(3)).removeAsync(eq(broker));
+        });
+    }
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
index 5ca9345d6def5..8c8d18e202a00 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java
@@ -18,10 +18,11 @@
  */
 package org.apache.pulsar.broker.loadbalance.extensions.scheduler;
 
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Failure;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Skip;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Label.Success;
-import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Balanced;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.CoolDown;
+import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.HitCount;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBrokers;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoBundles;
 import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.NoLoadData;
@@ -52,6 +53,7 @@
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.BiConsumer;
 import org.apache.commons.lang.reflect.FieldUtils;
 import org.apache.commons.math3.stat.descriptive.moment.Mean;
@@ -59,7 +61,11 @@
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.loadbalance.extensions.BrokerRegistry;
+import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl;
+import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerWrapper;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
 import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData;
@@ -97,6 +103,10 @@ public class TransferShedderTest {
     double setupLoadStd = 0.3982762860126121;
 
     PulsarService pulsar;
+    NamespaceService namespaceService;
+    ExtensibleLoadManagerWrapper loadManagerWrapper;
+    ExtensibleLoadManagerImpl loadManager;
+    ServiceUnitStateChannel channel;
     ServiceConfiguration conf;
     AntiAffinityGroupPolicyHelper antiAffinityGroupPolicyHelper;
     LocalPoliciesResources localPoliciesResources;
@@ -108,13 +118,16 @@ public class TransferShedderTest {
     @BeforeMethod
     public void init() throws MetadataStoreException {
         pulsar = mock(PulsarService.class);
+        loadManagerWrapper = mock(ExtensibleLoadManagerWrapper.class);
+        loadManager = mock(ExtensibleLoadManagerImpl.class);
+        channel = mock(ServiceUnitStateChannelImpl.class);
         conf = new ServiceConfiguration();
         conf.setLoadBalancerSheddingBundlesWithPoliciesEnabled(true);
         var pulsarResources = mock(PulsarResources.class);
         var namespaceResources = mock(NamespaceResources.class);
         var isolationPolicyResources = mock(NamespaceResources.IsolationPolicyResources.class);
         var factory = mock(NamespaceBundleFactory.class);
-        var namespaceService = mock(NamespaceService.class);
+        namespaceService = mock(NamespaceService.class);
         localPoliciesResources = mock(LocalPoliciesResources.class);
         antiAffinityGroupPolicyHelper = mock(AntiAffinityGroupPolicyHelper.class);
         doReturn(conf).when(pulsar).getConfiguration();
@@ -136,25 +149,30 @@ public void init() throws MetadataStoreException {
                     (upperEndpoint.equals(NamespaceBundles.FULL_UPPER_BOUND)) ? BoundType.CLOSED : BoundType.OPEN);
             return new NamespaceBundle(NamespaceName.get(namespace), hashRange, factory);
         }).when(factory).getBundle(anyString(), anyString());
-        doReturn(true).when(antiAffinityGroupPolicyHelper).canUnload(any(), any(), any(), any());
+        doReturn(new AtomicReference<>(loadManagerWrapper)).when(pulsar).getLoadManager();
+        doReturn(loadManager).when(loadManagerWrapper).get();
+        doReturn(channel).when(loadManager).getServiceUnitStateChannel();
+        doReturn(true).when(channel).isOwner(any(), any());
     }
 
     public LoadManagerContext setupContext(){
         var ctx = getContext();
 
-        var brokerLoadDataStore = ctx.brokerLoadDataStore();
-        brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx,  2));
-        brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx,  4));
-        brokerLoadDataStore.pushAsync("broker3", getCpuLoad(ctx,  6));
-        brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx,  80));
-        brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx,  90));
+
 
         var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
-        topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 2000000, 1000000));
-        topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB", 3000000, 1000000));
-        topBundlesLoadDataStore.pushAsync("broker3", getTopBundlesLoad("my-tenant/my-namespaceC", 4000000, 2000000));
-        topBundlesLoadDataStore.pushAsync("broker4", getTopBundlesLoad("my-tenant/my-namespaceD", 6000000, 2000000));
-        topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 7000000, 2000000));
+        topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 1000000, 2000000));
+        topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB", 1000000, 3000000));
+        topBundlesLoadDataStore.pushAsync("broker3", getTopBundlesLoad("my-tenant/my-namespaceC", 2000000, 4000000));
+        topBundlesLoadDataStore.pushAsync("broker4", getTopBundlesLoad("my-tenant/my-namespaceD", 2000000, 6000000));
+        topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 2000000, 7000000));
+
+        var brokerLoadDataStore = ctx.brokerLoadDataStore();
+        brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 2, "broker1"));
+        brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 4, "broker2"));
+        brokerLoadDataStore.pushAsync("broker3", getCpuLoad(ctx, 6, "broker3"));
+        brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx, 80, "broker4"));
+        brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx, 90, "broker5"));
         return ctx;
     }
 
@@ -166,8 +184,8 @@ public LoadManagerContext setupContext(int clusterSize) {
 
         Random rand = new Random();
         for (int i = 0; i < clusterSize; i++) {
-            int brokerLoad = rand.nextInt(100);
-            brokerLoadDataStore.pushAsync("broker" + i, getCpuLoad(ctx,  brokerLoad));
+            int brokerLoad = rand.nextInt(1000);
+            brokerLoadDataStore.pushAsync("broker" + i, getCpuLoad(ctx,  brokerLoad, "broker" + i));
             int bundleLoad = rand.nextInt(brokerLoad + 1);
             topBundlesLoadDataStore.pushAsync("broker" + i, getTopBundlesLoad("my-tenant/my-namespace" + i,
                     bundleLoad, brokerLoad - bundleLoad));
@@ -175,7 +193,7 @@ public LoadManagerContext setupContext(int clusterSize) {
         return ctx;
     }
 
-    public BrokerLoadData getCpuLoad(LoadManagerContext ctx, int load) {
+    public BrokerLoadData getCpuLoad(LoadManagerContext ctx, int load, String broker) {
         var loadData = new BrokerLoadData();
         SystemResourceUsage usage1 = new SystemResourceUsage();
         var cpu = new ResourceUsage(load, 100.0);
@@ -188,8 +206,17 @@ public BrokerLoadData getCpuLoad(LoadManagerContext ctx, int load) {
         usage1.setDirectMemory(directMemory);
         usage1.setBandwidthIn(bandwidthIn);
         usage1.setBandwidthOut(bandwidthOut);
-        loadData.update(usage1, 1,2,3,4,5,6,
-                ctx.brokerConfiguration());
+        if (ctx.topBundleLoadDataStore()
+                .get(broker).isPresent()) {
+            var throughputOut = ctx.topBundleLoadDataStore()
+                    .get(broker).get()
+                    .getTopBundlesLoadData().stream().mapToDouble(v -> v.stats().msgThroughputOut).sum();
+            loadData.update(usage1, 1, throughputOut, 3, 4, 5, 6,
+                    ctx.brokerConfiguration());
+        } else {
+            loadData.update(usage1, 1, 2, 3, 4, 5, 6,
+                    ctx.brokerConfiguration());
+        }
         return loadData;
     }
 
@@ -204,6 +231,29 @@ public TopBundlesLoadData getTopBundlesLoad(String bundlePrefix, int load1, int
         return topKBundles.getLoadData();
     }
 
+    public TopBundlesLoadData getTopBundlesLoad(String bundlePrefix,
+                                                int load1, int load2, int load3, int load4, int load5) {
+        var namespaceBundleStats1 = new NamespaceBundleStats();
+        namespaceBundleStats1.msgThroughputOut = load1;
+        var namespaceBundleStats2 = new NamespaceBundleStats();
+        namespaceBundleStats2.msgThroughputOut = load2;
+        var namespaceBundleStats3 = new NamespaceBundleStats();
+        namespaceBundleStats3.msgThroughputOut = load3;
+        var namespaceBundleStats4 = new NamespaceBundleStats();
+        namespaceBundleStats4.msgThroughputOut = load4;
+        var namespaceBundleStats5 = new NamespaceBundleStats();
+        namespaceBundleStats5.msgThroughputOut = load5;
+        var topKBundles = new TopKBundles(pulsar);
+        topKBundles.update(Map.of(
+                bundlePrefix + "/0x00000000_0x1FFFFFFF", namespaceBundleStats1,
+                bundlePrefix + "/0x1FFFFFFF_0x2FFFFFFF", namespaceBundleStats2,
+                bundlePrefix + "/0x2FFFFFFF_0x3FFFFFFF", namespaceBundleStats3,
+                bundlePrefix + "/0x3FFFFFFF_0x4FFFFFFF", namespaceBundleStats4,
+                bundlePrefix + "/0x4FFFFFFF_0x5FFFFFFF", namespaceBundleStats5
+        ), 5);
+        return topKBundles.getLoadData();
+    }
+
     public TopBundlesLoadData getTopBundlesLoad(String bundlePrefix, int load1) {
         var namespaceBundleStats1 = new NamespaceBundleStats();
         namespaceBundleStats1.msgThroughputOut = load1;
@@ -230,6 +280,7 @@ public LoadManagerContext getContext(){
         var conf = new ServiceConfiguration();
         conf.setLoadBalancerDebugModeEnabled(true);
         conf.setLoadBalancerSheddingBundlesWithPoliciesEnabled(false);
+        conf.setLoadBalancerSheddingConditionHitCountThreshold(0);
         var brokerLoadDataStore = new LoadDataStore() {
             Map map = new HashMap<>();
             @Override
@@ -357,29 +408,44 @@ public void testEmptyBrokerLoadData() {
         UnloadCounter counter = new UnloadCounter();
         TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = getContext();
-        ctx.brokerConfiguration().setLoadBalancerDebugModeEnabled(true);
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
         assertTrue(res.isEmpty());
         assertEquals(counter.getBreakdownCounters().get(Skip).get(NoBrokers).get(), 1);
     }
 
+    @Test
+    public void testNoOwnerLoadData() throws IllegalAccessException {
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
+        FieldUtils.writeDeclaredField(transferShedder, "channel", channel, true);
+        var ctx = setupContext();
+        doReturn(false).when(channel).isOwner(any(), any());
+        var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
+        assertTrue(res.isEmpty());
+        assertEquals(counter.getBreakdownCounters().get(Skip).get(NoBundles).get(), 1);
+        assertEquals(counter.getLoadAvg(), setupLoadAvg);
+        assertEquals(counter.getLoadStd(), setupLoadStd);
+    }
+
     @Test
     public void testEmptyTopBundlesLoadData() {
         UnloadCounter counter = new UnloadCounter();
         TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = getContext();
-        ctx.brokerConfiguration().setLoadBalancerDebugModeEnabled(true);
         var brokerLoadDataStore = ctx.brokerLoadDataStore();
 
-        brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx,  90));
-        brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx,  10));
-        brokerLoadDataStore.pushAsync("broker3", getCpuLoad(ctx,  20));
+        brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 2, "broker1"));
+        brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 4, "broker2"));
+        brokerLoadDataStore.pushAsync("broker3", getCpuLoad(ctx, 6, "broker3"));
+        brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx, 80, "broker4"));
+        brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx, 90, "broker5"));
+
 
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
         assertTrue(res.isEmpty());
         assertEquals(counter.getBreakdownCounters().get(Skip).get(NoLoadData).get(), 1);
-        assertEquals(counter.getLoadAvg(), 0.39999999999999997);
-        assertEquals(counter.getLoadStd(), 0.35590260840104376);
+        assertEquals(counter.getLoadAvg(), setupLoadAvg);
+        assertEquals(counter.getLoadStd(), setupLoadStd);
     }
 
     @Test
@@ -445,16 +511,18 @@ public void testRecentlyUnloadedBundles() {
         recentlyUnloadedBundles.put(bundleD1, now);
         recentlyUnloadedBundles.put(bundleD2, now);
         var res = transferShedder.findBundlesForUnloading(ctx, recentlyUnloadedBundles, Map.of());
-
-        assertTrue(res.isEmpty());
-        assertEquals(counter.getBreakdownCounters().get(Skip).get(NoBundles).get(), 1);
+        var expected = new HashSet();
+        expected.add(new UnloadDecision(new Unload("broker3",
+                "my-tenant/my-namespaceC/0x00000000_0x0FFFFFFF",
+                Optional.of("broker1")),
+                Success, Overloaded));
+        assertEquals(res, expected);
         assertEquals(counter.getLoadAvg(), setupLoadAvg);
         assertEquals(counter.getLoadStd(), setupLoadStd);
     }
 
     @Test
     public void testGetAvailableBrokersFailed() {
-        var pulsar = getMockPulsar();
         UnloadCounter counter = new UnloadCounter();
         AntiAffinityGroupPolicyHelper affinityGroupPolicyHelper = mock(AntiAffinityGroupPolicyHelper.class);
         TransferShedder transferShedder = new TransferShedder(pulsar, counter, affinityGroupPolicyHelper);
@@ -462,17 +530,16 @@ public void testGetAvailableBrokersFailed() {
         BrokerRegistry registry = ctx.brokerRegistry();
         doReturn(FutureUtil.failedFuture(new TimeoutException())).when(registry).getAvailableBrokerLookupDataAsync();
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
-        assertEquals(counter.getBreakdownCounters().get(Skip).get(Unknown).get(), 1);
-        assertEquals(counter.getLoadAvg(), setupLoadAvg);
-        assertEquals(counter.getLoadStd(), setupLoadStd);
+        assertEquals(counter.getBreakdownCounters().get(Failure).get(Unknown).get(), 1);
+        assertEquals(counter.getLoadAvg(), 0.0);
+        assertEquals(counter.getLoadStd(), 0.0);
     }
 
     @Test(timeOut = 30 * 1000)
     public void testBundlesWithIsolationPolicies() throws IllegalAccessException {
-        var pulsar = getMockPulsar();
+        doReturn(true).when(antiAffinityGroupPolicyHelper).canUnload(any(), any(), any(), any());
         UnloadCounter counter = new UnloadCounter();
         TransferShedder transferShedder = spy(new TransferShedder(pulsar, counter, antiAffinityGroupPolicyHelper));
-
         var allocationPoliciesSpy = (SimpleResourceAllocationPolicies)
                 spy(FieldUtils.readDeclaredField(transferShedder, "allocationPolicies", true));
         FieldUtils.writeDeclaredField(transferShedder, "allocationPolicies", allocationPoliciesSpy, true);
@@ -484,7 +551,7 @@ public void testBundlesWithIsolationPolicies() throws IllegalAccessException {
         var ctx = setupContext();
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
         var expected = new HashSet();
-        expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")),
+        expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker1")),
                 Success, Overloaded));
         assertEquals(res, expected);
         assertEquals(counter.getLoadAvg(), setupLoadAvg);
@@ -572,29 +639,9 @@ private void setIsolationPolicies(SimpleResourceAllocationPolicies policies,
         }).when(policies).shouldFailoverToSecondaries(eq(namespaceName), anyInt());
     }
 
-    private PulsarService getMockPulsar() {
-        var pulsar = mock(PulsarService.class);
-        var namespaceService = mock(NamespaceService.class);
-        doReturn(namespaceService).when(pulsar).getNamespaceService();
-        NamespaceBundleFactory factory = mock(NamespaceBundleFactory.class);
-        doReturn(factory).when(namespaceService).getNamespaceBundleFactory();
-        doAnswer(answer -> {
-            String namespace = answer.getArgument(0, String.class);
-            String bundleRange = answer.getArgument(1, String.class);
-            String[] boundaries = bundleRange.split("_");
-            Long lowerEndpoint = Long.decode(boundaries[0]);
-            Long upperEndpoint = Long.decode(boundaries[1]);
-            Range hashRange = Range.range(lowerEndpoint, BoundType.CLOSED, upperEndpoint,
-                    (upperEndpoint.equals(NamespaceBundles.FULL_UPPER_BOUND)) ? BoundType.CLOSED : BoundType.OPEN);
-            return new NamespaceBundle(NamespaceName.get(namespace), hashRange, factory);
-        }).when(factory).getBundle(anyString(), anyString());
-        return pulsar;
-    }
-
 
     @Test
     public void testBundlesWithAntiAffinityGroup() throws IllegalAccessException, MetadataStoreException {
-        var pulsar = getMockPulsar();
         var counter = new UnloadCounter();
         TransferShedder transferShedder = new TransferShedder(pulsar, counter, antiAffinityGroupPolicyHelper);
         var allocationPoliciesSpy = (SimpleResourceAllocationPolicies)
@@ -606,7 +653,10 @@ public void testBundlesWithAntiAffinityGroup() throws IllegalAccessException, Me
         doReturn(Optional.of(localPolicies)).when(localPoliciesResources).getLocalPolicies(any());
 
         var ctx = setupContext();
-        doReturn(false).when(antiAffinityGroupPolicyHelper).canUnload(any(), any(), any(), any());
+        var antiAffinityGroupPolicyHelperSpy = (AntiAffinityGroupPolicyHelper)
+                spy(FieldUtils.readDeclaredField(transferShedder, "antiAffinityGroupPolicyHelper", true));
+        doReturn(false).when(antiAffinityGroupPolicyHelperSpy).canUnload(any(), any(), any(), any());
+        FieldUtils.writeDeclaredField(transferShedder, "antiAffinityGroupPolicyHelper", antiAffinityGroupPolicyHelperSpy, true);
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
 
         assertTrue(res.isEmpty());
@@ -615,7 +665,7 @@ public void testBundlesWithAntiAffinityGroup() throws IllegalAccessException, Me
         assertEquals(counter.getLoadStd(), setupLoadStd);
 
 
-        doReturn(true).when(antiAffinityGroupPolicyHelper).canUnload(any(), eq(bundleE1), any(), any());
+        doReturn(true).when(antiAffinityGroupPolicyHelperSpy).canUnload(any(), eq(bundleE1), any(), any());
         var res2 = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
         var expected2 = new HashSet<>();
         expected2.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")),
@@ -630,11 +680,18 @@ public void testTargetStd() {
         UnloadCounter counter = new UnloadCounter();
         TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = getContext();
+        BrokerRegistry brokerRegistry = mock(BrokerRegistry.class);
+        doReturn(CompletableFuture.completedFuture(Map.of(
+                "broker1", mock(BrokerLookupData.class),
+                "broker2", mock(BrokerLookupData.class),
+                "broker3", mock(BrokerLookupData.class)
+        ))).when(brokerRegistry).getAvailableBrokerLookupDataAsync();
+        doReturn(brokerRegistry).when(ctx).brokerRegistry();
         ctx.brokerConfiguration().setLoadBalancerDebugModeEnabled(true);
         var brokerLoadDataStore = ctx.brokerLoadDataStore();
-        brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx,  10));
-        brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx,  20));
-        brokerLoadDataStore.pushAsync("broker3", getCpuLoad(ctx,  30));
+        brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 10, "broker1"));
+        brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 20, "broker2"));
+        brokerLoadDataStore.pushAsync("broker3", getCpuLoad(ctx, 30, "broker3"));
 
         var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
 
@@ -645,7 +702,7 @@ public void testTargetStd() {
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
 
         assertTrue(res.isEmpty());
-        assertEquals(counter.getBreakdownCounters().get(Skip).get(Balanced).get(), 1);
+        assertEquals(counter.getBreakdownCounters().get(Skip).get(HitCount).get(), 1);
         assertEquals(counter.getLoadAvg(), 0.2000000063578288);
         assertEquals(counter.getLoadStd(), 0.08164966587949089);
     }
@@ -669,6 +726,26 @@ public void testSingleTopBundlesLoadData() {
         assertEquals(counter.getLoadStd(), setupLoadStd);
     }
 
+    @Test
+    public void testBundleThroughputLargerThanOffloadThreshold() {
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
+        var ctx = setupContext();
+        var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
+        topBundlesLoadDataStore.pushAsync("broker4", getTopBundlesLoad("my-tenant/my-namespaceD", 1000000000, 1000000000));
+        topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 1000000000, 1000000000));
+        var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
+
+        var expected = new HashSet();
+        expected.add(new UnloadDecision(new Unload("broker3",
+                "my-tenant/my-namespaceC/0x00000000_0x0FFFFFFF",
+                Optional.of("broker1")),
+                Success, Overloaded));
+        assertEquals(res, expected);
+        assertEquals(counter.getLoadAvg(), setupLoadAvg);
+        assertEquals(counter.getLoadStd(), setupLoadStd);
+    }
+
 
     @Test
     public void testTargetStdAfterTransfer() {
@@ -676,8 +753,8 @@ public void testTargetStdAfterTransfer() {
         TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = setupContext();
         var brokerLoadDataStore = ctx.brokerLoadDataStore();
-        brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx,  55));
-        brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx,  65));
+        brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx,  55, "broker4"));
+        brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx,  65, "broker5"));
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
 
         var expected = new HashSet();
@@ -688,19 +765,186 @@ public void testTargetStdAfterTransfer() {
         assertEquals(counter.getLoadStd(), 0.27644891028904417);
     }
 
+    @Test
+    public void testUnloadBundlesGreaterThanTargetThroughput() throws IllegalAccessException {
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
+        var ctx = getContext();
+
+        var brokerRegistry = mock(BrokerRegistry.class);
+        doReturn(brokerRegistry).when(ctx).brokerRegistry();
+        doReturn(CompletableFuture.completedFuture(Map.of(
+                "broker1", mock(BrokerLookupData.class),
+                "broker2", mock(BrokerLookupData.class)
+        ))).when(brokerRegistry).getAvailableBrokerLookupDataAsync();
+
+        var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
+        topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 1000000, 2000000, 3000000, 4000000, 5000000));
+        topBundlesLoadDataStore.pushAsync("broker2",
+                getTopBundlesLoad("my-tenant/my-namespaceB", 100000000, 180000000, 220000000, 250000000, 250000000));
+
+
+        var brokerLoadDataStore = ctx.brokerLoadDataStore();
+        brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 10, "broker1"));
+        brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 1000, "broker2"));
+
+
+        var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
+        var expected = new HashSet();
+        expected.add(new UnloadDecision(
+                new Unload("broker2", "my-tenant/my-namespaceB/0x00000000_0x1FFFFFFF", Optional.of("broker1")),
+                Success, Overloaded));
+        expected.add(new UnloadDecision(
+                new Unload("broker2", "my-tenant/my-namespaceB/0x1FFFFFFF_0x2FFFFFFF", Optional.of("broker1")),
+                Success, Overloaded));
+        expected.add(new UnloadDecision(
+                new Unload("broker2", "my-tenant/my-namespaceB/0x2FFFFFFF_0x3FFFFFFF", Optional.of("broker1")),
+                Success, Overloaded));
+        expected.add(new UnloadDecision(
+                new Unload("broker1", "my-tenant/my-namespaceA/0x00000000_0x1FFFFFFF", Optional.of("broker2")),
+                Success, Overloaded));
+        expected.add(new UnloadDecision(
+                new Unload("broker1", "my-tenant/my-namespaceA/0x1FFFFFFF_0x2FFFFFFF", Optional.of("broker2")),
+                Success, Overloaded));
+        expected.add(new UnloadDecision(
+                new Unload("broker1","my-tenant/my-namespaceA/0x2FFFFFFF_0x3FFFFFFF", Optional.of("broker2")),
+                Success, Overloaded));
+        expected.add(new UnloadDecision(
+                new Unload("broker1","my-tenant/my-namespaceA/0x3FFFFFFF_0x4FFFFFFF", Optional.of("broker2")),
+                Success, Overloaded));
+        assertEquals(counter.getLoadAvg(), 5.05);
+        assertEquals(counter.getLoadStd(), 4.95);
+        assertEquals(res, expected);
+        var stats = (TransferShedder.LoadStats)
+                FieldUtils.readDeclaredField(transferShedder, "stats", true);
+        assertEquals(stats.std(), 0.050000004900021836);
+    }
+
+    @Test
+    public void testSkipBundlesGreaterThanTargetThroughputAfterSplit() {
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
+        var ctx = getContext();
+        ctx.brokerConfiguration().setLoadBalancerBrokerLoadTargetStd(0.20);
+
+        var brokerRegistry = mock(BrokerRegistry.class);
+        doReturn(brokerRegistry).when(ctx).brokerRegistry();
+        doReturn(CompletableFuture.completedFuture(Map.of(
+                "broker1", mock(BrokerLookupData.class),
+                "broker2", mock(BrokerLookupData.class)
+        ))).when(brokerRegistry).getAvailableBrokerLookupDataAsync();
+
+        var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
+        topBundlesLoadDataStore.pushAsync("broker1",
+                getTopBundlesLoad("my-tenant/my-namespaceA", 1, 500000000));
+        topBundlesLoadDataStore.pushAsync("broker2",
+                getTopBundlesLoad("my-tenant/my-namespaceB", 500000000, 500000000));
+
+
+        var brokerLoadDataStore = ctx.brokerLoadDataStore();
+        brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 50, "broker1"));
+        brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 100, "broker2"));
+
+
+        var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
+        assertTrue(res.isEmpty());
+        assertEquals(counter.getBreakdownCounters().get(Skip).get(NoBundles).get(), 1);
+    }
+
+
+    @Test
+    public void testUnloadBundlesLessThanTargetThroughputAfterSplit() throws IllegalAccessException {
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
+        var ctx = getContext();
+
+        var brokerRegistry = mock(BrokerRegistry.class);
+        doReturn(brokerRegistry).when(ctx).brokerRegistry();
+        doReturn(CompletableFuture.completedFuture(Map.of(
+                "broker1", mock(BrokerLookupData.class),
+                "broker2", mock(BrokerLookupData.class)
+        ))).when(brokerRegistry).getAvailableBrokerLookupDataAsync();
+
+        var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
+        topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 1000000, 2000000, 3000000, 4000000, 5000000));
+        topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB", 490000000, 510000000));
+
+
+        var brokerLoadDataStore = ctx.brokerLoadDataStore();
+        brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 10, "broker1"));
+        brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 1000, "broker2"));
+
+
+        var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
+        var expected = new HashSet();
+        expected.add(new UnloadDecision(
+                new Unload("broker2", "my-tenant/my-namespaceB/0x00000000_0x0FFFFFFF", Optional.of("broker1")),
+                Success, Overloaded));
+        assertEquals(counter.getLoadAvg(), 5.05);
+        assertEquals(counter.getLoadStd(), 4.95);
+        assertEquals(res, expected);
+        var stats = (TransferShedder.LoadStats)
+                FieldUtils.readDeclaredField(transferShedder, "stats", true);
+        assertEquals(stats.std(), 0.050000004900021836);
+
+    }
+
+
+    @Test
+    public void testUnloadBundlesGreaterThanTargetThroughputAfterSplit() throws IllegalAccessException {
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
+        var ctx = getContext();
+
+        var brokerRegistry = mock(BrokerRegistry.class);
+        doReturn(brokerRegistry).when(ctx).brokerRegistry();
+        doReturn(CompletableFuture.completedFuture(Map.of(
+                "broker1", mock(BrokerLookupData.class),
+                "broker2", mock(BrokerLookupData.class)
+        ))).when(brokerRegistry).getAvailableBrokerLookupDataAsync();
+
+        var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
+        topBundlesLoadDataStore.pushAsync("broker1", getTopBundlesLoad("my-tenant/my-namespaceA", 2400000, 2400000));
+        topBundlesLoadDataStore.pushAsync("broker2", getTopBundlesLoad("my-tenant/my-namespaceB", 5000000, 5000000));
+
+
+        var brokerLoadDataStore = ctx.brokerLoadDataStore();
+        brokerLoadDataStore.pushAsync("broker1", getCpuLoad(ctx, 48, "broker1"));
+        brokerLoadDataStore.pushAsync("broker2", getCpuLoad(ctx, 100, "broker2"));
+        var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
+
+        var expected = new HashSet();
+        expected.add(new UnloadDecision(
+                new Unload("broker1",
+                        res.stream().filter(x -> x.getUnload().sourceBroker().equals("broker1")).findFirst().get()
+                        .getUnload().serviceUnit(), Optional.of("broker2")),
+                Success, Overloaded));
+        expected.add(new UnloadDecision(
+                new Unload("broker2",
+                        res.stream().filter(x -> x.getUnload().sourceBroker().equals("broker2")).findFirst().get()
+                                .getUnload().serviceUnit(), Optional.of("broker1")),
+                Success, Overloaded));
+        assertEquals(counter.getLoadAvg(), 0.74);
+        assertEquals(counter.getLoadStd(), 0.26);
+        assertEquals(res, expected);
+        var stats = (TransferShedder.LoadStats)
+                FieldUtils.readDeclaredField(transferShedder, "stats", true);
+        assertEquals(stats.std(), 2.5809568279517847E-8);
+    }
+
+
     @Test
     public void testMinBrokerWithZeroTraffic() throws IllegalAccessException {
         UnloadCounter counter = new UnloadCounter();
         TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = setupContext();
         var brokerLoadDataStore = ctx.brokerLoadDataStore();
-        brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx,  55));
-        brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx,  65));
 
-        var load = getCpuLoad(ctx,  4);
-        FieldUtils.writeDeclaredField(load,"msgThroughputIn", 0, true);
-        FieldUtils.writeDeclaredField(load,"msgThroughputOut", 0, true);
+        var load = getCpuLoad(ctx,  4, "broker2");
+        FieldUtils.writeDeclaredField(load,"msgThroughputEMA", 0, true);
         brokerLoadDataStore.pushAsync("broker2", load);
+        brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx,  55, "broker4"));
+        brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx,  65, "broker5"));
 
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
 
@@ -714,13 +958,63 @@ public void testMinBrokerWithZeroTraffic() throws IllegalAccessException {
         assertEquals(counter.getLoadStd(), 0.27644891028904417);
     }
 
+    @Test
+    public void testMinBrokerWithLowerLoadThanAvg() throws IllegalAccessException {
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
+        var ctx = setupContext();
+        var brokerLoadDataStore = ctx.brokerLoadDataStore();
+
+        var load = getCpuLoad(ctx,  3 , "broker2");
+        brokerLoadDataStore.pushAsync("broker2", load);
+        brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx,  55, "broker4"));
+        brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx,  65, "broker5"));
+
+        var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
+
+        var expected = new HashSet();
+        expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")),
+                Success, Overloaded));
+        expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")),
+                Success, Underloaded));
+        assertEquals(res, expected);
+        assertEquals(counter.getLoadAvg(), 0.262);
+        assertEquals(counter.getLoadStd(), 0.2780935094532054);
+    }
+
     @Test
     public void testMaxNumberOfTransfersPerShedderCycle() {
         UnloadCounter counter = new UnloadCounter();
         TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = setupContext();
         ctx.brokerConfiguration()
-                .setLoadBalancerMaxNumberOfBrokerTransfersPerCycle(10);
+                .setLoadBalancerMaxNumberOfBrokerSheddingPerCycle(10);
+        var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
+        var expected = new HashSet();
+        expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")),
+                Success, Overloaded));
+        expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")),
+                Success, Overloaded));
+        assertEquals(res, expected);
+        assertEquals(counter.getLoadAvg(), setupLoadAvg);
+        assertEquals(counter.getLoadStd(), setupLoadStd);
+    }
+
+    @Test
+    public void testLoadBalancerSheddingConditionHitCountThreshold() {
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
+        var ctx = setupContext();
+        int max = 3;
+        ctx.brokerConfiguration()
+                .setLoadBalancerSheddingConditionHitCountThreshold(max);
+        for (int i = 0; i < max; i++) {
+            var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
+            assertTrue(res.isEmpty());
+            assertEquals(counter.getBreakdownCounters().get(Skip).get(HitCount).get(), i+1);
+            assertEquals(counter.getLoadAvg(), setupLoadAvg);
+            assertEquals(counter.getLoadStd(), setupLoadStd);
+        }
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
         var expected = new HashSet();
         expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")),
@@ -738,7 +1032,7 @@ public void testRemainingTopBundles() {
         TransferShedder transferShedder = new TransferShedder(counter);
         var ctx = setupContext();
         var topBundlesLoadDataStore = ctx.topBundleLoadDataStore();
-        topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 3000000, 2000000));
+        topBundlesLoadDataStore.pushAsync("broker5", getTopBundlesLoad("my-tenant/my-namespaceE", 2000000, 3000000));
         var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
 
         var expected = new HashSet();
@@ -751,6 +1045,33 @@ public void testRemainingTopBundles() {
         assertEquals(counter.getLoadStd(), setupLoadStd);
     }
 
+    @Test
+    public void testLoadMoreThan100() throws IllegalAccessException {
+        UnloadCounter counter = new UnloadCounter();
+        TransferShedder transferShedder = new TransferShedder(counter);
+        var ctx = setupContext();
+
+        var brokerLoadDataStore = ctx.brokerLoadDataStore();
+        brokerLoadDataStore.pushAsync("broker4", getCpuLoad(ctx,  200, "broker4"));
+        brokerLoadDataStore.pushAsync("broker5", getCpuLoad(ctx,  1000, "broker5"));
+        var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of());
+
+        var expected = new HashSet();
+        expected.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")),
+                Success, Overloaded));
+        expected.add(new UnloadDecision(new Unload("broker4", bundleD1, Optional.of("broker2")),
+                Success, Overloaded));
+        assertEquals(res, expected);
+        assertEquals(counter.getLoadAvg(), 2.4240000000000004);
+        assertEquals(counter.getLoadStd(), 3.8633332758124816);
+
+
+        var stats = (TransferShedder.LoadStats)
+                FieldUtils.readDeclaredField(transferShedder, "stats", true);
+        assertEquals(stats.avg(), 2.4240000000000004);
+        assertEquals(stats.std(), 2.781643776903451);
+    }
+
     @Test
     public void testRandomLoad() throws IllegalAccessException {
         UnloadCounter counter = new UnloadCounter();
@@ -767,7 +1088,7 @@ public void testRandomLoad() throws IllegalAccessException {
     }
 
     @Test
-    public void testLoadStats() {
+    public void testRandomLoadStats() {
         int numBrokers = 10;
         double delta = 0.0001;
         for (int t = 0; t < 5; t++) {
@@ -776,8 +1097,13 @@ public void testLoadStats() {
             var loadStore = ctx.brokerLoadDataStore();
             stats.setLoadDataStore(loadStore);
             var conf = ctx.brokerConfiguration();
-            stats.update(loadStore, Map.of(), conf);
             double[] loads = new double[numBrokers];
+            final Map availableBrokers = new HashMap<>();
+            for (int i = 0; i < loads.length; i++) {
+                availableBrokers.put("broker" + i, mock(BrokerLookupData.class));
+            }
+            stats.update(loadStore, availableBrokers, Map.of(), conf);
+
             var brokerLoadDataStore = ctx.brokerLoadDataStore();
             for (int i = 0; i < loads.length; i++) {
                 loads[i] = loadStore.get("broker" + i).get().getWeightedMaxEMA();
@@ -785,16 +1111,16 @@ public void testLoadStats() {
             int i = 0;
             int j = loads.length - 1;
             Arrays.sort(loads);
-            for (int k = 0; k < conf.getLoadBalancerMaxNumberOfBrokerTransfersPerCycle(); k++) {
+            for (int k = 0; k < conf.getLoadBalancerMaxNumberOfBrokerSheddingPerCycle(); k++) {
                 double minLoad = loads[i];
                 double maxLoad = loads[j];
                 double offload = (maxLoad - minLoad) / 2;
                 Mean mean = new Mean();
                 StandardDeviation std = new StandardDeviation(false);
                 assertEquals(minLoad,
-                        loadStore.get(stats.minBrokers().pollLast()).get().getWeightedMaxEMA());
+                        loadStore.get(stats.peekMinBroker()).get().getWeightedMaxEMA());
                 assertEquals(maxLoad,
-                        loadStore.get(stats.maxBrokers().pollLast()).get().getWeightedMaxEMA());
+                        loadStore.get(stats.pollMaxBroker()).get().getWeightedMaxEMA());
                 assertEquals(stats.totalBrokers(), numBrokers);
                 assertEquals(stats.avg(), mean.evaluate(loads), delta);
                 assertEquals(stats.std(), std.evaluate(loads), delta);
@@ -804,4 +1130,41 @@ public void testLoadStats() {
             }
         }
     }
+
+    @Test
+    public void testHighVarianceLoadStats() {
+        int[] loads = {1, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100};
+        var ctx = getContext();
+        TransferShedder.LoadStats stats = new TransferShedder.LoadStats();
+        var loadStore = ctx.brokerLoadDataStore();
+        stats.setLoadDataStore(loadStore);
+        var conf = ctx.brokerConfiguration();
+        final Map availableBrokers = new HashMap<>();
+        for (int i = 0; i < loads.length; i++) {
+            availableBrokers.put("broker" + i, mock(BrokerLookupData.class));
+            loadStore.pushAsync("broker" + i, getCpuLoad(ctx,  loads[i], "broker" + i));
+        }
+        stats.update(loadStore, availableBrokers, Map.of(), conf);
+
+        assertEquals(stats.avg(), 0.9417647058823528);
+        assertEquals(stats.std(), 0.23294117647058868);
+    }
+
+    @Test
+    public void testLowVarianceLoadStats() {
+        int[] loads = {390, 391, 392, 393, 394, 395, 396, 397, 398, 399};
+        var ctx = getContext();
+        TransferShedder.LoadStats stats = new TransferShedder.LoadStats();
+        var loadStore = ctx.brokerLoadDataStore();
+        stats.setLoadDataStore(loadStore);
+        var conf = ctx.brokerConfiguration();
+        final Map availableBrokers = new HashMap<>();
+        for (int i = 0; i < loads.length; i++) {
+            availableBrokers.put("broker" + i, mock(BrokerLookupData.class));
+            loadStore.pushAsync("broker" + i, getCpuLoad(ctx,  loads[i], "broker" + i));
+        }
+        stats.update(loadStore, availableBrokers, Map.of(), conf);
+        assertEquals(stats.avg(), 3.9449999999999994);
+        assertEquals(stats.std(), 0.028722813232795824);
+    }
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java
index 7d9cf556360e4..38d4e9904e649 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/UnloadSchedulerTest.java
@@ -54,8 +54,8 @@
 
 @Test(groups = "broker")
 public class UnloadSchedulerTest {
-
-   private ScheduledExecutorService loadManagerExecutor;
+    private PulsarService pulsar;
+    private ScheduledExecutorService loadManagerExecutor;
 
     public LoadManagerContext setupContext(){
         var ctx = getContext();
@@ -65,8 +65,12 @@ public LoadManagerContext setupContext(){
 
     @BeforeMethod
     public void setUp() {
-        this.loadManagerExecutor = Executors
-                .newSingleThreadScheduledExecutor(new ExecutorProvider.ExtendedThreadFactory("pulsar-load-manager"));
+        this.pulsar = mock(PulsarService.class);
+        loadManagerExecutor = Executors
+                .newSingleThreadScheduledExecutor(new
+                        ExecutorProvider.ExtendedThreadFactory("pulsar-load-manager"));
+        doReturn(loadManagerExecutor)
+                .when(pulsar).getLoadManagerExecutor();
     }
 
     @AfterMethod
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java
index deff6fb00fafb..0aa055b58acd3 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/DefaultNamespaceBundleSplitStrategyTest.java
@@ -31,10 +31,15 @@
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.loadbalance.extensions.BrokerRegistry;
+import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl;
+import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerWrapper;
 import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel;
+import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl;
 import org.apache.pulsar.broker.loadbalance.extensions.models.Split;
 import org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision;
 import org.apache.pulsar.broker.loadbalance.extensions.models.SplitCounter;
@@ -50,6 +55,9 @@
 public class DefaultNamespaceBundleSplitStrategyTest {
 
     PulsarService pulsar;
+    ExtensibleLoadManagerWrapper loadManagerWrapper;
+    ExtensibleLoadManagerImpl loadManager;
+    ServiceUnitStateChannel channel;
     BrokerService brokerService;
     PulsarStats pulsarStats;
     Map bundleStats;
@@ -76,7 +84,7 @@ void setup() {
         config.setLoadBalancerNamespaceBundleMaxMsgRate(100);
         config.setLoadBalancerNamespaceBundleMaxBandwidthMbytes(100);
         config.setLoadBalancerMaxNumberOfBundlesToSplitPerCycle(1);
-        config.setLoadBalancerNamespaceBundleSplitConditionThreshold(3);
+        config.setLoadBalancerNamespaceBundleSplitConditionHitCountThreshold(3);
 
         pulsar = mock(PulsarService.class);
         brokerService = mock(BrokerService.class);
@@ -85,8 +93,9 @@ void setup() {
         namespaceBundleFactory = mock(NamespaceBundleFactory.class);
         loadManagerContext = mock(LoadManagerContext.class);
         brokerRegistry = mock(BrokerRegistry.class);
-
-
+        loadManagerWrapper = mock(ExtensibleLoadManagerWrapper.class);
+        loadManager = mock(ExtensibleLoadManagerImpl.class);
+        channel = mock(ServiceUnitStateChannelImpl.class);
 
         doReturn(brokerService).when(pulsar).getBrokerService();
         doReturn(config).when(pulsar).getConfiguration();
@@ -96,6 +105,10 @@ void setup() {
         doReturn(true).when(namespaceBundleFactory).canSplitBundle(any());
         doReturn(brokerRegistry).when(loadManagerContext).brokerRegistry();
         doReturn(broker).when(brokerRegistry).getBrokerId();
+        doReturn(new AtomicReference(loadManagerWrapper)).when(pulsar).getLoadManager();
+        doReturn(loadManager).when(loadManagerWrapper).get();
+        doReturn(channel).when(loadManager).getServiceUnitStateChannel();
+        doReturn(true).when(channel).isOwner(any());
 
 
         bundleStats = new LinkedHashMap<>();
@@ -109,7 +122,7 @@ void setup() {
     }
 
     public void testNamespaceBundleSplitConditionThreshold() {
-        config.setLoadBalancerNamespaceBundleSplitConditionThreshold(0);
+        config.setLoadBalancerNamespaceBundleSplitConditionHitCountThreshold(0);
         bundleStats.values().forEach(v -> v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() + 1);
         var strategy = new DefaultNamespaceBundleSplitStrategyImpl(new SplitCounter());
         var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
@@ -118,7 +131,7 @@ public void testNamespaceBundleSplitConditionThreshold() {
 
 
     public void testNotEnoughTopics() {
-        config.setLoadBalancerNamespaceBundleSplitConditionThreshold(0);
+        config.setLoadBalancerNamespaceBundleSplitConditionHitCountThreshold(0);
         bundleStats.values().forEach(v -> v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() + 1);
         var strategy = new DefaultNamespaceBundleSplitStrategyImpl(new SplitCounter());
         bundleStats.values().forEach(v -> v.topics = 1);
@@ -128,7 +141,7 @@ public void testNotEnoughTopics() {
     }
 
     public void testNamespaceMaximumBundles() throws Exception {
-        config.setLoadBalancerNamespaceBundleSplitConditionThreshold(0);
+        config.setLoadBalancerNamespaceBundleSplitConditionHitCountThreshold(0);
         bundleStats.values().forEach(v -> v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() + 1);
         var strategy = new DefaultNamespaceBundleSplitStrategyImpl(new SplitCounter());
         doReturn(config.getLoadBalancerNamespaceMaximumBundles()).when(namespaceService).getBundleCount(any());
@@ -138,7 +151,7 @@ public void testNamespaceMaximumBundles() throws Exception {
     }
 
     public void testEmptyBundleStats() {
-        config.setLoadBalancerNamespaceBundleSplitConditionThreshold(0);
+        config.setLoadBalancerNamespaceBundleSplitConditionHitCountThreshold(0);
         bundleStats.values().forEach(v -> v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() + 1);
         var strategy = new DefaultNamespaceBundleSplitStrategyImpl(new SplitCounter());
         bundleStats.clear();
@@ -147,9 +160,21 @@ public void testEmptyBundleStats() {
         assertEquals(actual, expected);
     }
 
+    public void testNoBundleOwner() {
+        var counter = spy(new SplitCounter());
+        config.setLoadBalancerNamespaceBundleSplitConditionHitCountThreshold(0);
+        bundleStats.values().forEach(v -> v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() + 1);
+        doReturn(false).when(channel).isOwner(any());
+        var strategy = new DefaultNamespaceBundleSplitStrategyImpl(counter);
+        var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
+        var expected = Set.of();
+        assertEquals(actual, expected);
+        verify(counter, times(2)).update(eq(SplitDecision.Label.Failure), eq(Unknown));
+    }
+
     public void testError() throws Exception {
         var counter = spy(new SplitCounter());
-        config.setLoadBalancerNamespaceBundleSplitConditionThreshold(0);
+        config.setLoadBalancerNamespaceBundleSplitConditionHitCountThreshold(0);
         bundleStats.values().forEach(v -> v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() + 1);
         var strategy = new DefaultNamespaceBundleSplitStrategyImpl(counter);
         doThrow(new RuntimeException()).when(namespaceService).getBundleCount(any());
@@ -162,7 +187,7 @@ public void testError() throws Exception {
     public void testMaxMsgRate() {
         var counter = spy(new SplitCounter());
         var strategy = new DefaultNamespaceBundleSplitStrategyImpl(counter);
-        int threshold = config.getLoadBalancerNamespaceBundleSplitConditionThreshold();
+        int threshold = config.getLoadBalancerNamespaceBundleSplitConditionHitCountThreshold();
         bundleStats.values().forEach(v -> {
             v.msgRateOut = config.getLoadBalancerNamespaceBundleMaxMsgRate() / 2 + 1;
             v.msgRateIn = config.getLoadBalancerNamespaceBundleMaxMsgRate() / 2 + 1;
@@ -193,7 +218,7 @@ public void testMaxMsgRate() {
     public void testMaxTopics() {
         var counter = spy(new SplitCounter());
         var strategy = new DefaultNamespaceBundleSplitStrategyImpl(counter);
-        int threshold = config.getLoadBalancerNamespaceBundleSplitConditionThreshold();
+        int threshold = config.getLoadBalancerNamespaceBundleSplitConditionHitCountThreshold();
         bundleStats.values().forEach(v -> v.topics = config.getLoadBalancerNamespaceBundleMaxTopics() + 1);
         for (int i = 0; i < threshold + 2; i++) {
             var actual = strategy.findBundlesToSplit(loadManagerContext, pulsar);
@@ -221,7 +246,7 @@ public void testMaxTopics() {
     public void testMaxSessions() {
         var counter = spy(new SplitCounter());
         var strategy = new DefaultNamespaceBundleSplitStrategyImpl(counter);
-        int threshold = config.getLoadBalancerNamespaceBundleSplitConditionThreshold();
+        int threshold = config.getLoadBalancerNamespaceBundleSplitConditionHitCountThreshold();
         bundleStats.values().forEach(v -> {
             v.producerCount = config.getLoadBalancerNamespaceBundleMaxSessions() / 2 + 1;
             v.consumerCount = config.getLoadBalancerNamespaceBundleMaxSessions() / 2 + 1;
@@ -252,7 +277,7 @@ public void testMaxSessions() {
     public void testMaxBandwidthMbytes() {
         var counter = spy(new SplitCounter());
         var strategy = new DefaultNamespaceBundleSplitStrategyImpl(counter);
-        int threshold = config.getLoadBalancerNamespaceBundleSplitConditionThreshold();
+        int threshold = config.getLoadBalancerNamespaceBundleSplitConditionHitCountThreshold();
         bundleStats.values().forEach(v -> {
             v.msgThroughputOut = config.getLoadBalancerNamespaceBundleMaxBandwidthMbytes() * 1024 * 1024 / 2 + 1;
             v.msgThroughputIn = config.getLoadBalancerNamespaceBundleMaxBandwidthMbytes() * 1024 * 1024 / 2 + 1;
diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/NamespaceBundleStats.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/NamespaceBundleStats.java
index dc1cdd3d5cbce..cd364990842a1 100644
--- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/NamespaceBundleStats.java
+++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/NamespaceBundleStats.java
@@ -20,10 +20,12 @@
 
 import java.io.Serializable;
 import lombok.EqualsAndHashCode;
+import lombok.ToString;
 
 /**
  */
 @EqualsAndHashCode
+@ToString
 public class NamespaceBundleStats implements Comparable, Serializable {
 
     public double msgRateIn;
diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/ResourceUsage.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/ResourceUsage.java
index eed8b33b39178..ae12e931071bc 100644
--- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/ResourceUsage.java
+++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/policies/data/loadbalancer/ResourceUsage.java
@@ -19,11 +19,13 @@
 package org.apache.pulsar.policies.data.loadbalancer;
 
 import lombok.EqualsAndHashCode;
+import lombok.ToString;
 
 /**
  * POJO used to represent any system specific resource usage this is the format that load manager expects it in.
  */
 @EqualsAndHashCode
+@ToString
 public class ResourceUsage {
     public final double usage;
     public final double limit;

From 38485e09ce1b01803bd6d5ab95901f6284428685 Mon Sep 17 00:00:00 2001
From: tison 
Date: Wed, 29 Mar 2023 17:55:48 +0800
Subject: [PATCH 163/174] [improve][build] Create source jar for
 pulsar-client-all shaded jar (#19956)

Signed-off-by: tison 
---
 pulsar-client-all/pom.xml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml
index 00da6e4895097..73621954e1fb8 100644
--- a/pulsar-client-all/pom.xml
+++ b/pulsar-client-all/pom.xml
@@ -141,6 +141,8 @@
               shade
             
             
+              true
+              true
               true
               true
               false

From 07acdbc8541c1eeb724361713e7fd136d4c93fc3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= 
Date: Wed, 29 Mar 2023 14:38:02 +0200
Subject: [PATCH 164/174] [fix][sec] Fix transitive critical CVEs in
 file-system tiered storage (#19957)

---
 pom.xml                            |  6 +++---
 tiered-storage/file-system/pom.xml | 25 -------------------------
 2 files changed, 3 insertions(+), 28 deletions(-)

diff --git a/pom.xml b/pom.xml
index ac640f1a6a05f..4c489355281ba 100644
--- a/pom.xml
+++ b/pom.xml
@@ -180,8 +180,8 @@ flexible messaging model and an intuitive client API.
     0.3.2-patch11
     2.7.5
     0.4.4-hotfix1
-    3.3.3
-    2.4.7
+    3.3.5
+    2.4.10
     1.2.4
     8.5.2
     363
@@ -257,7 +257,7 @@ flexible messaging model and an intuitive client API.
     3.1
     4.2.0
     1.2.22
-    1.5.3
+    1.5.4
     5.4.0
     2.33.2
 
diff --git a/tiered-storage/file-system/pom.xml b/tiered-storage/file-system/pom.xml
index 1d8aab48cbff8..745da1a95c70d 100644
--- a/tiered-storage/file-system/pom.xml
+++ b/tiered-storage/file-system/pom.xml
@@ -53,31 +53,6 @@
             
         
         
-        
-            com.sun.jersey
-            jersey-json
-            
-            1.19
-            
-                
-                    org.codehaus.jackson
-                    jackson-core-asl
-                
-                
-                    org.codehaus.jackson
-                    jackson-mapper-asl
-                
-                
-                    org.codehaus.jackson
-                    jackson-jaxrs
-                
-                
-                    org.codehaus.jackson
-                    jackson-xc
-                
-            
-        
-        
         
             org.apache.avro
             avro

From 55523ac8f31fd6d54aacba326edef1f53028877e Mon Sep 17 00:00:00 2001
From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com>
Date: Wed, 29 Mar 2023 13:55:22 -0700
Subject: [PATCH 165/174] [improve][io] KCA: flag to force optional primitive
 schemas (#19951)

Motivation
Kafka's schema has "Optional" flag that used there to validate data/allow nulls.
Pulsar's schema does not have such info which makes conversion to kafka schema lossy.

Modifications
Added a config parameter that lets one force primitive schemas into optional ones.
KV schema is always optional.

Default is false, to match existing behavior.
---
 .../io/kafka/connect/KafkaConnectSink.java    |  16 +-
 .../connect/PulsarKafkaConnectSinkConfig.java |   6 +
 .../schema/PulsarSchemaToKafkaSchema.java     |  55 ++++++-
 .../kafka/connect/KafkaConnectSinkTest.java   |  41 +++--
 .../PulsarSchemaToKafkaSchemaTest.java        | 149 +++++++++++++-----
 5 files changed, 203 insertions(+), 64 deletions(-)

diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java
index 475724cc4e545..10efc91ccdaad 100644
--- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java
+++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java
@@ -90,6 +90,7 @@ public class KafkaConnectSink implements Sink {
     private PulsarKafkaConnectSinkConfig kafkaSinkConfig;
 
     protected String topicName;
+    protected boolean useOptionalPrimitives;
 
     private boolean sanitizeTopicName = false;
     // Thi is a workaround for https://github.com/apache/pulsar/issues/19922
@@ -164,6 +165,7 @@ public void open(Map config, SinkContext ctx) throws Exception {
         unwrapKeyValueIfAvailable = kafkaSinkConfig.isUnwrapKeyValueIfAvailable();
         sanitizeTopicName = kafkaSinkConfig.isSanitizeTopicName();
         collapsePartitionedTopics = kafkaSinkConfig.isCollapsePartitionedTopics();
+        useOptionalPrimitives = kafkaSinkConfig.isUseOptionalPrimitives();
 
         useIndexAsOffset = kafkaSinkConfig.isUseIndexAsOffset();
         maxBatchBitsForOffset = kafkaSinkConfig.getMaxBatchBitsForOffset();
@@ -446,8 +448,11 @@ protected SinkRecord toSinkRecord(Record sourceRecord) {
                 && sourceRecord.getSchema().getSchemaInfo() != null
                 && sourceRecord.getSchema().getSchemaInfo().getType() == SchemaType.KEY_VALUE) {
             KeyValueSchema kvSchema = (KeyValueSchema) sourceRecord.getSchema();
-            keySchema = PulsarSchemaToKafkaSchema.getKafkaConnectSchema(kvSchema.getKeySchema());
-            valueSchema = PulsarSchemaToKafkaSchema.getKafkaConnectSchema(kvSchema.getValueSchema());
+            // Assume Key_Value schema's key and value are always optional
+            keySchema = PulsarSchemaToKafkaSchema
+                    .getOptionalKafkaConnectSchema(kvSchema.getKeySchema(), useOptionalPrimitives);
+            valueSchema = PulsarSchemaToKafkaSchema
+                    .getOptionalKafkaConnectSchema(kvSchema.getValueSchema(), useOptionalPrimitives);
 
             Object nativeObject = sourceRecord.getValue().getNativeObject();
 
@@ -464,12 +469,13 @@ protected SinkRecord toSinkRecord(Record sourceRecord) {
         } else {
             if (sourceRecord.getMessage().get().hasBase64EncodedKey()) {
                 key = sourceRecord.getMessage().get().getKeyBytes();
-                keySchema = Schema.BYTES_SCHEMA;
+                keySchema = useOptionalPrimitives ? Schema.OPTIONAL_BYTES_SCHEMA : Schema.BYTES_SCHEMA;
             } else {
                 key = sourceRecord.getKey().orElse(null);
-                keySchema = Schema.STRING_SCHEMA;
+                keySchema = useOptionalPrimitives ? Schema.OPTIONAL_STRING_SCHEMA : Schema.STRING_SCHEMA;
             }
-            valueSchema = PulsarSchemaToKafkaSchema.getKafkaConnectSchema(sourceRecord.getSchema());
+            valueSchema = PulsarSchemaToKafkaSchema
+                    .getKafkaConnectSchema(sourceRecord.getSchema(), useOptionalPrimitives);
             value = KafkaConnectData.getKafkaConnectData(sourceRecord.getValue().getNativeObject(), valueSchema);
         }
 
diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaConnectSinkConfig.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaConnectSinkConfig.java
index 2525081a41ebb..96519e63e0afa 100644
--- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaConnectSinkConfig.java
+++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/PulsarKafkaConnectSinkConfig.java
@@ -99,6 +99,12 @@ public class PulsarKafkaConnectSinkConfig implements Serializable {
             help = "Supply kafka record with topic name without -partition- suffix for partitioned topics.")
     private boolean collapsePartitionedTopics = false;
 
+    @FieldDoc(
+            defaultValue = "false",
+            help = "Pulsar schema does not contain information whether the Schema is optional, Kafka's does. \n"
+                    + "This provides a way to force all primitive schemas to be optional for Kafka. \n")
+    private boolean useOptionalPrimitives = false;
+
     public static PulsarKafkaConnectSinkConfig load(String yamlFile) throws IOException {
         ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
         return mapper.readValue(new File(yamlFile), PulsarKafkaConnectSinkConfig.class);
diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/PulsarSchemaToKafkaSchema.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/PulsarSchemaToKafkaSchema.java
index faf28585e8aed..21a0a0f42e025 100644
--- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/PulsarSchemaToKafkaSchema.java
+++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/PulsarSchemaToKafkaSchema.java
@@ -115,11 +115,15 @@ public Schema schema() {
     }
 
     private static final ImmutableMap pulsarSchemaTypeToKafkaSchema;
+    private static final ImmutableMap pulsarSchemaTypeToOptionalKafkaSchema;
     private static final ImmutableSet kafkaLogicalSchemas;
     private static final AvroData avroData = new AvroData(1000);
     private static final Cache schemaCache =
             CacheBuilder.newBuilder().maximumSize(10000)
                     .expireAfterAccess(30, TimeUnit.MINUTES).build();
+    private static final Cache optionalSchemaCache =
+            CacheBuilder.newBuilder().maximumSize(1000)
+                    .expireAfterAccess(30, TimeUnit.MINUTES).build();
 
     static {
         pulsarSchemaTypeToKafkaSchema = ImmutableMap.builder()
@@ -134,6 +138,17 @@ public Schema schema() {
                 .put(SchemaType.BYTES, Schema.BYTES_SCHEMA)
                 .put(SchemaType.DATE, Date.SCHEMA)
                 .build();
+        pulsarSchemaTypeToOptionalKafkaSchema = ImmutableMap.builder()
+                .put(SchemaType.BOOLEAN, Schema.OPTIONAL_BOOLEAN_SCHEMA)
+                .put(SchemaType.INT8, Schema.OPTIONAL_INT8_SCHEMA)
+                .put(SchemaType.INT16, Schema.OPTIONAL_INT16_SCHEMA)
+                .put(SchemaType.INT32, Schema.OPTIONAL_INT32_SCHEMA)
+                .put(SchemaType.INT64, Schema.OPTIONAL_INT64_SCHEMA)
+                .put(SchemaType.FLOAT, Schema.OPTIONAL_FLOAT32_SCHEMA)
+                .put(SchemaType.DOUBLE, Schema.OPTIONAL_FLOAT64_SCHEMA)
+                .put(SchemaType.STRING, Schema.OPTIONAL_STRING_SCHEMA)
+                .put(SchemaType.BYTES, Schema.OPTIONAL_BYTES_SCHEMA)
+                .build();
         kafkaLogicalSchemas = ImmutableSet.builder()
                 .add(Timestamp.LOGICAL_NAME)
                 .add(Date.LOGICAL_NAME)
@@ -153,12 +168,33 @@ private static org.apache.avro.Schema parseAvroSchema(String schemaJson) {
         return parser.parse(schemaJson);
     }
 
-    public static Schema getOptionalKafkaConnectSchema(org.apache.pulsar.client.api.Schema pulsarSchema) {
-        Schema s = getKafkaConnectSchema(pulsarSchema);
-        return new OptionalForcingSchema(s);
+    public static Schema makeOptional(Schema s) {
+        if (s == null || s.isOptional()) {
+            return s;
+        }
+
+        String logicalSchemaName = s.name();
+        if (kafkaLogicalSchemas.contains(logicalSchemaName)) {
+            return s;
+        }
+
+        try {
+            return optionalSchemaCache.get(s, () -> new OptionalForcingSchema(s));
+        } catch (ExecutionException | UncheckedExecutionException | ExecutionError ee) {
+            String msg = "Failed to create optional schema for " + s;
+            log.error(msg);
+            throw new IllegalStateException(msg, ee);
+        }
     }
 
-    public static Schema getKafkaConnectSchema(org.apache.pulsar.client.api.Schema pulsarSchema) {
+    public static Schema getOptionalKafkaConnectSchema(org.apache.pulsar.client.api.Schema pulsarSchema,
+                                               boolean useOptionalPrimitives) {
+        return makeOptional(getKafkaConnectSchema(pulsarSchema, useOptionalPrimitives));
+
+    }
+
+    public static Schema getKafkaConnectSchema(org.apache.pulsar.client.api.Schema pulsarSchema,
+                                               boolean useOptionalPrimitives) {
         if (pulsarSchema == null || pulsarSchema.getSchemaInfo() == null) {
             throw logAndThrowOnUnsupportedSchema(pulsarSchema, "Schema is required.", null);
         }
@@ -191,6 +227,11 @@ public static Schema getKafkaConnectSchema(org.apache.pulsar.client.api.Schema p
             throw new IllegalStateException("Unsupported Kafka Logical Schema " + logicalSchemaName);
         }
 
+        if (useOptionalPrimitives
+                && pulsarSchemaTypeToOptionalKafkaSchema.containsKey(pulsarSchema.getSchemaInfo().getType())) {
+            return pulsarSchemaTypeToOptionalKafkaSchema.get(pulsarSchema.getSchemaInfo().getType());
+        }
+
         if (pulsarSchemaTypeToKafkaSchema.containsKey(pulsarSchema.getSchemaInfo().getType())) {
             return pulsarSchemaTypeToKafkaSchema.get(pulsarSchema.getSchemaInfo().getType());
         }
@@ -199,8 +240,10 @@ public static Schema getKafkaConnectSchema(org.apache.pulsar.client.api.Schema p
             return schemaCache.get(pulsarSchema.getSchemaInfo().getSchema(), () -> {
                 if (pulsarSchema.getSchemaInfo().getType() == SchemaType.KEY_VALUE) {
                     KeyValueSchema kvSchema = (KeyValueSchema) pulsarSchema;
-                    return SchemaBuilder.map(getKafkaConnectSchema(kvSchema.getKeySchema()),
-                                    getOptionalKafkaConnectSchema(kvSchema.getValueSchema()))
+                    return SchemaBuilder.map(
+                            makeOptional(getKafkaConnectSchema(kvSchema.getKeySchema(), useOptionalPrimitives)),
+                            makeOptional(getKafkaConnectSchema(kvSchema.getValueSchema(), useOptionalPrimitives)))
+                                .optional()
                                 .build();
                 }
                 org.apache.avro.Schema avroSchema = parseAvroSchema(
diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java
index 6ccfa3b71a2f7..5410e0bb8d664 100644
--- a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java
+++ b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java
@@ -162,7 +162,7 @@ public T answer(InvocationOnMock invocationOnMock) throws Throwable {
         }
     }
 
-    private String offsetTopicName = "persistent://my-property/my-ns/kafka-connect-sink-offset";
+    final private String offsetTopicName = "persistent://my-property/my-ns/kafka-connect-sink-offset";
 
     private Path file;
     private Map props;
@@ -797,7 +797,9 @@ public void kafkaLogicalTypesTimestampTest() {
                 .build());
 
         org.apache.kafka.connect.data.Schema kafkaSchema = PulsarSchemaToKafkaSchema
-                .getKafkaConnectSchema(schema);
+                .getKafkaConnectSchema(schema, true);
+
+        Assert.assertFalse(kafkaSchema.isOptional());
 
         java.util.Date date = getDateFromString("12/30/1999 11:12:13");
         Object connectData = KafkaConnectData
@@ -815,7 +817,9 @@ public void kafkaLogicalTypesTimeTest() {
                 .build());
 
         org.apache.kafka.connect.data.Schema kafkaSchema = PulsarSchemaToKafkaSchema
-                .getKafkaConnectSchema(schema);
+                .getKafkaConnectSchema(schema, true);
+
+        Assert.assertFalse(kafkaSchema.isOptional());
 
         java.util.Date date = getDateFromString("01/01/1970 11:12:13");
         Object connectData = KafkaConnectData
@@ -833,7 +837,9 @@ public void kafkaLogicalTypesDateTest() {
                 .build());
 
         org.apache.kafka.connect.data.Schema kafkaSchema = PulsarSchemaToKafkaSchema
-                .getKafkaConnectSchema(schema);
+                .getKafkaConnectSchema(schema, true);
+
+        Assert.assertFalse(kafkaSchema.isOptional());
 
         java.util.Date date = getDateFromString("12/31/2022 00:00:00");
         Object connectData = KafkaConnectData
@@ -854,7 +860,9 @@ public void kafkaLogicalTypesDecimalTest() {
                 .build());
 
         org.apache.kafka.connect.data.Schema kafkaSchema = PulsarSchemaToKafkaSchema
-                .getKafkaConnectSchema(schema);
+                .getKafkaConnectSchema(schema, true);
+
+        Assert.assertFalse(kafkaSchema.isOptional());
 
         Object connectData = KafkaConnectData
                 .getKafkaConnectData(Decimal.fromLogical(kafkaSchema, BigDecimal.valueOf(100L, 10)), kafkaSchema);
@@ -874,11 +882,11 @@ public void connectDataComplexAvroSchemaGenericRecordTest() {
                 getGenericRecord(value, pulsarAvroSchema));
 
         org.apache.kafka.connect.data.Schema kafkaSchema = PulsarSchemaToKafkaSchema
-                .getKafkaConnectSchema(Schema.KeyValue(pulsarAvroSchema, pulsarAvroSchema));
+                .getKafkaConnectSchema(Schema.KeyValue(pulsarAvroSchema, pulsarAvroSchema), false);
 
-        Object connectData = KafkaConnectData.getKafkaConnectData(kv, kafkaSchema);
-
-        org.apache.kafka.connect.data.ConnectSchema.validateValue(kafkaSchema, connectData);
+        Assert.assertTrue(kafkaSchema.isOptional());
+        Assert.assertTrue(kafkaSchema.keySchema().isOptional());
+        Assert.assertTrue(kafkaSchema.valueSchema().isOptional());
     }
 
     @Test
@@ -990,7 +998,8 @@ private void testPojoAsAvroAndJsonConversionToConnectData(Object pojo, AvroSchem
         Object value = pojoAsAvroRecord(pojo, pulsarAvroSchema);
 
         org.apache.kafka.connect.data.Schema kafkaSchema = PulsarSchemaToKafkaSchema
-                .getKafkaConnectSchema(pulsarAvroSchema);
+                .getKafkaConnectSchema(pulsarAvroSchema, false);
+        Assert.assertFalse(kafkaSchema.isOptional());
 
         Object connectData = KafkaConnectData.getKafkaConnectData(value, kafkaSchema);
 
@@ -999,6 +1008,18 @@ private void testPojoAsAvroAndJsonConversionToConnectData(Object pojo, AvroSchem
         Object jsonNode = pojoAsJsonNode(pojo);
         connectData = KafkaConnectData.getKafkaConnectData(jsonNode, kafkaSchema);
         org.apache.kafka.connect.data.ConnectSchema.validateValue(kafkaSchema, connectData);
+
+        kafkaSchema = PulsarSchemaToKafkaSchema
+                .getKafkaConnectSchema(pulsarAvroSchema, true);
+        Assert.assertFalse(kafkaSchema.isOptional());
+
+        connectData = KafkaConnectData.getKafkaConnectData(value, kafkaSchema);
+
+        org.apache.kafka.connect.data.ConnectSchema.validateValue(kafkaSchema, connectData);
+
+        jsonNode = pojoAsJsonNode(pojo);
+        connectData = KafkaConnectData.getKafkaConnectData(jsonNode, kafkaSchema);
+        org.apache.kafka.connect.data.ConnectSchema.validateValue(kafkaSchema, connectData);
     }
 
     private JsonNode pojoAsJsonNode(Object pojo) {
diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/PulsarSchemaToKafkaSchemaTest.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/PulsarSchemaToKafkaSchemaTest.java
index 9cc6db034c870..b236365bbb8a1 100644
--- a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/PulsarSchemaToKafkaSchemaTest.java
+++ b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/PulsarSchemaToKafkaSchemaTest.java
@@ -32,6 +32,7 @@
 import org.apache.pulsar.client.impl.schema.KeyValueSchemaImpl;
 import org.apache.pulsar.io.kafka.connect.schema.KafkaConnectData;
 import org.apache.pulsar.io.kafka.connect.schema.PulsarSchemaToKafkaSchema;
+import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
 import java.math.BigInteger;
@@ -39,6 +40,8 @@
 import java.util.Map;
 
 import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
 
 /**
  * Test the conversion of PulsarSchema To KafkaSchema\.
@@ -132,101 +135,134 @@ static class ComplexStruct {
         String[] stringArr;
     }
 
-    @Test
-    public void bytesSchemaTest() {
+    @DataProvider(name = "useOptionalPrimitives")
+    public static Object[][] useOptionalPrimitives() {
+        return new Object[][] {
+                {true},
+                {false}
+        };
+    }
+
+    @Test(dataProvider = "useOptionalPrimitives")
+    public void bytesSchemaTest(boolean useOptionalPrimitives) {
         org.apache.kafka.connect.data.Schema kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.BYTES);
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.BYTES, useOptionalPrimitives);
         assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.BYTES);
+        assertEquals(useOptionalPrimitives, kafkaSchema.isOptional());
 
         kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.BYTEBUFFER);
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.BYTEBUFFER, useOptionalPrimitives);
         assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.BYTES);
+        assertEquals(useOptionalPrimitives, kafkaSchema.isOptional());
     }
 
-    @Test
-    public void stringSchemaTest() {
+    @Test(dataProvider = "useOptionalPrimitives")
+    public void stringSchemaTest(boolean useOptionalPrimitives) {
         org.apache.kafka.connect.data.Schema kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.STRING);
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.STRING, useOptionalPrimitives);
         assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.STRING);
+        assertEquals(useOptionalPrimitives, kafkaSchema.isOptional());
     }
 
-    @Test
-    public void booleanSchemaTest() {
+    @Test(dataProvider = "useOptionalPrimitives")
+    public void booleanSchemaTest(boolean useOptionalPrimitives) {
         org.apache.kafka.connect.data.Schema kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.BOOL);
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.BOOL, useOptionalPrimitives);
         assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.BOOLEAN);
+        assertEquals(useOptionalPrimitives, kafkaSchema.isOptional());
     }
 
-    @Test
-    public void int8SchemaTest() {
+    @Test(dataProvider = "useOptionalPrimitives")
+    public void int8SchemaTest(boolean useOptionalPrimitives) {
         org.apache.kafka.connect.data.Schema kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.INT8);
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.INT8, useOptionalPrimitives);
         assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.INT8);
+        assertEquals(useOptionalPrimitives, kafkaSchema.isOptional());
     }
 
-    @Test
-    public void int16SchemaTest() {
+    @Test(dataProvider = "useOptionalPrimitives")
+    public void int16SchemaTest(boolean useOptionalPrimitives) {
         org.apache.kafka.connect.data.Schema kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.INT16);
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.INT16, useOptionalPrimitives);
         assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.INT16);
+        assertEquals(useOptionalPrimitives, kafkaSchema.isOptional());
     }
 
-    @Test
-    public void int32SchemaTest() {
+    @Test(dataProvider = "useOptionalPrimitives")
+    public void int32SchemaTest(boolean useOptionalPrimitives) {
         org.apache.kafka.connect.data.Schema kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.INT32);
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.INT32, useOptionalPrimitives);
         assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.INT32);
+        assertEquals(useOptionalPrimitives, kafkaSchema.isOptional());
     }
 
-    @Test
-    public void int64SchemaTest() {
+    @Test(dataProvider = "useOptionalPrimitives")
+    public void int64SchemaTest(boolean useOptionalPrimitives) {
         org.apache.kafka.connect.data.Schema kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.INT64);
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.INT64, useOptionalPrimitives);
         assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.INT64);
+        assertEquals(useOptionalPrimitives, kafkaSchema.isOptional());
     }
 
-    @Test
-    public void float32SchemaTest() {
+    @Test(dataProvider = "useOptionalPrimitives")
+    public void float32SchemaTest(boolean useOptionalPrimitives) {
         org.apache.kafka.connect.data.Schema kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.FLOAT);
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.FLOAT, useOptionalPrimitives);
         assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.FLOAT32);
+        assertEquals(useOptionalPrimitives, kafkaSchema.isOptional());
     }
 
-    @Test
-    public void float64SchemaTest() {
+    @Test(dataProvider = "useOptionalPrimitives")
+    public void float64SchemaTest(boolean useOptionalPrimitives) {
         org.apache.kafka.connect.data.Schema kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.DOUBLE);
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.DOUBLE, useOptionalPrimitives);
         assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.FLOAT64);
+        assertEquals(useOptionalPrimitives, kafkaSchema.isOptional());
     }
 
-    @Test
-    public void kvBytesSchemaTest() {
+    @Test(dataProvider = "useOptionalPrimitives")
+    public void kvBytesSchemaTest(boolean useOptionalPrimitives) {
         org.apache.kafka.connect.data.Schema kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.KV_BYTES());
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.KV_BYTES(), useOptionalPrimitives);
         assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.MAP);
         assertEquals(kafkaSchema.keySchema().type(), org.apache.kafka.connect.data.Schema.Type.BYTES);
         assertEquals(kafkaSchema.valueSchema().type(), org.apache.kafka.connect.data.Schema.Type.BYTES);
+        assertTrue(kafkaSchema.isOptional());
+
+        // key and value are always optional
+        assertTrue(kafkaSchema.keySchema().isOptional());
+        assertTrue(kafkaSchema.valueSchema().isOptional());
     }
 
     @Test
     public void kvBytesIntSchemaTests() {
         Schema pulsarKvSchema = KeyValueSchemaImpl.of(Schema.STRING, Schema.INT64);
         org.apache.kafka.connect.data.Schema kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(pulsarKvSchema);
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(pulsarKvSchema, false);
         assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.MAP);
         assertEquals(kafkaSchema.keySchema().type(), org.apache.kafka.connect.data.Schema.Type.STRING);
         assertEquals(kafkaSchema.valueSchema().type(), org.apache.kafka.connect.data.Schema.Type.INT64);
+        assertTrue(kafkaSchema.isOptional());
+
+        // key and value are always optional
+        assertTrue(kafkaSchema.keySchema().isOptional());
+        assertTrue(kafkaSchema.valueSchema().isOptional());
     }
 
     @Test
     public void avroSchemaTest() {
         AvroSchema pulsarAvroSchema = AvroSchema.of(StructWithAnnotations.class);
         org.apache.kafka.connect.data.Schema kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(pulsarAvroSchema);
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(pulsarAvroSchema, false);
+        org.apache.kafka.connect.data.Schema kafkaSchemaOpt =
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(pulsarAvroSchema, true);
         assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.STRUCT);
         assertEquals(kafkaSchema.fields().size(), STRUCT_FIELDS.size());
         for (String name: STRUCT_FIELDS) {
             assertEquals(kafkaSchema.field(name).name(), name);
+            // set by avro schema
+            assertEquals(kafkaSchema.field(name).schema().isOptional(),
+                    kafkaSchemaOpt.field(name).schema().isOptional());
         }
     }
 
@@ -234,11 +270,16 @@ public void avroSchemaTest() {
     public void avroComplexSchemaTest() {
         AvroSchema pulsarAvroSchema = AvroSchema.of(ComplexStruct.class);
         org.apache.kafka.connect.data.Schema kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(pulsarAvroSchema);
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(pulsarAvroSchema, false);
+        org.apache.kafka.connect.data.Schema kafkaSchemaOpt =
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(pulsarAvroSchema, true);
         assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.STRUCT);
         assertEquals(kafkaSchema.fields().size(), COMPLEX_STRUCT_FIELDS.size());
         for (String name: COMPLEX_STRUCT_FIELDS) {
             assertEquals(kafkaSchema.field(name).name(), name);
+            // set by avro schema
+            assertEquals(kafkaSchema.field(name).schema().isOptional(),
+                    kafkaSchemaOpt.field(name).schema().isOptional());
         }
     }
 
@@ -250,11 +291,16 @@ public void jsonSchemaTest() {
                 .withAlwaysAllowNull(false)
                 .build());
         org.apache.kafka.connect.data.Schema kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(jsonSchema);
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(jsonSchema, false);
+        org.apache.kafka.connect.data.Schema kafkaSchemaOpt =
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(jsonSchema, true);
         assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.STRUCT);
         assertEquals(kafkaSchema.fields().size(), STRUCT_FIELDS.size());
         for (String name: STRUCT_FIELDS) {
             assertEquals(kafkaSchema.field(name).name(), name);
+            // set by schema
+            assertEquals(kafkaSchema.field(name).schema().isOptional(),
+                    kafkaSchemaOpt.field(name).schema().isOptional());
         }
     }
 
@@ -266,11 +312,27 @@ public void jsonComplexSchemaTest() {
                         .withAlwaysAllowNull(false)
                         .build());
         org.apache.kafka.connect.data.Schema kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(jsonSchema);
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(jsonSchema, false);
         assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.STRUCT);
         assertEquals(kafkaSchema.fields().size(), COMPLEX_STRUCT_FIELDS.size());
         for (String name: COMPLEX_STRUCT_FIELDS) {
             assertEquals(kafkaSchema.field(name).name(), name);
+            assertFalse(kafkaSchema.field(name).schema().isOptional());
+        }
+
+        kafkaSchema =
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(jsonSchema, true);
+        assertEquals(kafkaSchema.type(), org.apache.kafka.connect.data.Schema.Type.STRUCT);
+        assertEquals(kafkaSchema.fields().size(), COMPLEX_STRUCT_FIELDS.size());
+        for (String name: COMPLEX_STRUCT_FIELDS) {
+            assertEquals(kafkaSchema.field(name).name(), name);
+            assertFalse(kafkaSchema.field(name).schema().isOptional());
+
+            if (kafkaSchema.field(name).schema().type().isPrimitive()) {
+                // false because .withAlwaysAllowNull(false), avroschema values are used
+                assertFalse(kafkaSchema.field(name).schema().isOptional(),
+                        kafkaSchema.field(name).schema().type().getName());
+            }
         }
     }
 
@@ -308,39 +370,40 @@ public void castToKafkaSchemaTest() {
     @Test
     public void dateSchemaTest() {
         org.apache.kafka.connect.data.Schema kafkaSchema =
-                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.DATE);
+                PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.DATE, true);
         assertEquals(kafkaSchema.type(), Date.SCHEMA.type());
+        assertFalse(kafkaSchema.isOptional());
     }
 
     // not supported schemas below:
     @Test(expectedExceptions = IllegalStateException.class)
     public void timeSchemaTest() {
-        PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.TIME);
+        PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.TIME, false);
     }
 
     @Test(expectedExceptions = IllegalStateException.class)
     public void timestampSchemaTest() {
-        PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.TIMESTAMP);
+        PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.TIMESTAMP, false);
     }
 
     @Test(expectedExceptions = IllegalStateException.class)
     public void instantSchemaTest() {
-        PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.INSTANT);
+        PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.INSTANT, false);
     }
 
     @Test(expectedExceptions = IllegalStateException.class)
     public void localDateSchemaTest() {
-        PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.LOCAL_DATE);
+        PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.LOCAL_DATE, false);
     }
 
     @Test(expectedExceptions = IllegalStateException.class)
     public void localTimeSchemaTest() {
-        PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.LOCAL_TIME);
+        PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.LOCAL_TIME, false);
     }
 
     @Test(expectedExceptions = IllegalStateException.class)
     public void localDatetimeSchemaTest() {
-        PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.LOCAL_DATE_TIME);
+        PulsarSchemaToKafkaSchema.getKafkaConnectSchema(Schema.LOCAL_DATE_TIME, false);
     }
 
 }

From 68c10eed7604aa3dcc3a6d8b548575e99b94dca2 Mon Sep 17 00:00:00 2001
From: Cong Zhao 
Date: Thu, 30 Mar 2023 16:32:13 +0800
Subject: [PATCH 166/174] [feat][broker][PIP-195] Add metrics for bucket
 delayed message tracker (#19716)

---
 .../pulsar/broker/ServiceConfiguration.java   |   2 +-
 .../delayed/DelayedDeliveryTracker.java       |   6 -
 .../InMemoryDelayedDeliveryTracker.java       |   5 -
 .../BookkeeperBucketSnapshotStorage.java      |  15 +-
 .../bucket/BucketDelayedDeliveryTracker.java  |  74 +++++++--
 .../BucketDelayedMessageIndexStats.java       | 146 ++++++++++++++++++
 .../delayed/bucket/ImmutableBucket.java       |  28 +++-
 .../broker/delayed/bucket/MutableBucket.java  |   6 +-
 ...PersistentDispatcherMultipleConsumers.java |  15 +-
 .../persistent/PersistentSubscription.java    |   3 +
 .../service/persistent/PersistentTopic.java   |   9 ++
 .../prometheus/AggregatedNamespaceStats.java  |  19 +++
 .../AggregatedSubscriptionStats.java          |   3 +
 .../prometheus/NamespaceStatsAggregator.java  |  13 +-
 .../broker/stats/prometheus/TopicStats.java   |  22 +++
 .../delayed/MockBucketSnapshotStorage.java    |   2 +-
 .../BucketDelayedDeliveryTrackerTest.java     |  23 +--
 .../persistent/BucketDelayedDeliveryTest.java | 143 +++++++++++++++++
 .../data/stats/SubscriptionStatsImpl.java     |  12 ++
 .../policies/data/stats/TopicMetricBean.java  |  30 ++++
 .../policies/data/stats/TopicStatsImpl.java   |  13 ++
 21 files changed, 532 insertions(+), 57 deletions(-)
 create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndexStats.java
 create mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicMetricBean.java

diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index 03e0fec0a5346..52cebe15f6a5c 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -366,7 +366,7 @@ The delayed message index time step(in seconds) in per bucket snapshot segment,
     private int delayedDeliveryMaxTimeStepPerBucketSnapshotSegmentSeconds = 300;
 
     @FieldContext(category = CATEGORY_SERVER, doc = """
-            The max number of delayed message index in per bucket snapshot segment, -1 means no limitation\
+            The max number of delayed message index in per bucket snapshot segment, -1 means no limitation, \
             after reaching the max number limitation, the snapshot segment will be cut off.""")
     private int delayedDeliveryMaxIndexesPerBucketSnapshotSegment = 5000;
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java
index 3cc2da8db1e4d..78229fef25a5a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java
@@ -67,12 +67,6 @@ public interface DelayedDeliveryTracker extends AutoCloseable {
      */
     boolean shouldPauseAllDeliveries();
 
-    /**
-     * Tells whether this DelayedDeliveryTracker contains this message index,
-     * if the tracker is not supported it or disabled this feature also will return false.
-     */
-    boolean containsMessage(long ledgerId, long entryId);
-
     /**
      *  Reset tick time use zk policies cache.
      * @param tickTime
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java
index 8de6ee58e2ce5..58358b06a46bb 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java
@@ -178,11 +178,6 @@ && getNumberOfDelayedMessages() >= fixedDelayDetectionLookahead
                 && !hasMessageAvailable();
     }
 
-    @Override
-    public boolean containsMessage(long ledgerId, long entryId) {
-        return false;
-    }
-
     protected long nextDeliveryTime() {
         return priorityQueue.peekN1();
     }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java
index 08202bb19155d..e7d4f9301dd36 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java
@@ -67,7 +67,7 @@ public CompletableFuture createBucketSnapshot(SnapshotMetadata snapshotMet
     @Override
     public CompletableFuture getBucketSnapshotMetadata(long bucketId) {
         return openLedger(bucketId).thenCompose(
-                ledgerHandle -> getLedgerEntryThenCloseLedger(ledgerHandle, 0, 0).
+                ledgerHandle -> getLedgerEntry(ledgerHandle, 0, 0).
                         thenApply(entryEnumeration -> parseSnapshotMetadataEntry(entryEnumeration.nextElement())));
     }
 
@@ -75,17 +75,13 @@ public CompletableFuture getBucketSnapshotMetadata(long bucket
     public CompletableFuture> getBucketSnapshotSegment(long bucketId, long firstSegmentEntryId,
                                                                              long lastSegmentEntryId) {
         return openLedger(bucketId).thenCompose(
-                ledgerHandle -> getLedgerEntryThenCloseLedger(ledgerHandle, firstSegmentEntryId,
+                ledgerHandle -> getLedgerEntry(ledgerHandle, firstSegmentEntryId,
                         lastSegmentEntryId).thenApply(this::parseSnapshotSegmentEntries));
     }
 
     @Override
     public CompletableFuture getBucketSnapshotLength(long bucketId) {
-        return openLedger(bucketId).thenApply(ledgerHandle -> {
-            long length = ledgerHandle.getLength();
-            closeLedger(ledgerHandle);
-            return length;
-        });
+        return openLedger(bucketId).thenApply(LedgerHandle::getLength);
     }
 
     @Override
@@ -212,8 +208,8 @@ private CompletableFuture addEntry(LedgerHandle ledgerHandle, byte[] data)
         });
     }
 
-    CompletableFuture> getLedgerEntryThenCloseLedger(LedgerHandle ledger,
-                                                                              long firstEntryId, long lastEntryId) {
+    CompletableFuture> getLedgerEntry(LedgerHandle ledger,
+                                                               long firstEntryId, long lastEntryId) {
         final CompletableFuture> future = new CompletableFuture<>();
         ledger.asyncReadEntries(firstEntryId, lastEntryId,
                 (rc, handle, entries, ctx) -> {
@@ -222,7 +218,6 @@ CompletableFuture> getLedgerEntryThenCloseLedger(Ledger
                     } else {
                         future.complete(entries);
                     }
-                    closeLedger(handle);
                 }, null
         );
         return future;
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
index ef7be187cec3d..a34bd51af98e4 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java
@@ -57,6 +57,7 @@
 import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat;
 import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex;
 import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers;
+import org.apache.pulsar.common.policies.data.stats.TopicMetricBean;
 import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue;
 import org.roaringbitmap.RoaringBitmap;
@@ -69,6 +70,10 @@ public class BucketDelayedDeliveryTracker extends AbstractDelayedDeliveryTracker
 
     static final int AsyncOperationTimeoutSeconds = 60;
 
+    private static final Long INVALID_BUCKET_ID = -1L;
+
+    private static final int MAX_MERGE_NUM = 4;
+
     private final long minIndexCountPerBucket;
 
     private final long timeStepPerBucketSnapshotSegmentInMillis;
@@ -93,9 +98,7 @@ public class BucketDelayedDeliveryTracker extends AbstractDelayedDeliveryTracker
 
     private final Table snapshotSegmentLastIndexTable;
 
-    private static final Long INVALID_BUCKET_ID = -1L;
-
-    private static final int MAX_MERGE_NUM = 4;
+    private final BucketDelayedMessageIndexStats stats;
 
     public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher,
                                  Timer timer, long tickTimeMillis,
@@ -125,6 +128,7 @@ public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispat
         this.lastMutableBucket =
                 new MutableBucket(dispatcher.getName(), dispatcher.getCursor(), FutureUtil.Sequencer.create(),
                         bucketSnapshotStorage);
+        this.stats = new BucketDelayedMessageIndexStats();
         this.numberDelayedMessages = recoverBucketSnapshot();
     }
 
@@ -161,8 +165,9 @@ private synchronized long recoverBucketSnapshot() throws RuntimeException {
         }
 
         try {
-            FutureUtil.waitForAll(futures.values()).get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS);
+            FutureUtil.waitForAll(futures.values()).get(AsyncOperationTimeoutSeconds * 2, TimeUnit.SECONDS);
         } catch (InterruptedException | ExecutionException | TimeoutException e) {
+            log.error("[{}] Failed to recover delayed message index bucket snapshot.", dispatcher.getName(), e);
             if (e instanceof InterruptedException) {
                 Thread.currentThread().interrupt();
             }
@@ -193,7 +198,7 @@ private synchronized long recoverBucketSnapshot() throws RuntimeException {
             ImmutableBucket immutableBucket = mapEntry.getValue();
             immutableBucketMap.remove(key);
             // delete asynchronously without waiting for completion
-            immutableBucket.asyncDeleteBucketSnapshot();
+            immutableBucket.asyncDeleteBucketSnapshot(stats);
         }
 
         MutableLong numberDelayedMessages = new MutableLong(0);
@@ -246,7 +251,8 @@ private Optional findImmutableBucket(long ledgerId) {
         return Optional.ofNullable(immutableBuckets.get(ledgerId));
     }
 
-    private void afterCreateImmutableBucket(Pair immutableBucketDelayedIndexPair) {
+    private void afterCreateImmutableBucket(Pair immutableBucketDelayedIndexPair,
+                                            long startTime) {
         if (immutableBucketDelayedIndexPair != null) {
             ImmutableBucket immutableBucket = immutableBucketDelayedIndexPair.getLeft();
             immutableBuckets.put(Range.closed(immutableBucket.startLedgerId, immutableBucket.endLedgerId),
@@ -260,14 +266,19 @@ private void afterCreateImmutableBucket(Pair immu
                 CompletableFuture future = createFuture.handle((bucketId, ex) -> {
                     if (ex == null) {
                         immutableBucket.setSnapshotSegments(null);
+                        immutableBucket.asyncUpdateSnapshotLength();
                         log.info("[{}] Creat bucket snapshot finish, bucketKey: {}", dispatcher.getName(),
                                 immutableBucket.bucketKey());
+
+                        stats.recordSuccessEvent(BucketDelayedMessageIndexStats.Type.create,
+                                System.currentTimeMillis() - startTime);
+
                         return bucketId;
                     }
 
-                    //TODO Record create snapshot failed
-                    log.error("[{}] Failed to create bucket snapshot, bucketKey: {}",
-                            dispatcher.getName(), immutableBucket.bucketKey(), ex);
+                    log.error("[{}] Failed to create bucket snapshot, bucketKey: {}", dispatcher.getName(),
+                            immutableBucket.bucketKey(), ex);
+                    stats.recordFailEvent(BucketDelayedMessageIndexStats.Type.create);
 
                     // Put indexes back into the shared queue and downgrade to memory mode
                     synchronized (BucketDelayedDeliveryTracker.this) {
@@ -311,12 +322,14 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver
         if (!existBucket && ledgerId > lastMutableBucket.endLedgerId
                 && lastMutableBucket.size() >= minIndexCountPerBucket
                 && !lastMutableBucket.isEmpty()) {
+            long createStartTime = System.currentTimeMillis();
+            stats.recordTriggerEvent(BucketDelayedMessageIndexStats.Type.create);
             Pair immutableBucketDelayedIndexPair =
                     lastMutableBucket.sealBucketAndAsyncPersistent(
                             this.timeStepPerBucketSnapshotSegmentInMillis,
                             this.maxIndexesPerBucketSnapshotSegment,
                             this.sharedBucketPriorityQueue);
-            afterCreateImmutableBucket(immutableBucketDelayedIndexPair);
+            afterCreateImmutableBucket(immutableBucketDelayedIndexPair, createStartTime);
             lastMutableBucket.resetLastMutableBucketRange();
 
             if (immutableBuckets.asMapOfRanges().size() > maxNumBuckets) {
@@ -374,7 +387,7 @@ private synchronized List selectMergedBuckets(final List= 0) {
-            return values.subList(minIndex, minIndex + MAX_MERGE_NUM);
+            return values.subList(minIndex, minIndex + mergeNum);
         } else if (mergeNum > 2){
             return selectMergedBuckets(values, mergeNum - 1);
         } else {
@@ -400,6 +413,9 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot() {
         for (ImmutableBucket immutableBucket : toBeMergeImmutableBuckets) {
             immutableBucket.merging = true;
         }
+
+        long mergeStartTime = System.currentTimeMillis();
+        stats.recordTriggerEvent(BucketDelayedMessageIndexStats.Type.merge);
         return asyncMergeBucketSnapshot(toBeMergeImmutableBuckets).whenComplete((__, ex) -> {
             synchronized (this) {
                 for (ImmutableBucket immutableBucket : toBeMergeImmutableBuckets) {
@@ -409,9 +425,14 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot() {
             if (ex != null) {
                 log.error("[{}] Failed to merge bucket snapshot, bucketKeys: {}",
                         dispatcher.getName(), bucketsStr, ex);
+
+                stats.recordFailEvent(BucketDelayedMessageIndexStats.Type.merge);
             } else {
                 log.info("[{}] Merge bucket snapshot finish, bucketKeys: {}, bucketNum: {}",
                         dispatcher.getName(), bucketsStr, immutableBuckets.asMapOfRanges().size());
+
+                stats.recordSuccessEvent(BucketDelayedMessageIndexStats.Type.merge,
+                        System.currentTimeMillis() - mergeStartTime);
             }
         });
     }
@@ -436,6 +457,8 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot(List {
                         synchronized (BucketDelayedDeliveryTracker.this) {
+                            long createStartTime = System.currentTimeMillis();
+                            stats.recordTriggerEvent(BucketDelayedMessageIndexStats.Type.create);
                             Pair immutableBucketDelayedIndexPair =
                                     lastMutableBucket.createImmutableBucketAndAsyncPersistent(
                                             timeStepPerBucketSnapshotSegmentInMillis,
@@ -461,12 +484,12 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot(List {
                                         List> removeFutures =
-                                                buckets.stream().map(ImmutableBucket::asyncDeleteBucketSnapshot)
+                                                buckets.stream().map(bucket -> bucket.asyncDeleteBucketSnapshot(stats))
                                                         .toList();
                                         return FutureUtil.waitForAll(removeFutures);
                                     });
@@ -557,15 +580,17 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa
 
                     if (bucket.currentSegmentEntryId == bucket.lastSegmentEntryId) {
                         immutableBuckets.asMapOfRanges().remove(Range.closed(bucket.startLedgerId, bucket.endLedgerId));
-                        bucket.asyncDeleteBucketSnapshot();
+                        bucket.asyncDeleteBucketSnapshot(stats);
                         continue;
                     }
 
+                    long loadStartTime = System.currentTimeMillis();
+                    stats.recordTriggerEvent(BucketDelayedMessageIndexStats.Type.load);
                     bucket.asyncLoadNextBucketSnapshotEntry().thenAccept(indexList -> {
                         if (CollectionUtils.isEmpty(indexList)) {
                             immutableBuckets.asMapOfRanges()
                                     .remove(Range.closed(bucket.startLedgerId, bucket.endLedgerId));
-                            bucket.asyncDeleteBucketSnapshot();
+                            bucket.asyncDeleteBucketSnapshot(stats);
                             return;
                         }
                         DelayedMessageIndexBucketSnapshotFormat.DelayedIndex
@@ -583,9 +608,14 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa
 
                             log.error("[{}] Failed to load bucket snapshot segment, bucketKey: {}, segmentEntryId: {}",
                                     dispatcher.getName(), bucket.bucketKey(), preSegmentEntryId + 1, ex);
+
+                            stats.recordFailEvent(BucketDelayedMessageIndexStats.Type.load);
                         } else {
                             log.info("[{}] Load next bucket snapshot segment finish, bucketKey: {}, segmentEntryId: {}",
                                     dispatcher.getName(), bucket.bucketKey(), preSegmentEntryId + 1);
+
+                            stats.recordSuccessEvent(BucketDelayedMessageIndexStats.Type.load,
+                                    System.currentTimeMillis() - loadStartTime);
                         }
                     }).get(AsyncOperationTimeoutSeconds * (MaxRetryTimes + 1), TimeUnit.SECONDS);
                 } catch (Exception e) {
@@ -645,7 +675,7 @@ private CompletableFuture cleanImmutableBuckets() {
         Iterator iterator = immutableBuckets.asMapOfRanges().values().iterator();
         while (iterator.hasNext()) {
             ImmutableBucket bucket = iterator.next();
-            futures.add(bucket.clear());
+            futures.add(bucket.clear(stats));
             numberDelayedMessages -= bucket.getNumberBucketDelayedMessages();
             iterator.remove();
         }
@@ -661,7 +691,6 @@ private boolean removeIndexBit(long ledgerId, long entryId) {
                 .orElse(false);
     }
 
-    @Override
     public boolean containsMessage(long ledgerId, long entryId) {
         if (lastMutableBucket.containsMessage(ledgerId, entryId)) {
             return true;
@@ -670,4 +699,15 @@ public boolean containsMessage(long ledgerId, long entryId) {
         return findImmutableBucket(ledgerId).map(bucket -> bucket.containsMessage(ledgerId, entryId))
                 .orElse(false);
     }
+
+    public Map genTopicMetricMap() {
+        stats.recordNumOfBuckets(immutableBuckets.asMapOfRanges().size() + 1);
+        stats.recordDelayedMessageIndexLoaded(this.sharedBucketPriorityQueue.size() + this.lastMutableBucket.size());
+        MutableLong totalSnapshotLength = new MutableLong();
+        immutableBuckets.asMapOfRanges().values().forEach(immutableBucket -> {
+            totalSnapshotLength.add(immutableBucket.getSnapshotLength());
+        });
+        stats.recordBucketSnapshotSizeBytes(totalSnapshotLength.longValue());
+        return stats.genTopicMetricMap();
+    }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndexStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndexStats.java
new file mode 100644
index 0000000000000..68788c359d560
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndexStats.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.delayed.bucket;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.LongAdder;
+import org.apache.bookkeeper.mledger.util.StatsBuckets;
+import org.apache.pulsar.common.policies.data.stats.TopicMetricBean;
+
+public class BucketDelayedMessageIndexStats {
+
+    private static final long[] BUCKETS = new long[]{50, 100, 500, 1000, 5000, 30000, 60000};
+
+    enum State {
+        succeed,
+        failed,
+        all
+    }
+
+    enum Type {
+        create,
+        load,
+        delete,
+        merge
+    }
+
+    private static final String BUCKET_TOTAL_NAME = "pulsar_delayed_message_index_bucket_total";
+    private static final String INDEX_LOADED_NAME = "pulsar_delayed_message_index_loaded";
+    private static final String SNAPSHOT_SIZE_BYTES_NAME = "pulsar_delayed_message_index_bucket_snapshot_size_bytes";
+    private static final String OP_COUNT_NAME = "pulsar_delayed_message_index_bucket_op_count";
+    private static final String OP_LATENCY_NAME = "pulsar_delayed_message_index_bucket_op_latency_ms";
+
+    private final AtomicInteger delayedMessageIndexBucketTotal = new AtomicInteger();
+    private final AtomicLong delayedMessageIndexLoaded = new AtomicLong();
+    private final AtomicLong delayedMessageIndexBucketSnapshotSizeBytes = new AtomicLong();
+    private final Map delayedMessageIndexBucketOpLatencyMs = new ConcurrentHashMap<>();
+    private final Map delayedMessageIndexBucketOpCount = new ConcurrentHashMap<>();
+
+    public BucketDelayedMessageIndexStats() {
+    }
+
+    public Map genTopicMetricMap() {
+        Map metrics = new HashMap<>();
+
+        metrics.put(BUCKET_TOTAL_NAME,
+                new TopicMetricBean(BUCKET_TOTAL_NAME, delayedMessageIndexBucketTotal.get(), null));
+
+        metrics.put(INDEX_LOADED_NAME,
+                new TopicMetricBean(INDEX_LOADED_NAME, delayedMessageIndexLoaded.get(), null));
+
+        metrics.put(SNAPSHOT_SIZE_BYTES_NAME,
+                new TopicMetricBean(SNAPSHOT_SIZE_BYTES_NAME, delayedMessageIndexBucketSnapshotSizeBytes.get(), null));
+
+        delayedMessageIndexBucketOpCount.forEach((k, count) -> {
+            String[] labels = splitKey(k);
+            String[] labelsAndValues = new String[] {"state", labels[0], "type", labels[1]};
+            String key = OP_COUNT_NAME + joinKey(labelsAndValues);
+            metrics.put(key, new TopicMetricBean(OP_COUNT_NAME, count.sumThenReset(), labelsAndValues));
+        });
+
+        delayedMessageIndexBucketOpLatencyMs.forEach((typeName, statsBuckets) -> {
+            statsBuckets.refresh();
+            long[] buckets = statsBuckets.getBuckets();
+            for (int i = 0; i < buckets.length; i++) {
+                long count = buckets[i];
+                if (count == 0L) {
+                    continue;
+                }
+                String quantile;
+                if (i == BUCKETS.length) {
+                    quantile = "overflow";
+                } else {
+                    quantile = String.valueOf(BUCKETS[i]);
+                }
+                String[] labelsAndValues = new String[] {"type", typeName, "quantile", quantile};
+                String key = OP_LATENCY_NAME + joinKey(labelsAndValues);
+
+                metrics.put(key, new TopicMetricBean(OP_LATENCY_NAME, count, labelsAndValues));
+            }
+            String[] labelsAndValues = new String[] {"type", typeName};
+            metrics.put(OP_LATENCY_NAME + "_count" + joinKey(labelsAndValues),
+                    new TopicMetricBean(OP_LATENCY_NAME + "_count", statsBuckets.getCount(), labelsAndValues));
+            metrics.put(OP_LATENCY_NAME + "_sum" + joinKey(labelsAndValues),
+                    new TopicMetricBean(OP_LATENCY_NAME + "_sum", statsBuckets.getSum(), labelsAndValues));
+        });
+
+        return metrics;
+    }
+
+    public void recordNumOfBuckets(int numOfBuckets) {
+        delayedMessageIndexBucketTotal.set(numOfBuckets);
+    }
+
+    public void recordDelayedMessageIndexLoaded(long num) {
+        delayedMessageIndexLoaded.set(num);
+    }
+
+    public void recordBucketSnapshotSizeBytes(long sizeBytes) {
+        delayedMessageIndexBucketSnapshotSizeBytes.set(sizeBytes);
+    }
+
+    public void recordTriggerEvent(Type eventType) {
+        delayedMessageIndexBucketOpCount.computeIfAbsent(joinKey(State.all.name(), eventType.name()),
+                k -> new LongAdder()).increment();
+    }
+
+    public void recordSuccessEvent(Type eventType, long cost) {
+        delayedMessageIndexBucketOpCount.computeIfAbsent(joinKey(State.succeed.name(), eventType.name()),
+                k -> new LongAdder()).increment();
+        delayedMessageIndexBucketOpLatencyMs.computeIfAbsent(eventType.name(),
+                k -> new StatsBuckets(BUCKETS)).addValue(cost);
+    }
+
+    public void recordFailEvent(Type eventType) {
+        delayedMessageIndexBucketOpCount.computeIfAbsent(joinKey(State.failed.name(), eventType.name()),
+                k -> new LongAdder()).increment();
+    }
+
+    public static String joinKey(String... values) {
+        return String.join("_", values);
+    }
+
+    public static String[] splitKey(String key) {
+        return key.split("_");
+    }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java
index 969d326e28187..82e98cefa5d98 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java
@@ -131,6 +131,9 @@ private CompletableFuture> asyncLoadNextBucketSnapshotEntry(b
                         List indexList =
                                 snapshotSegment.getIndexesList();
                         this.setCurrentSegmentEntryId(nextSegmentEntryId);
+                        if (isRecover) {
+                            this.asyncUpdateSnapshotLength();
+                        }
                         return indexList;
                     });
         });
@@ -175,7 +178,9 @@ CompletableFuture>
         }, BucketSnapshotPersistenceException.class, MaxRetryTimes);
     }
 
-    CompletableFuture asyncDeleteBucketSnapshot() {
+    CompletableFuture asyncDeleteBucketSnapshot(BucketDelayedMessageIndexStats stats) {
+        long deleteStartTime = System.currentTimeMillis();
+        stats.recordTriggerEvent(BucketDelayedMessageIndexStats.Type.delete);
         String bucketKey = bucketKey();
         long bucketId = getAndUpdateBucketId();
         return removeBucketCursorProperty(bucketKey).thenCompose(__ ->
@@ -184,16 +189,33 @@ CompletableFuture asyncDeleteBucketSnapshot() {
                     if (ex != null) {
                         log.error("[{}] Failed to delete bucket snapshot, bucketId: {}, bucketKey: {}",
                                 dispatcherName, bucketId, bucketKey, ex);
+
+                        stats.recordFailEvent(BucketDelayedMessageIndexStats.Type.delete);
                     } else {
                         log.info("[{}] Delete bucket snapshot finish, bucketId: {}, bucketKey: {}",
                                  dispatcherName, bucketId, bucketKey);
+
+                        stats.recordSuccessEvent(BucketDelayedMessageIndexStats.Type.delete,
+                                System.currentTimeMillis() - deleteStartTime);
                     }
         });
     }
 
-    CompletableFuture clear() {
+    CompletableFuture clear(BucketDelayedMessageIndexStats stats) {
         delayedIndexBitMap.clear();
         return getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE).exceptionally(e -> null)
-                .thenCompose(__ -> asyncDeleteBucketSnapshot());
+                .thenCompose(__ -> asyncDeleteBucketSnapshot(stats));
+    }
+
+    protected CompletableFuture asyncUpdateSnapshotLength() {
+        long bucketId = getAndUpdateBucketId();
+        return bucketSnapshotStorage.getBucketSnapshotLength(bucketId).whenComplete((length, ex) -> {
+            if (ex != null) {
+                log.error("[{}] Failed to get snapshot length, bucketId: {}, bucketKey: {}",
+                        dispatcherName, bucketId, bucketKey(), ex);
+            } else {
+                setSnapshotLength(length);
+            }
+        });
     }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
index f8a4ecc7a4ddb..e49ebe9606e01 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java
@@ -62,8 +62,10 @@ Pair createImmutableBucketAndAsyncPersistent(
             final long timeStepPerBucketSnapshotSegment, final int maxIndexesPerBucketSnapshotSegment,
             TripleLongPriorityQueue sharedQueue, DelayedIndexQueue delayedIndexQueue, final long startLedgerId,
             final long endLedgerId) {
-        log.info("[{}] Creating bucket snapshot, startLedgerId: {}, endLedgerId: {}", dispatcherName,
-                startLedgerId, endLedgerId);
+        if (log.isDebugEnabled()) {
+            log.debug("[{}] Creating bucket snapshot, startLedgerId: {}, endLedgerId: {}", dispatcherName,
+                    startLedgerId, endLedgerId);
+        }
 
         if (delayedIndexQueue.isEmpty()) {
             return null;
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
index e5c9e85bac3f2..7ff6e72d02aed 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
@@ -72,6 +72,7 @@
 import org.apache.pulsar.client.impl.Backoff;
 import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType;
 import org.apache.pulsar.common.api.proto.MessageMetadata;
+import org.apache.pulsar.common.policies.data.stats.TopicMetricBean;
 import org.apache.pulsar.common.protocol.Commands;
 import org.apache.pulsar.common.util.Codec;
 import org.slf4j.Logger;
@@ -332,7 +333,7 @@ public synchronized void readMoreEntries() {
                     Predicate skipCondition = null;
                     final DelayedDeliveryTracker deliveryTracker = delayedDeliveryTracker.get();
                     if (deliveryTracker instanceof BucketDelayedDeliveryTracker) {
-                        skipCondition = position -> deliveryTracker
+                        skipCondition = position -> ((BucketDelayedDeliveryTracker) deliveryTracker)
                                 .containsMessage(position.getLedgerId(), position.getEntryId());
                     }
                     cursor.asyncReadEntriesWithSkipOrWait(messagesToRead, bytesToRead, this, ReadType.Normal,
@@ -1180,6 +1181,18 @@ public long getDelayedTrackerMemoryUsage() {
         return 0;
     }
 
+    public Map getBucketDelayedIndexStats() {
+        if (delayedDeliveryTracker.isEmpty()) {
+            return Collections.emptyMap();
+        }
+
+        if (delayedDeliveryTracker.get() instanceof BucketDelayedDeliveryTracker) {
+            return ((BucketDelayedDeliveryTracker) delayedDeliveryTracker.get()).genTopicMetricMap();
+        }
+
+        return Collections.emptyMap();
+    }
+
     public ManagedCursor getCursor() {
         return cursor;
     }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
index bdb3c9fc391ce..4ed191a9b4f61 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
@@ -1147,6 +1147,9 @@ public SubscriptionStatsImpl getStats(Boolean getPreciseBacklog, boolean subscri
         if (dispatcher instanceof PersistentDispatcherMultipleConsumers) {
             subStats.delayedMessageIndexSizeInBytes =
                     ((PersistentDispatcherMultipleConsumers) dispatcher).getDelayedTrackerMemoryUsage();
+
+            subStats.bucketDelayedIndexStats =
+                    ((PersistentDispatcherMultipleConsumers) dispatcher).getBucketDelayedIndexStats();
         }
 
         if (Subscription.isIndividualAckMode(subType)) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
index 82a4f5312357a..fa08330ff3c35 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
@@ -153,6 +153,7 @@
 import org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl;
 import org.apache.pulsar.common.policies.data.stats.ReplicatorStatsImpl;
 import org.apache.pulsar.common.policies.data.stats.SubscriptionStatsImpl;
+import org.apache.pulsar.common.policies.data.stats.TopicMetricBean;
 import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl;
 import org.apache.pulsar.common.protocol.Commands;
 import org.apache.pulsar.common.protocol.schema.SchemaData;
@@ -2183,6 +2184,14 @@ public CompletableFuture asyncGetStats(boolean getPreciseBacklog
             stats.nonContiguousDeletedMessagesRangesSerializedSize +=
                     subStats.nonContiguousDeletedMessagesRangesSerializedSize;
             stats.delayedMessageIndexSizeInBytes += subStats.delayedMessageIndexSizeInBytes;
+
+            subStats.bucketDelayedIndexStats.forEach((k, v) -> {
+                TopicMetricBean topicMetricBean =
+                        stats.bucketDelayedIndexStats.computeIfAbsent(k, __ -> new TopicMetricBean());
+                topicMetricBean.name = v.name;
+                topicMetricBean.labelsAndValues = v.labelsAndValues;
+                topicMetricBean.value += v.value;
+            });
         });
 
         replicators.forEach((cluster, replicator) -> {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java
index 0a905daa341f3..ea77bd69302a0 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java
@@ -21,6 +21,7 @@
 import java.util.HashMap;
 import java.util.Map;
 import org.apache.bookkeeper.mledger.util.StatsBuckets;
+import org.apache.pulsar.common.policies.data.stats.TopicMetricBean;
 import org.apache.pulsar.compaction.CompactionRecord;
 
 public class AggregatedNamespaceStats {
@@ -65,6 +66,8 @@ public class AggregatedNamespaceStats {
     StatsBuckets compactionLatencyBuckets = new StatsBuckets(CompactionRecord.WRITE_LATENCY_BUCKETS_USEC);
     int delayedMessageIndexSizeInBytes;
 
+    Map bucketDelayedIndexStats = new HashMap<>();
+
     void updateStats(TopicStats stats) {
         topicsCount++;
 
@@ -83,6 +86,14 @@ void updateStats(TopicStats stats) {
         msgOutCounter += stats.msgOutCounter;
         delayedMessageIndexSizeInBytes += stats.delayedMessageIndexSizeInBytes;
 
+        stats.bucketDelayedIndexStats.forEach((k, v) -> {
+            TopicMetricBean topicMetricBean =
+                    bucketDelayedIndexStats.computeIfAbsent(k, __ -> new TopicMetricBean());
+            topicMetricBean.name = v.name;
+            topicMetricBean.labelsAndValues = v.labelsAndValues;
+            topicMetricBean.value += v.value;
+        });
+
         this.ongoingTxnCount += stats.ongoingTxnCount;
         this.abortedTxnCount += stats.abortedTxnCount;
         this.committedTxnCount += stats.committedTxnCount;
@@ -132,6 +143,13 @@ void updateStats(TopicStats stats) {
             subsStats.filterRejectedMsgCount += as.filterRejectedMsgCount;
             subsStats.filterRescheduledMsgCount += as.filterRescheduledMsgCount;
             subsStats.delayedMessageIndexSizeInBytes += as.delayedMessageIndexSizeInBytes;
+            as.bucketDelayedIndexStats.forEach((k, v) -> {
+                TopicMetricBean topicMetricBean =
+                        subsStats.bucketDelayedIndexStats.computeIfAbsent(k, __ -> new TopicMetricBean());
+                topicMetricBean.name = v.name;
+                topicMetricBean.labelsAndValues = v.labelsAndValues;
+                topicMetricBean.value += v.value;
+            });
             as.consumerStat.forEach((c, v) -> {
                 AggregatedConsumerStats consumerStats =
                         subsStats.consumerStat.computeIfAbsent(c, k -> new AggregatedConsumerStats());
@@ -172,5 +190,6 @@ public void reset() {
         replicationStats.clear();
         subscriptionStats.clear();
         delayedMessageIndexSizeInBytes = 0;
+        bucketDelayedIndexStats.clear();
     }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedSubscriptionStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedSubscriptionStats.java
index 383c671754dc1..da0324c55655c 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedSubscriptionStats.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedSubscriptionStats.java
@@ -21,6 +21,7 @@
 import java.util.HashMap;
 import java.util.Map;
 import org.apache.pulsar.broker.service.Consumer;
+import org.apache.pulsar.common.policies.data.stats.TopicMetricBean;
 
 public class AggregatedSubscriptionStats {
 
@@ -75,4 +76,6 @@ public class AggregatedSubscriptionStats {
     public Map consumerStat = new HashMap<>();
 
     long delayedMessageIndexSizeInBytes;
+
+    public Map bucketDelayedIndexStats = new HashMap<>();
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java
index 918aef539cff4..32fb06ea3ce8c 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java
@@ -28,6 +28,7 @@
 import org.apache.bookkeeper.client.LedgerHandle;
 import org.apache.bookkeeper.mledger.ManagedLedger;
 import org.apache.bookkeeper.mledger.impl.ManagedLedgerMBeanImpl;
+import org.apache.commons.lang3.ArrayUtils;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.service.Topic;
 import org.apache.pulsar.broker.service.persistent.PersistentTopic;
@@ -155,6 +156,7 @@ private static void aggregateTopicStats(TopicStats stats, SubscriptionStatsImpl
         subsStats.filterRejectedMsgCount = subscriptionStats.filterRejectedMsgCount;
         subsStats.filterRescheduledMsgCount = subscriptionStats.filterRescheduledMsgCount;
         subsStats.delayedMessageIndexSizeInBytes = subscriptionStats.delayedMessageIndexSizeInBytes;
+        subsStats.bucketDelayedIndexStats = subscriptionStats.bucketDelayedIndexStats;
     }
 
     private static void getTopicStats(Topic topic, TopicStats stats, boolean includeConsumerMetrics,
@@ -197,6 +199,7 @@ private static void getTopicStats(Topic topic, TopicStats stats, boolean include
         stats.averageMsgSize = tStatus.averageMsgSize;
         stats.publishRateLimitedTimes = tStatus.publishRateLimitedTimes;
         stats.delayedMessageIndexSizeInBytes = tStatus.delayedMessageIndexSizeInBytes;
+        stats.bucketDelayedIndexStats = tStatus.bucketDelayedIndexStats;
         stats.abortedTxnCount = tStatus.abortedTxnCount;
         stats.ongoingTxnCount = tStatus.ongoingTxnCount;
         stats.committedTxnCount = tStatus.committedTxnCount;
@@ -379,6 +382,10 @@ private static void printNamespaceStats(PrometheusMetricStreams stream, Aggregat
         writeMetric(stream, "pulsar_delayed_message_index_size_bytes", stats.delayedMessageIndexSizeInBytes, cluster,
                 namespace);
 
+        stats.bucketDelayedIndexStats.forEach((k, metric) -> {
+            writeMetric(stream, metric.name, metric.value, cluster, namespace, metric.labelsAndValues);
+        });
+
         writePulsarMsgBacklog(stream, stats.msgBacklog, cluster, namespace);
 
         stats.managedLedgerStats.storageWriteLatencyBuckets.refresh();
@@ -472,8 +479,10 @@ private static void writeMetric(PrometheusMetricStreams stream, String metricNam
     }
 
     private static void writeMetric(PrometheusMetricStreams stream, String metricName, Number value, String cluster,
-                                    String namespace) {
-        stream.writeSample(metricName, value, "cluster", cluster, "namespace", namespace);
+                                    String namespace, String... extraLabelsAndValues) {
+        String[] labelsAndValues = new String[]{"cluster", cluster, "namespace", namespace};
+        String[] labels = ArrayUtils.addAll(labelsAndValues, extraLabelsAndValues);
+        stream.writeSample(metricName, value, labels);
     }
 
     private static void writeReplicationStat(PrometheusMetricStreams stream, String metricName,
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java
index abc6979484e58..3a2563a87587b 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java
@@ -25,6 +25,7 @@
 import org.apache.bookkeeper.mledger.util.StatsBuckets;
 import org.apache.commons.lang3.ArrayUtils;
 import org.apache.pulsar.broker.service.Consumer;
+import org.apache.pulsar.common.policies.data.stats.TopicMetricBean;
 import org.apache.pulsar.compaction.CompactionRecord;
 import org.apache.pulsar.compaction.CompactorMXBean;
 
@@ -70,6 +71,8 @@ class TopicStats {
     StatsBuckets compactionLatencyBuckets = new StatsBuckets(CompactionRecord.WRITE_LATENCY_BUCKETS_USEC);
     public long delayedMessageIndexSizeInBytes;
 
+    Map bucketDelayedIndexStats = new HashMap<>();
+
     public void reset() {
         subscriptionsCount = 0;
         producersCount = 0;
@@ -107,6 +110,7 @@ public void reset() {
         compactionCompactedEntriesSize = 0;
         compactionLatencyBuckets.reset();
         delayedMessageIndexSizeInBytes = 0;
+        bucketDelayedIndexStats.clear();
     }
 
     public static void printTopicStats(PrometheusMetricStreams stream, TopicStats stats,
@@ -162,6 +166,11 @@ public static void printTopicStats(PrometheusMetricStreams stream, TopicStats st
         writeMetric(stream, "pulsar_delayed_message_index_size_bytes", stats.delayedMessageIndexSizeInBytes,
                 cluster, namespace, topic, splitTopicAndPartitionIndexLabel);
 
+        for (TopicMetricBean topicMetricBean : stats.bucketDelayedIndexStats.values()) {
+            writeTopicMetric(stream, topicMetricBean.name, topicMetricBean.value, cluster, namespace,
+                    topic, splitTopicAndPartitionIndexLabel, topicMetricBean.labelsAndValues);
+        }
+
         long[] latencyBuckets = stats.managedLedgerStats.storageWriteLatencyBuckets.getBuckets();
         writeMetric(stream, "pulsar_storage_write_latency_le_0_5",
                 latencyBuckets[0], cluster, namespace, topic, splitTopicAndPartitionIndexLabel);
@@ -310,6 +319,13 @@ public static void printTopicStats(PrometheusMetricStreams stream, TopicStats st
                     subsStats.delayedMessageIndexSizeInBytes, cluster, namespace, topic, sub,
                     splitTopicAndPartitionIndexLabel);
 
+            final String[] subscriptionLabel = {"subscription", sub};
+            for (TopicMetricBean topicMetricBean : subsStats.bucketDelayedIndexStats.values()) {
+                String[] labelsAndValues = ArrayUtils.addAll(subscriptionLabel, topicMetricBean.labelsAndValues);
+                writeTopicMetric(stream, topicMetricBean.name, topicMetricBean.value, cluster, namespace,
+                        topic, splitTopicAndPartitionIndexLabel, labelsAndValues);
+            }
+
             subsStats.consumerStat.forEach((c, consumerStats) -> {
                 writeConsumerMetric(stream, "pulsar_consumer_msg_rate_redeliver", consumerStats.msgRateRedeliver,
                         cluster, namespace, topic, sub, c, splitTopicAndPartitionIndexLabel);
@@ -409,6 +425,12 @@ public static void printTopicStats(PrometheusMetricStreams stream, TopicStats st
             writeMetric(stream, "pulsar_compaction_latency_count",
                     stats.compactionLatencyBuckets.getCount(), cluster, namespace, topic,
                     splitTopicAndPartitionIndexLabel);
+
+            for (TopicMetricBean topicMetricBean : stats.bucketDelayedIndexStats.values()) {
+                String[] labelsAndValues = topicMetricBean.labelsAndValues;
+                writeTopicMetric(stream, topicMetricBean.name, topicMetricBean.value, cluster, namespace,
+                        topic, splitTopicAndPartitionIndexLabel, labelsAndValues);
+            }
         }
     }
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java
index cf7310c7067c3..9e924bdeda341 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java
@@ -174,7 +174,7 @@ public CompletableFuture getBucketSnapshotLength(long bucketId) {
             long length = 0;
             List bufList = this.bucketSnapshots.get(bucketId);
             for (ByteBuf byteBuf : bufList) {
-                length += byteBuf.array().length;
+                length += byteBuf.readableBytes();
             }
             return length;
         }, executorService);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
index 717bada7705dc..95234d688f6a0 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java
@@ -46,7 +46,6 @@
 import org.apache.bookkeeper.mledger.ManagedCursor;
 import org.apache.bookkeeper.mledger.impl.PositionImpl;
 import org.apache.pulsar.broker.delayed.AbstractDeliveryTrackerTest;
-import org.apache.pulsar.broker.delayed.DelayedDeliveryTracker;
 import org.apache.pulsar.broker.delayed.MockBucketSnapshotStorage;
 import org.apache.pulsar.broker.delayed.MockManagedCursor;
 import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers;
@@ -158,7 +157,7 @@ public Object[][] provider(Method method) throws Exception {
     }
 
     @Test(dataProvider = "delayedTracker")
-    public void testContainsMessage(DelayedDeliveryTracker tracker) {
+    public void testContainsMessage(BucketDelayedDeliveryTracker tracker) {
         tracker.addMessage(1, 1, 10);
         tracker.addMessage(2, 2, 20);
 
@@ -191,6 +190,12 @@ public void testRecoverSnapshot(BucketDelayedDeliveryTracker tracker) {
 
         clockTime.set(1 * 10);
 
+        Awaitility.await().untilAsserted(() -> {
+            Assert.assertTrue(
+                    tracker.getImmutableBuckets().asMapOfRanges().values().stream().noneMatch(x -> x.merging ||
+                            !x.getSnapshotCreateFuture().get().isDone()));
+        });
+
         assertTrue(tracker.hasMessageAvailable());
         Set scheduledMessages = tracker.getScheduledMessages(100);
 
@@ -202,16 +207,16 @@ public void testRecoverSnapshot(BucketDelayedDeliveryTracker tracker) {
 
         clockTime.set(30 * 10);
 
-        tracker = new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock,
-                true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1,50);
+        BucketDelayedDeliveryTracker tracker2 = new BucketDelayedDeliveryTracker(dispatcher, timer, 1000, clock,
+                true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1, 50);
 
-        assertFalse(tracker.containsMessage(101, 101));
-        assertEquals(tracker.getNumberOfDelayedMessages(), 70);
+        assertFalse(tracker2.containsMessage(101, 101));
+        assertEquals(tracker2.getNumberOfDelayedMessages(), 70);
 
         clockTime.set(100 * 10);
 
-        assertTrue(tracker.hasMessageAvailable());
-        scheduledMessages = tracker.getScheduledMessages(70);
+        assertTrue(tracker2.hasMessageAvailable());
+        scheduledMessages = tracker2.getScheduledMessages(70);
 
         assertEquals(scheduledMessages.size(), 70);
 
@@ -221,7 +226,7 @@ public void testRecoverSnapshot(BucketDelayedDeliveryTracker tracker) {
             i++;
         }
 
-        tracker.close();
+        tracker2.close();
     }
 
     @Test
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java
index 09b7cbbf1b99d..5480a2e7a704a 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java
@@ -19,18 +19,26 @@
 package org.apache.pulsar.broker.service.persistent;
 
 import static org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.CURSOR_INTERNAL_PROPERTY_PREFIX;
+import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertTrue;
+import com.google.common.collect.Multimap;
+import java.io.ByteArrayOutputStream;
+import java.nio.charset.StandardCharsets;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 import lombok.Cleanup;
 import org.apache.bookkeeper.client.BKException;
 import org.apache.bookkeeper.client.BookKeeper;
 import org.apache.bookkeeper.client.LedgerHandle;
 import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl;
+import org.apache.commons.lang3.mutable.MutableInt;
 import org.apache.pulsar.broker.BrokerTestUtil;
 import org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTrackerFactory;
 import org.apache.pulsar.broker.service.Dispatcher;
+import org.apache.pulsar.broker.stats.PrometheusMetricsTest;
+import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsGenerator;
 import org.apache.pulsar.client.api.Consumer;
 import org.apache.pulsar.client.api.Producer;
 import org.apache.pulsar.client.api.Schema;
@@ -162,4 +170,139 @@ public void testUnsubscribe() throws Exception {
             }
         }
     }
+
+
+    @Test
+    public void testBucketDelayedIndexMetrics() throws Exception {
+        cleanup();
+        setup();
+
+        String topic = BrokerTestUtil.newUniqueName("persistent://public/default/testBucketDelayedIndexMetrics");
+
+        @Cleanup
+        Consumer consumer = pulsarClient.newConsumer(Schema.STRING)
+                .topic(topic)
+                .subscriptionName("test_sub")
+                .subscriptionType(SubscriptionType.Shared)
+                .subscribe();
+
+        @Cleanup
+        Consumer consumer2 = pulsarClient.newConsumer(Schema.STRING)
+                .topic(topic)
+                .subscriptionName("test_sub2")
+                .subscriptionType(SubscriptionType.Shared)
+                .subscribe();
+
+        @Cleanup
+        Producer producer = pulsarClient.newProducer(Schema.STRING)
+                .topic(topic)
+                .create();
+
+        final int N = 101;
+
+        for (int i = 0; i < N; i++) {
+            producer.newMessage()
+                    .value("msg-" + i)
+                    .deliverAfter(3600 + i, TimeUnit.SECONDS)
+                    .sendAsync();
+        }
+        producer.flush();
+
+        Thread.sleep(2000);
+
+        ByteArrayOutputStream output = new ByteArrayOutputStream();
+        PrometheusMetricsGenerator.generate(pulsar, true, true, true, output);
+        String metricsStr = output.toString(StandardCharsets.UTF_8);
+        Multimap metricsMap = PrometheusMetricsTest.parseMetrics(metricsStr);
+
+        List bucketsMetrics =
+                metricsMap.get("pulsar_delayed_message_index_bucket_total").stream()
+                        .filter(metric -> metric.tags.get("topic").equals(topic)).toList();
+        MutableInt bucketsSum = new MutableInt();
+        bucketsMetrics.stream().filter(metric -> metric.tags.containsKey("subscription")).forEach(metric -> {
+            assertEquals(3, metric.value);
+            bucketsSum.add(metric.value);
+        });
+        assertEquals(6, bucketsSum.intValue());
+        Optional bucketsTopicMetric =
+                bucketsMetrics.stream().filter(metric -> !metric.tags.containsKey("subscription")).findFirst();
+        assertTrue(bucketsTopicMetric.isPresent());
+        assertEquals(bucketsSum.intValue(), bucketsTopicMetric.get().value);
+
+        List loadedIndexMetrics =
+                metricsMap.get("pulsar_delayed_message_index_loaded").stream()
+                        .filter(metric -> metric.tags.get("topic").equals(topic)).toList();
+        MutableInt loadedIndexSum = new MutableInt();
+        long count = loadedIndexMetrics.stream().filter(metric -> metric.tags.containsKey("subscription")).peek(metric -> {
+            assertTrue(metric.value > 0 && metric.value <= N);
+            loadedIndexSum.add(metric.value);
+        }).count();
+        assertEquals(2, count);
+        Optional loadedIndexTopicMetrics =
+                bucketsMetrics.stream().filter(metric -> !metric.tags.containsKey("subscription")).findFirst();
+        assertTrue(loadedIndexTopicMetrics.isPresent());
+        assertEquals(loadedIndexSum.intValue(), loadedIndexTopicMetrics.get().value);
+
+        List snapshotSizeBytesMetrics =
+                metricsMap.get("pulsar_delayed_message_index_bucket_snapshot_size_bytes").stream()
+                        .filter(metric -> metric.tags.get("topic").equals(topic)).toList();
+        MutableInt snapshotSizeBytesSum = new MutableInt();
+        count = snapshotSizeBytesMetrics.stream().filter(metric -> metric.tags.containsKey("subscription"))
+                .peek(metric -> {
+                    assertTrue(metric.value > 0);
+                    snapshotSizeBytesSum.add(metric.value);
+                }).count();
+        assertEquals(2, count);
+        Optional snapshotSizeBytesTopicMetrics =
+                snapshotSizeBytesMetrics.stream().filter(metric -> !metric.tags.containsKey("subscription")).findFirst();
+        assertTrue(snapshotSizeBytesTopicMetrics.isPresent());
+        assertEquals(snapshotSizeBytesSum.intValue(), snapshotSizeBytesTopicMetrics.get().value);
+
+        List opCountMetrics =
+                metricsMap.get("pulsar_delayed_message_index_bucket_op_count").stream()
+                        .filter(metric -> metric.tags.get("topic").equals(topic)).toList();
+        MutableInt opCountMetricsSum = new MutableInt();
+        count = opCountMetrics.stream()
+                .filter(metric -> metric.tags.get("state").equals("succeed") && metric.tags.get("type").equals("create")
+                        && metric.tags.containsKey("subscription"))
+                .peek(metric -> {
+                    assertTrue(metric.value >= 2);
+                    opCountMetricsSum.add(metric.value);
+                }).count();
+        assertEquals(2, count);
+        Optional opCountTopicMetrics =
+                opCountMetrics.stream()
+                        .filter(metric -> metric.tags.get("state").equals("succeed") && metric.tags.get("type")
+                                .equals("create") && !metric.tags.containsKey("subscription")).findFirst();
+        assertTrue(opCountTopicMetrics.isPresent());
+        assertEquals(opCountMetricsSum.intValue(), opCountTopicMetrics.get().value);
+
+        List opLatencyMetrics =
+                metricsMap.get("pulsar_delayed_message_index_bucket_op_latency_ms").stream()
+                        .filter(metric -> metric.tags.get("topic").equals(topic)).toList();
+        MutableInt opLatencyMetricsSum = new MutableInt();
+        count = opLatencyMetrics.stream()
+                .filter(metric -> metric.tags.get("type").equals("create")
+                        && metric.tags.containsKey("subscription"))
+                .peek(metric -> {
+                    assertTrue(metric.tags.containsKey("quantile"));
+                    opLatencyMetricsSum.add(metric.value);
+                }).count();
+        assertTrue(count >= 2);
+        Optional opLatencyTopicMetrics =
+                opCountMetrics.stream()
+                        .filter(metric -> metric.tags.get("type").equals("create")
+                                && !metric.tags.containsKey("subscription")).findFirst();
+        assertTrue(opLatencyTopicMetrics.isPresent());
+        assertEquals(opLatencyMetricsSum.intValue(), opLatencyTopicMetrics.get().value);
+
+        ByteArrayOutputStream namespaceOutput = new ByteArrayOutputStream();
+        PrometheusMetricsGenerator.generate(pulsar, false, true, true, namespaceOutput);
+        Multimap namespaceMetricsMap = PrometheusMetricsTest.parseMetrics(namespaceOutput.toString(StandardCharsets.UTF_8));
+
+        Optional namespaceMetric =
+                namespaceMetricsMap.get("pulsar_delayed_message_index_bucket_total").stream().findFirst();
+        assertTrue(namespaceMetric.isPresent());
+        assertEquals(6, namespaceMetric.get().value);
+    }
 }
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java
index 9c7e24ba02118..25fa666523f3c 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java
@@ -134,6 +134,8 @@ public class SubscriptionStatsImpl implements SubscriptionStats {
     /** The size of InMemoryDelayedDeliveryTracer memory usage. */
     public long delayedMessageIndexSizeInBytes;
 
+    public Map bucketDelayedIndexStats;
+
     /** SubscriptionProperties (key/value strings) associated with this subscribe. */
     public Map subscriptionProperties;
 
@@ -149,6 +151,7 @@ public SubscriptionStatsImpl() {
         this.consumers = new ArrayList<>();
         this.consumersAfterMarkDeletePosition = new LinkedHashMap<>();
         this.subscriptionProperties = new HashMap<>();
+        this.bucketDelayedIndexStats = new HashMap<>();
     }
 
     public void reset() {
@@ -175,6 +178,7 @@ public void reset() {
         filterAcceptedMsgCount = 0;
         filterRejectedMsgCount = 0;
         filterRescheduledMsgCount = 0;
+        bucketDelayedIndexStats.clear();
     }
 
     // if the stats are added for the 1st time, we will need to make a copy of these stats and add it to the current
@@ -215,6 +219,14 @@ public SubscriptionStatsImpl add(SubscriptionStatsImpl stats) {
         this.filterAcceptedMsgCount += stats.filterAcceptedMsgCount;
         this.filterRejectedMsgCount += stats.filterRejectedMsgCount;
         this.filterRescheduledMsgCount += stats.filterRescheduledMsgCount;
+        stats.bucketDelayedIndexStats.forEach((k, v) -> {
+            TopicMetricBean topicMetricBean =
+                    this.bucketDelayedIndexStats.computeIfAbsent(k, __ -> new TopicMetricBean());
+            topicMetricBean.name = v.name;
+            topicMetricBean.labelsAndValues = v.labelsAndValues;
+            topicMetricBean.value += v.value;
+        });
+
         return this;
     }
 }
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicMetricBean.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicMetricBean.java
new file mode 100644
index 0000000000000..e01a9d7aa71f3
--- /dev/null
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicMetricBean.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.common.policies.data.stats;
+
+import lombok.AllArgsConstructor;
+import lombok.NoArgsConstructor;
+
+@NoArgsConstructor
+@AllArgsConstructor
+public class TopicMetricBean {
+    public String name;
+    public double value;
+    public String[] labelsAndValues;
+}
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java
index 12d30124f7dcd..c9c4739b904f6 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java
@@ -139,6 +139,9 @@ public class TopicStatsImpl implements TopicStats {
     /** The size of InMemoryDelayedDeliveryTracer memory usage. */
     public long delayedMessageIndexSizeInBytes;
 
+    /** Map of bucket delayed index statistics. */
+    public Map bucketDelayedIndexStats;
+
     /** The compaction stats. */
     public CompactionStatsImpl compaction;
 
@@ -182,6 +185,7 @@ public TopicStatsImpl() {
         this.subscriptions = new HashMap<>();
         this.replication = new TreeMap<>();
         this.compaction = new CompactionStatsImpl();
+        this.bucketDelayedIndexStats = new HashMap<>();
     }
 
     public void reset() {
@@ -214,6 +218,7 @@ public void reset() {
         this.delayedMessageIndexSizeInBytes = 0;
         this.compaction.reset();
         this.ownerBroker = null;
+        this.bucketDelayedIndexStats.clear();
     }
 
     // if the stats are added for the 1st time, we will need to make a copy of these stats and add it to the current
@@ -244,6 +249,14 @@ public TopicStatsImpl add(TopicStats ts) {
         this.abortedTxnCount = stats.abortedTxnCount;
         this.committedTxnCount = stats.committedTxnCount;
 
+        stats.bucketDelayedIndexStats.forEach((k, v) -> {
+            TopicMetricBean topicMetricBean =
+                    this.bucketDelayedIndexStats.computeIfAbsent(k, __ -> new TopicMetricBean());
+            topicMetricBean.name = v.name;
+            topicMetricBean.labelsAndValues = v.labelsAndValues;
+            topicMetricBean.value += v.value;
+        });
+
         for (int index = 0; index < stats.getPublishers().size(); index++) {
            PublisherStats s = stats.getPublishers().get(index);
            if (s.isSupportsPartialProducer() && s.getProducerName() != null) {

From 02147454c425b92f0cd1caefa73b9339db6a0269 Mon Sep 17 00:00:00 2001
From: tison 
Date: Sat, 1 Apr 2023 14:18:02 +0800
Subject: [PATCH 167/174] [feat][client] Support configure MessageCrypto in
 ProducerBuilder (#19939)

Signed-off-by: tison 
---
 .../pulsar/client/api/ProducerBuilder.java    | 11 ++++++
 .../client/impl/ProducerBuilderImpl.java      |  7 ++++
 .../client/impl/ProducerBuilderImplTest.java  | 39 +++++++++++--------
 3 files changed, 41 insertions(+), 16 deletions(-)

diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerBuilder.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerBuilder.java
index d2231e71c237a..896c22313247a 100644
--- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerBuilder.java
+++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerBuilder.java
@@ -389,6 +389,17 @@ public interface ProducerBuilder extends Cloneable {
      */
     ProducerBuilder defaultCryptoKeyReader(Map publicKeys);
 
+    /**
+     * Sets a {@link MessageCrypto}.
+     *
+     * 
Contains methods to encrypt/decrypt messages for end-to-end encryption.
+     *
+     * @param messageCrypto
+     *            MessageCrypto object
+     * @return the producer builder instance
+     */
+    ProducerBuilder messageCrypto(MessageCrypto messageCrypto);
+
     /**
      * Add public encryption key, used by producer to encrypt the data key.
      *
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBuilderImpl.java
index bfdbb63de1855..ecbdfa76b64ae 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBuilderImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBuilderImpl.java
@@ -34,6 +34,7 @@
 import org.apache.pulsar.client.api.CompressionType;
 import org.apache.pulsar.client.api.CryptoKeyReader;
 import org.apache.pulsar.client.api.HashingScheme;
+import org.apache.pulsar.client.api.MessageCrypto;
 import org.apache.pulsar.client.api.MessageRouter;
 import org.apache.pulsar.client.api.MessageRoutingMode;
 import org.apache.pulsar.client.api.Producer;
@@ -221,6 +222,12 @@ public ProducerBuilder defaultCryptoKeyReader(@NonNull Map pu
         return cryptoKeyReader(DefaultCryptoKeyReader.builder().publicKeys(publicKeys).build());
     }
 
+    @Override
+    public ProducerBuilder messageCrypto(MessageCrypto messageCrypto) {
+        conf.setMessageCrypto(messageCrypto);
+        return this;
+    }
+
     @Override
     public ProducerBuilder addEncryptionKey(String key) {
         checkArgument(StringUtils.isNotBlank(key), "Encryption key cannot be blank");
diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerBuilderImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerBuilderImplTest.java
index 66c552ef7934a..bb3e3fc3accf6 100644
--- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerBuilderImplTest.java
+++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerBuilderImplTest.java
@@ -18,6 +18,15 @@
  */
 package org.apache.pulsar.client.impl;
 
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertNotNull;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
 import org.apache.pulsar.client.api.Message;
 import org.apache.pulsar.client.api.MessageRouter;
 import org.apache.pulsar.client.api.MessageRoutingMode;
@@ -26,20 +35,10 @@
 import org.apache.pulsar.client.api.Schema;
 import org.apache.pulsar.client.api.TopicMetadata;
 import org.apache.pulsar.client.impl.conf.ProducerConfigurationData;
+import org.apache.pulsar.client.impl.crypto.MessageCryptoBc;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.TimeUnit;
-
-import static org.mockito.Mockito.any;
-import static org.mockito.Mockito.eq;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-import static org.testng.Assert.assertNotNull;
-
 /**
  * Unit tests of {@link ProducerBuilderImpl}.
  */
@@ -47,13 +46,13 @@ public class ProducerBuilderImplTest {
 
     private static final String TOPIC_NAME = "testTopicName";
     private PulsarClientImpl client;
-    private ProducerBuilderImpl producerBuilderImpl;
+    private ProducerBuilderImpl producerBuilderImpl;
 
     @BeforeClass(alwaysRun = true)
     public void setup() {
-        Producer producer = mock(Producer.class);
+        Producer producer = mock(Producer.class);
         client = mock(PulsarClientImpl.class);
-        producerBuilderImpl = new ProducerBuilderImpl(client, Schema.BYTES);
+        producerBuilderImpl = new ProducerBuilderImpl<>(client, Schema.BYTES);
         when(client.newProducer()).thenReturn(producerBuilderImpl);
 
         when(client.createProducerAsync(
@@ -66,8 +65,8 @@ public void testProducerBuilderImpl() throws PulsarClientException {
         Map properties = new HashMap<>();
         properties.put("Test-Key2", "Test-Value2");
 
-        producerBuilderImpl = new ProducerBuilderImpl(client, Schema.BYTES);
-        Producer producer = producerBuilderImpl.topic(TOPIC_NAME)
+        producerBuilderImpl = new ProducerBuilderImpl<>(client, Schema.BYTES);
+        Producer producer = producerBuilderImpl.topic(TOPIC_NAME)
                 .producerName("Test-Producer")
                 .maxPendingMessages(2)
                 .addEncryptionKey("Test-EncryptionKey")
@@ -78,6 +77,14 @@ public void testProducerBuilderImpl() throws PulsarClientException {
         assertNotNull(producer);
     }
 
+    @Test
+    public void testProducerBuilderImplWhenMessageCryptoSet() throws PulsarClientException {
+        producerBuilderImpl = new ProducerBuilderImpl<>(client, Schema.BYTES);
+        producerBuilderImpl.topic(TOPIC_NAME).messageCrypto(new MessageCryptoBc("ctx1", true));
+        assertNotNull(producerBuilderImpl.create());
+        assertNotNull(producerBuilderImpl.getConf().getMessageCrypto());
+    }
+
     @Test
     public void testProducerBuilderImplWhenMessageRoutingModeAndMessageRouterAreNotSet() throws PulsarClientException {
         producerBuilderImpl = new ProducerBuilderImpl(client, Schema.BYTES);

From be57e9a79b0b6fd40ef02414df7238050b7a885d Mon Sep 17 00:00:00 2001
From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com>
Date: Sat, 1 Apr 2023 19:21:58 +0800
Subject: [PATCH 168/174] [fix][schema]Fix AutoProduceBytes producer can not be
 created to a topic with ProtoBuf schema (#19767)

### Motivation
1. There is a topic1 with a protobuf schema.
2. Create a producer1 with AutoProduceBytes schema.
3. The producer1 will be created failed because the way to get the schema of protobuf schema is not supported. ### ###
### Modification
Because the Protobuf schema is implemented from the AvroBaseStructSchema. So we add a way to get Protobuf schema just like the AvroSchema.
---
 .../org/apache/pulsar/schema/SchemaTest.java    | 17 +++++++++++++++++
 .../pulsar/client/impl/PulsarClientImpl.java    |  9 ++++++++-
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java
index d99496f4a967b..0ec72b2ef470a 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java
@@ -70,6 +70,7 @@
 import org.apache.pulsar.client.api.schema.GenericRecord;
 import org.apache.pulsar.client.api.schema.SchemaDefinition;
 import org.apache.pulsar.client.impl.schema.KeyValueSchemaImpl;
+import org.apache.pulsar.client.impl.schema.ProtobufSchema;
 import org.apache.pulsar.client.impl.schema.SchemaInfoImpl;
 import org.apache.pulsar.client.impl.schema.generic.GenericJsonRecord;
 import org.apache.pulsar.client.impl.schema.writer.AvroWriter;
@@ -113,6 +114,22 @@ public void cleanup() throws Exception {
         super.internalCleanup();
     }
 
+    @Test
+    public void testGetSchemaWhenCreateAutoProduceBytesProducer() throws Exception{
+        final String tenant = PUBLIC_TENANT;
+        final String namespace = "test-namespace-" + randomName(16);
+        final String topic = tenant + "/" + namespace + "/test-getSchema";
+        admin.namespaces().createNamespace(
+                tenant + "/" + namespace,
+                Sets.newHashSet(CLUSTER_NAME)
+        );
+
+        ProtobufSchema protobufSchema =
+                ProtobufSchema.of(org.apache.pulsar.client.api.schema.proto.Test.TestMessage.class);
+        pulsarClient.newProducer(protobufSchema).topic(topic).create();
+        pulsarClient.newProducer(org.apache.pulsar.client.api.Schema.AUTO_PRODUCE_BYTES()).topic(topic).create();
+    }
+
     @Test
     public void testMultiTopicSetSchemaProvider() throws Exception {
         final String tenant = PUBLIC_TENANT;
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java
index ebc11f44ce749..f37709f3d84f6 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java
@@ -70,6 +70,7 @@
 import org.apache.pulsar.client.impl.conf.ReaderConfigurationData;
 import org.apache.pulsar.client.impl.schema.AutoConsumeSchema;
 import org.apache.pulsar.client.impl.schema.AutoProduceBytesSchema;
+import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema;
 import org.apache.pulsar.client.impl.schema.generic.MultiVersionSchemaInfoProvider;
 import org.apache.pulsar.client.impl.transaction.TransactionBuilderImpl;
 import org.apache.pulsar.client.impl.transaction.TransactionCoordinatorClientImpl;
@@ -81,6 +82,7 @@
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.partition.PartitionedTopicMetadata;
 import org.apache.pulsar.common.schema.SchemaInfo;
+import org.apache.pulsar.common.schema.SchemaType;
 import org.apache.pulsar.common.topics.TopicList;
 import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.common.util.netty.EventLoopUtil;
@@ -350,7 +352,12 @@ public  CompletableFuture> createProducerAsync(ProducerConfigurat
             return lookup.getSchema(TopicName.get(conf.getTopicName()))
                     .thenCompose(schemaInfoOptional -> {
                         if (schemaInfoOptional.isPresent()) {
-                            autoProduceBytesSchema.setSchema(Schema.getSchema(schemaInfoOptional.get()));
+                            SchemaInfo schemaInfo = schemaInfoOptional.get();
+                            if (schemaInfo.getType() == SchemaType.PROTOBUF) {
+                                autoProduceBytesSchema.setSchema(new GenericAvroSchema(schemaInfo));
+                            } else {
+                                autoProduceBytesSchema.setSchema(Schema.getSchema(schemaInfo));
+                            }
                         } else {
                             autoProduceBytesSchema.setSchema(Schema.BYTES);
                         }

From 2dcaf0ef692c929ebf6c510d328e0f2775f07f3e Mon Sep 17 00:00:00 2001
From: LinChen 
Date: Mon, 3 Apr 2023 12:55:16 +0800
Subject: [PATCH 169/174] [fix][broker] Fix the loss of bundle stats data
 reported to zookeeper, when the updateStats method is executed (#19887)

Co-authored-by: lordcheng10 
---
 .../java/org/apache/pulsar/broker/service/PulsarStats.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java
index 9cdf9d1dfc68d..2059aa04350d4 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java
@@ -233,7 +233,7 @@ public synchronized void updateStats(
         updatedAt = System.currentTimeMillis();
     }
 
-    public NamespaceBundleStats invalidBundleStats(String bundleName) {
+    public synchronized NamespaceBundleStats invalidBundleStats(String bundleName) {
         return bundleStats.remove(bundleName);
     }
 
@@ -254,7 +254,7 @@ public BrokerOperabilityMetrics getBrokerOperabilityMetrics() {
         return brokerOperabilityMetrics;
     }
 
-    public Map getBundleStats() {
+    public synchronized Map getBundleStats() {
         return bundleStats;
     }
 

From 67eb0fb4e830f2fb7affc9b3ab77b5423b4ec4ce Mon Sep 17 00:00:00 2001
From: Michael Marshall 
Date: Mon, 3 Apr 2023 01:31:35 -0500
Subject: [PATCH 170/174] [cleanup][proxy] ProxyConnection should not call
 super.exceptionCaught (#19990)

### Motivation

While debugging an issue, I noticed that we call `super.exceptionCaught(ctx, cause);` in the `ProxyConnection` class. This leads to the following log line:

> An exceptionCaught() event was fired, and it reached at the tail of the pipeline. It usually means the last handler in the pipeline did not handle the exception. io.netty.channel.unix.Errors$NativeIoException: recvAddress(..) failed: Connection reset by peer

Because we always handle exceptions, there is no need to forward them to the next handler.

### Modifications

* Remove a single method call

### Verifying this change

This is a trivial change. Note that we do not call the super method in any other handler implementations in the project.

### Documentation

- [x] `doc-not-needed` 

### Matching PR in forked repository

PR in forked repository: skipping PR for this trivial change
---
 .../java/org/apache/pulsar/proxy/server/ProxyConnection.java     | 1 -
 1 file changed, 1 deletion(-)

diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java
index 1e19d760c6d7b..b7f5534ee81fb 100644
--- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java
+++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java
@@ -208,7 +208,6 @@ public synchronized void channelInactive(ChannelHandlerContext ctx) throws Excep
 
     @Override
     public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
-        super.exceptionCaught(ctx, cause);
         LOG.warn("[{}] Got exception {} : Message: {} State: {}", remoteAddress, cause.getClass().getSimpleName(),
                 cause.getMessage(), state,
                 ClientCnx.isKnownException(cause) ? null : cause);

From 5ef3a21a068b837a56a432361aa4b33732a13cfb Mon Sep 17 00:00:00 2001
From: tison 
Date: Mon, 3 Apr 2023 15:59:17 +0800
Subject: [PATCH 171/174] [fix][build] Client modules should be built with Java
 8 (#19991)

---
 pom.xml                           |  4 +--
 pulsar-client-auth-athenz/pom.xml | 41 ++++++++++++++++++++++++++++++-
 pulsar-client-auth-sasl/pom.xml   | 39 +++++++++++++++++++++++++++++
 3 files changed, 81 insertions(+), 3 deletions(-)

diff --git a/pom.xml b/pom.xml
index 4c489355281ba..9f6f2909c1af3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1966,8 +1966,8 @@ flexible messaging model and an intuitive client API.
         
         8
         8
-        
-        
+        
+        
       
     
     
diff --git a/pulsar-client-auth-athenz/pom.xml b/pulsar-client-auth-athenz/pom.xml
index 0b757d747872a..3cdbdb48a2935 100644
--- a/pulsar-client-auth-athenz/pom.xml
+++ b/pulsar-client-auth-athenz/pom.xml
@@ -61,10 +61,49 @@
       org.apache.commons
       commons-lang3
     
-
   
+
   
     
+      
+        org.apache.maven.plugins
+        maven-compiler-plugin
+        
+          ${pulsar.client.compiler.release}
+        
+      
+
+      
+        org.apache.maven.plugins
+        maven-enforcer-plugin
+        ${maven-enforcer-plugin.version}
+        
+          
+            enforce-bytecode-version
+            
+              enforce
+            
+            
+              
+                
+                  ${pulsar.client.compiler.release}
+                  
+                    test
+                  
+                
+              
+            
+          
+        
+        
+          
+            org.codehaus.mojo
+            extra-enforcer-rules
+            ${extra-enforcer-rules.version}
+          
+        
+      
+
       
         org.gaul
         modernizer-maven-plugin
diff --git a/pulsar-client-auth-sasl/pom.xml b/pulsar-client-auth-sasl/pom.xml
index 9744a230cf3af..c1143bf69331e 100644
--- a/pulsar-client-auth-sasl/pom.xml
+++ b/pulsar-client-auth-sasl/pom.xml
@@ -71,6 +71,45 @@
 
   
     
+      
+        org.apache.maven.plugins
+        maven-compiler-plugin
+        
+          ${pulsar.client.compiler.release}
+        
+      
+
+      
+        org.apache.maven.plugins
+        maven-enforcer-plugin
+        ${maven-enforcer-plugin.version}
+        
+          
+            enforce-bytecode-version
+            
+              enforce
+            
+            
+              
+                
+                  ${pulsar.client.compiler.release}
+                  
+                    test
+                  
+                
+              
+            
+          
+        
+        
+          
+            org.codehaus.mojo
+            extra-enforcer-rules
+            ${extra-enforcer-rules.version}
+          
+        
+      
+
       
         org.gaul
         modernizer-maven-plugin

From d1fc7323cbf61a6d2955486fc123fdde5253e72c Mon Sep 17 00:00:00 2001
From: Yunze Xu 
Date: Tue, 4 Apr 2023 10:31:33 +0800
Subject: [PATCH 172/174] [fix][broker] Ignore and remove the replicator cursor
 when the remote cluster is absent (#19972)

---
 .../service/persistent/PersistentTopic.java   | 30 +++++++--
 .../persistent/PersistentTopicTest.java       | 63 +++++++++++++++++++
 2 files changed, 87 insertions(+), 6 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
index fa08330ff3c35..18a662c4b7a38 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
@@ -1696,14 +1696,32 @@ public void openCursorFailed(ManagedLedgerException exception, Object ctx) {
         return future;
     }
 
+    private CompletableFuture checkReplicationCluster(String remoteCluster) {
+        return brokerService.getPulsar().getPulsarResources().getNamespaceResources()
+                .getPoliciesAsync(TopicName.get(topic).getNamespaceObject())
+                .thenApply(optPolicies -> optPolicies.map(policies -> policies.replication_clusters)
+                        .orElse(Collections.emptySet()).contains(remoteCluster)
+                        || topicPolicies.getReplicationClusters().get().contains(remoteCluster));
+    }
+
     protected CompletableFuture addReplicationCluster(String remoteCluster, ManagedCursor cursor,
             String localCluster) {
         return AbstractReplicator.validatePartitionedTopicAsync(PersistentTopic.this.getName(), brokerService)
-                .thenCompose(__ -> brokerService.pulsar().getPulsarResources().getClusterResources()
-                        .getClusterAsync(remoteCluster)
-                        .thenApply(clusterData ->
-                                brokerService.getReplicationClient(remoteCluster, clusterData)))
+                .thenCompose(__ -> checkReplicationCluster(remoteCluster))
+                .thenCompose(clusterExists -> {
+                    if (!clusterExists) {
+                        log.warn("Remove the replicator because the cluster '{}' does not exist", remoteCluster);
+                        return removeReplicator(remoteCluster).thenApply(__ -> null);
+                    }
+                    return brokerService.pulsar().getPulsarResources().getClusterResources()
+                            .getClusterAsync(remoteCluster)
+                            .thenApply(clusterData ->
+                                    brokerService.getReplicationClient(remoteCluster, clusterData));
+                })
                 .thenAccept(replicationClient -> {
+                    if (replicationClient == null) {
+                        return;
+                    }
                     Replicator replicator = replicators.computeIfAbsent(remoteCluster, r -> {
                         try {
                             return new GeoPersistentReplicator(PersistentTopic.this, cursor, localCluster,
@@ -1727,8 +1745,8 @@ CompletableFuture removeReplicator(String remoteCluster) {
 
         String name = PersistentReplicator.getReplicatorName(replicatorPrefix, remoteCluster);
 
-        replicators.get(remoteCluster).disconnect().thenRun(() -> {
-
+        Optional.ofNullable(replicators.get(remoteCluster)).map(Replicator::disconnect)
+                .orElse(CompletableFuture.completedFuture(null)).thenRun(() -> {
             ledger.asyncDeleteCursor(name, new DeleteCursorCallback() {
                 @Override
                 public void deleteCursorComplete(Object ctx) {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java
index 80a79e0234de4..c63be7aad01cd 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java
@@ -40,15 +40,20 @@
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Supplier;
 import lombok.Cleanup;
 import org.apache.bookkeeper.client.LedgerHandle;
+import org.apache.bookkeeper.mledger.ManagedCursor;
 import org.apache.bookkeeper.mledger.ManagedLedger;
 import org.apache.pulsar.broker.service.BrokerService;
 import org.apache.pulsar.broker.service.BrokerTestBase;
@@ -57,6 +62,7 @@
 import org.apache.pulsar.client.admin.PulsarAdminException;
 import org.apache.pulsar.client.api.Consumer;
 import org.apache.pulsar.client.api.Message;
+import org.apache.pulsar.client.api.MessageId;
 import org.apache.pulsar.client.api.MessageListener;
 import org.apache.pulsar.client.api.MessageRoutingMode;
 import org.apache.pulsar.client.api.Producer;
@@ -66,7 +72,9 @@
 import org.apache.pulsar.client.api.SubscriptionType;
 import org.apache.pulsar.common.naming.NamespaceBundle;
 import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.policies.data.ClusterData;
 import org.apache.pulsar.common.policies.data.Policies;
+import org.apache.pulsar.common.policies.data.TenantInfo;
 import org.apache.pulsar.common.policies.data.TopicStats;
 import org.awaitility.Awaitility;
 import org.junit.Assert;
@@ -525,4 +533,59 @@ public void testDeleteTopicFail() throws Exception {
         makeDeletedFailed.set(false);
         persistentTopic.delete().get();
     }
+
+    @DataProvider(name = "topicLevelPolicy")
+    public static Object[][] topicLevelPolicy() {
+        return new Object[][] { { true }, { false } };
+    }
+
+    @Test(dataProvider = "topicLevelPolicy")
+    public void testCreateTopicWithZombieReplicatorCursor(boolean topicLevelPolicy) throws Exception {
+        final String namespace = "prop/ns-abc";
+        final String topicName = "persistent://" + namespace
+                + "/testCreateTopicWithZombieReplicatorCursor" + topicLevelPolicy;
+        final String remoteCluster = "remote";
+        admin.topics().createNonPartitionedTopic(topicName);
+        admin.topics().createSubscription(topicName, conf.getReplicatorPrefix() + "." + remoteCluster,
+                MessageId.earliest, true);
+
+        admin.clusters().createCluster(remoteCluster, ClusterData.builder()
+                .serviceUrl("http://localhost:11112")
+                .brokerServiceUrl("pulsar://localhost:11111")
+                .build());
+        TenantInfo tenantInfo = admin.tenants().getTenantInfo("prop");
+        tenantInfo.getAllowedClusters().add(remoteCluster);
+        admin.tenants().updateTenant("prop", tenantInfo);
+
+        if (topicLevelPolicy) {
+            admin.topics().setReplicationClusters(topicName, Collections.singletonList(remoteCluster));
+        } else {
+            admin.namespaces().setNamespaceReplicationClustersAsync(
+                    namespace, Collections.singleton(remoteCluster)).get();
+        }
+
+        final PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName, false)
+                .get(3, TimeUnit.SECONDS).orElse(null);
+        assertNotNull(topic);
+
+        final Supplier> getCursors = () -> {
+            final Set cursors = new HashSet<>();
+            final Iterable iterable = topic.getManagedLedger().getCursors();
+            iterable.forEach(c -> cursors.add(c.getName()));
+            return cursors;
+        };
+        assertEquals(getCursors.get(), Collections.singleton(conf.getReplicatorPrefix() + "." + remoteCluster));
+
+        if (topicLevelPolicy) {
+            admin.topics().setReplicationClusters(topicName, Collections.emptyList());
+        } else {
+            admin.namespaces().setNamespaceReplicationClustersAsync(namespace, Collections.emptySet()).get();
+        }
+        admin.clusters().deleteCluster(remoteCluster);
+        // Now the cluster and its related policy has been removed but the replicator cursor still exists
+
+        topic.initialize().get(3, TimeUnit.SECONDS);
+        Awaitility.await().atMost(3, TimeUnit.SECONDS)
+                .until(() -> !topic.getManagedLedger().getCursors().iterator().hasNext());
+    }
 }

From 8c50a6c2e91c81dbf187ce5e66cb39e2758a741e Mon Sep 17 00:00:00 2001
From: Yunze Xu 
Date: Tue, 4 Apr 2023 15:52:34 +0800
Subject: [PATCH 173/174] [improve][client] PIP-229: Add a common interface to
 get fields of MessageIdData (#19414)

---
 .../service/PersistentFailoverE2ETest.java    |   8 +-
 .../broker/service/SubscriptionSeekTest.java  |   2 +-
 .../client/api/CustomMessageIdTest.java       | 142 +++++++++++
 .../api/PartitionedProducerConsumerTest.java  |   2 +-
 .../pulsar/client/impl/ConsumerAckTest.java   |   4 +-
 .../pulsar/client/impl/MessageIdTest.java     |   3 -
 .../pulsar/client/api/MessageIdAdv.java       | 122 ++++++++++
 .../pulsar/client/api/TopicMessageId.java     |  43 +++-
 .../impl/AcknowledgmentsGroupingTracker.java  |   2 +-
 .../pulsar/client/impl/BatchMessageAcker.java |  95 --------
 .../impl/BatchMessageAckerDisabled.java       |  50 ----
 .../client/impl/BatchMessageIdImpl.java       |  82 +++----
 .../client/impl/ChunkMessageIdImpl.java       |   9 +-
 .../pulsar/client/impl/ConsumerBase.java      |  15 +-
 .../pulsar/client/impl/ConsumerImpl.java      | 222 +++++++-----------
 .../pulsar/client/impl/MessageIdAdvUtils.java |  74 ++++++
 .../pulsar/client/impl/MessageIdImpl.java     |  71 +-----
 .../pulsar/client/impl/MessageImpl.java       |   8 +-
 .../impl/MessagePayloadContextImpl.java       |   9 +-
 .../client/impl/MultiTopicsConsumerImpl.java  |  44 ++--
 .../client/impl/NegativeAcksTracker.java      |  10 +-
 ...rsistentAcknowledgmentGroupingTracker.java |   2 +-
 ...sistentAcknowledgmentsGroupingTracker.java | 158 ++++++-------
 .../pulsar/client/impl/ResetCursorData.java   |  20 +-
 .../client/impl/TopicMessageIdImpl.java       |  39 +--
 .../pulsar/client/impl/TopicMessageImpl.java  |   2 +-
 .../client/impl/ZeroQueueConsumerImpl.java    |   3 +-
 .../src/main/resources/findbugsExclude.xml    |   5 +
 .../AcknowledgementsGroupingTrackerTest.java  |  13 +-
 .../impl/BatchMessageAckerDisabledTest.java   |  47 ----
 .../client/impl/BatchMessageAckerTest.java    |  83 -------
 .../client/impl/BatchMessageIdImplTest.java   |  33 +--
 .../impl/MessageIdSerializationTest.java      |   3 +-
 .../functions/utils/FunctionCommon.java       |   3 +-
 .../io/kafka/connect/KafkaConnectSink.java    |  36 ++-
 35 files changed, 686 insertions(+), 778 deletions(-)
 create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/client/api/CustomMessageIdTest.java
 create mode 100644 pulsar-client-api/src/main/java/org/apache/pulsar/client/api/MessageIdAdv.java
 delete mode 100644 pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageAcker.java
 delete mode 100644 pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageAckerDisabled.java
 create mode 100644 pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdAdvUtils.java
 delete mode 100644 pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageAckerDisabledTest.java
 delete mode 100644 pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageAckerTest.java

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentFailoverE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentFailoverE2ETest.java
index b263d4448d87a..f5895ec3761bf 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentFailoverE2ETest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentFailoverE2ETest.java
@@ -41,11 +41,11 @@
 import org.apache.pulsar.client.api.ConsumerEventListener;
 import org.apache.pulsar.client.api.Message;
 import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.MessageIdAdv;
 import org.apache.pulsar.client.api.MessageRoutingMode;
 import org.apache.pulsar.client.api.Producer;
 import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.client.api.SubscriptionType;
-import org.apache.pulsar.client.impl.MessageIdImpl;
 import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType;
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.util.FutureUtil;
@@ -370,8 +370,7 @@ public void testSimpleConsumerEventsWithPartition() throws Exception {
             }
             totalMessages++;
             consumer1.acknowledge(msg);
-            MessageIdImpl msgId = MessageIdImpl.convertToMessageIdImpl(msg.getMessageId());
-            receivedPtns.add(msgId.getPartitionIndex());
+            receivedPtns.add(((MessageIdAdv) msg.getMessageId()).getPartitionIndex());
         }
 
         assertTrue(Sets.difference(listener1.activePtns, receivedPtns).isEmpty());
@@ -387,8 +386,7 @@ public void testSimpleConsumerEventsWithPartition() throws Exception {
             }
             totalMessages++;
             consumer2.acknowledge(msg);
-            MessageIdImpl msgId = MessageIdImpl.convertToMessageIdImpl(msg.getMessageId());
-            receivedPtns.add(msgId.getPartitionIndex());
+            receivedPtns.add(((MessageIdAdv) msg.getMessageId()).getPartitionIndex());
         }
         assertTrue(Sets.difference(listener1.inactivePtns, receivedPtns).isEmpty());
         assertTrue(Sets.difference(listener2.activePtns, receivedPtns).isEmpty());
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java
index 93f2a42bcda35..b11946069c9dd 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SubscriptionSeekTest.java
@@ -678,7 +678,7 @@ public void testSeekByFunction() throws Exception {
             if (message == null) {
                 break;
             }
-            received.add(MessageIdImpl.convertToMessageIdImpl(message.getMessageId()));
+            received.add(message.getMessageId());
         }
         int msgNumFromPartition1 = list.size() / 2;
         int msgNumFromPartition2 = 1;
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/CustomMessageIdTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/CustomMessageIdTest.java
new file mode 100644
index 0000000000000..52bfc9dda37e4
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/CustomMessageIdTest.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.client.api;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+import java.util.ArrayList;
+import java.util.concurrent.TimeUnit;
+import lombok.Cleanup;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+@Test(groups = "broker-api")
+public class CustomMessageIdTest extends ProducerConsumerBase {
+
+    @BeforeClass
+    @Override
+    protected void setup() throws Exception {
+        super.internalSetup();
+        super.producerBaseSetup();
+    }
+
+    @AfterClass(alwaysRun = true)
+    @Override
+    protected void cleanup() throws Exception {
+        super.internalCleanup();
+    }
+
+    @DataProvider
+    public static Object[][] enableBatching() {
+        return new Object[][]{
+                { true },
+                { false }
+        };
+    }
+
+    @Test
+    public void testSeek() throws Exception {
+        final var topic = "persistent://my-property/my-ns/test-seek-" + System.currentTimeMillis();
+        @Cleanup final var producer = pulsarClient.newProducer(Schema.INT32).topic(topic).create();
+        final var msgIds = new ArrayList();
+        for (int i = 0; i < 10; i++) {
+            msgIds.add(new SimpleMessageIdImpl((MessageIdAdv) producer.send(i)));
+        }
+        @Cleanup final var consumer = pulsarClient.newConsumer(Schema.INT32)
+                .topic(topic).subscriptionName("sub").subscribe();
+        consumer.seek(msgIds.get(6));
+        final var msg = consumer.receive(3, TimeUnit.SECONDS);
+        assertNotNull(msg);
+        assertEquals(msg.getValue(), 7);
+    }
+
+    @Test(dataProvider = "enableBatching")
+    public void testAcknowledgment(boolean enableBatching) throws Exception {
+        final var topic = "persistent://my-property/my-ns/test-ack-"
+                + enableBatching + System.currentTimeMillis();
+        final var producer = pulsarClient.newProducer(Schema.INT32)
+                .topic(topic)
+                .enableBatching(enableBatching)
+                .batchingMaxMessages(10)
+                .batchingMaxPublishDelay(300, TimeUnit.MILLISECONDS)
+                .create();
+        final var consumer = pulsarClient.newConsumer(Schema.INT32)
+                .topic(topic)
+                .subscriptionName("sub")
+                .enableBatchIndexAcknowledgment(true)
+                .isAckReceiptEnabled(true)
+                .subscribe();
+        for (int i = 0; i < 10; i++) {
+            producer.sendAsync(i);
+        }
+        final var msgIds = new ArrayList();
+        for (int i = 0; i < 10; i++) {
+            final var msg = consumer.receive();
+            final var msgId = new SimpleMessageIdImpl((MessageIdAdv) msg.getMessageId());
+            msgIds.add(msgId);
+            if (enableBatching) {
+                assertTrue(msgId.getBatchIndex() >= 0 && msgId.getBatchSize() > 0);
+            } else {
+                assertFalse(msgId.getBatchIndex() >= 0 && msgId.getBatchSize() > 0);
+            }
+        }
+        consumer.acknowledgeCumulative(msgIds.get(8));
+        consumer.redeliverUnacknowledgedMessages();
+        final var msg = consumer.receive(3, TimeUnit.SECONDS);
+        assertNotNull(msg);
+        assertEquals(msg.getValue(), 9);
+    }
+
+    private record SimpleMessageIdImpl(long ledgerId, long entryId, int batchIndex, int batchSize)
+            implements MessageIdAdv {
+
+        public SimpleMessageIdImpl(MessageIdAdv msgId) {
+            this(msgId.getLedgerId(), msgId.getEntryId(), msgId.getBatchIndex(), msgId.getBatchSize());
+        }
+
+        @Override
+        public byte[] toByteArray() {
+            return new byte[0]; // never used
+        }
+
+        @Override
+        public long getLedgerId() {
+            return ledgerId;
+        }
+
+        @Override
+        public long getEntryId() {
+            return entryId;
+        }
+
+        @Override
+        public int getBatchIndex() {
+            return batchIndex;
+        }
+
+        @Override
+        public int getBatchSize() {
+            return batchSize;
+        }
+    }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/PartitionedProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/PartitionedProducerConsumerTest.java
index cd384e587898d..13ae991a0e89c 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/PartitionedProducerConsumerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/PartitionedProducerConsumerTest.java
@@ -767,7 +767,7 @@ public void testMessageIdForSubscribeToSinglePartition() throws Exception {
 
             for (int i = 0; i < totalMessages; i ++) {
                 msg = consumer1.receive(5, TimeUnit.SECONDS);
-                Assert.assertEquals(MessageIdImpl.convertToMessageIdImpl(msg.getMessageId()).getPartitionIndex(), 2);
+                Assert.assertEquals(((MessageIdAdv) msg.getMessageId()).getPartitionIndex(), 2);
                 consumer1.acknowledge(msg);
             }
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConsumerAckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConsumerAckTest.java
index 42da60906483c..a83283bc267b5 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConsumerAckTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConsumerAckTest.java
@@ -176,10 +176,10 @@ private AckTestData prepareDataForAck(String topic) throws PulsarClientException
             messageIds.add(message.getMessageId());
         }
         MessageId firstEntryMessageId = messageIds.get(0);
-        MessageId secondEntryMessageId = ((BatchMessageIdImpl) messageIds.get(1)).toMessageIdImpl();
+        MessageId secondEntryMessageId = MessageIdAdvUtils.discardBatch(messageIds.get(1));
         // Verify messages 2 to N must be in the same entry
         for (int i = 2; i < messageIds.size(); i++) {
-            assertEquals(((BatchMessageIdImpl) messageIds.get(i)).toMessageIdImpl(), secondEntryMessageId);
+            assertEquals(MessageIdAdvUtils.discardBatch(messageIds.get(i)), secondEntryMessageId);
         }
 
         assertTrue(interceptor.individualAckedMessageIdList.isEmpty());
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageIdTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageIdTest.java
index ceb5c51e6aa77..375bbff8a4df4 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageIdTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageIdTest.java
@@ -118,9 +118,6 @@ public void producerSendAsync(TopicType topicType) throws PulsarClientException,
             Message message = consumer.receive();
             assertEquals(new String(message.getData()), messagePrefix + i);
             MessageId messageId = message.getMessageId();
-            if (topicType == TopicType.PARTITIONED) {
-                messageId = MessageIdImpl.convertToMessageIdImpl(messageId);
-            }
             assertTrue(messageIds.remove(messageId), "Failed to receive message");
         }
         log.info("Remaining message IDs = {}", messageIds);
diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/MessageIdAdv.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/MessageIdAdv.java
new file mode 100644
index 0000000000000..73ecfed0ad059
--- /dev/null
+++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/MessageIdAdv.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.client.api;
+
+import java.util.BitSet;
+
+/**
+ * The {@link MessageId} interface provided for advanced users.
+ * 

+ * All built-in MessageId implementations should be able to be cast to MessageIdAdv.
+ * 

+ */
+public interface MessageIdAdv extends MessageId {
+
+    /**
+     * Get the ledger ID.
+     *
+     * @return the ledger ID
+     */
+    long getLedgerId();
+
+    /**
+     * Get the entry ID.
+     *
+     * @return the entry ID
+     */
+    long getEntryId();
+
+    /**
+     * Get the partition index.
+     *
+     * @return -1 if the message is from a non-partitioned topic, otherwise the non-negative partition index
+     */
+    default int getPartitionIndex() {
+        return -1;
+    }
+
+    /**
+     * Get the batch index.
+     *
+     * @return -1 if the message is not in a batch
+     */
+    default int getBatchIndex() {
+        return -1;
+    }
+
+    /**
+     * Get the batch size.
+     *
+     * @return 0 if the message is not in a batch
+     */
+    default int getBatchSize() {
+        return 0;
+    }
+
+    /**
+     * Get the BitSet that indicates which messages in the batch.
+     *
+     * @implNote The message IDs of a batch should share a BitSet. For example, given 3 messages in the same batch whose
+     * size is 3, all message IDs of them should return "111" (i.e. a BitSet whose size is 3 and all bits are 1). If the
+     * 1st message has been acknowledged, the returned BitSet should become "011" (i.e. the 1st bit become 0).
+     *
+     * @return null if the message is a non-batched message
+     */
+    default BitSet getAckSet() {
+        return null;
+    }
+
+    /**
+     * Get the message ID of the first chunk if the current message ID represents the position of a chunked message.
+     *
+     * @implNote A chunked message is distributed across different BookKeeper entries. The message ID of a chunked
+     * message is composed of two message IDs that represent positions of the first and the last chunk. The message ID
+     * itself represents the position of the last chunk.
+     *
+     * @return null if the message is not a chunked message
+     */
+    default MessageIdAdv getFirstChunkMessageId() {
+        return null;
+    }
+
+    /**
+     * The default implementation of {@link Comparable#compareTo(Object)}.
+     */
+    default int compareTo(MessageId o) {
+        if (!(o instanceof MessageIdAdv)) {
+            throw new UnsupportedOperationException("Unknown MessageId type: "
+                    + ((o != null) ? o.getClass().getName() : "null"));
+        }
+        final MessageIdAdv other = (MessageIdAdv) o;
+        int result = Long.compare(this.getLedgerId(), other.getLedgerId());
+        if (result != 0) {
+            return result;
+        }
+        result = Long.compare(this.getEntryId(), other.getEntryId());
+        if (result != 0) {
+            return result;
+        }
+        // TODO: Correct the following compare logics, see https://github.com/apache/pulsar/pull/18981
+        result = Integer.compare(this.getPartitionIndex(), other.getPartitionIndex());
+        if (result != 0) {
+            return result;
+        }
+        return Integer.compare(this.getBatchIndex(), other.getBatchIndex());
+    }
+}
diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TopicMessageId.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TopicMessageId.java
index f6109d5f8e87e..b70267bb0fb8b 100644
--- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TopicMessageId.java
+++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TopicMessageId.java
@@ -18,6 +18,8 @@
  */
 package org.apache.pulsar.client.api;
 
+import java.util.BitSet;
+
 /**
  * The MessageId used for a consumer that subscribes multiple topics or partitioned topics.
  *
@@ -49,13 +51,13 @@ static TopicMessageId create(String topic, MessageId messageId) {
     /**
      * The simplest implementation of a TopicMessageId interface.
      */
-    class Impl implements TopicMessageId {
+    class Impl implements MessageIdAdv, TopicMessageId {
         private final String topic;
-        private final MessageId messageId;
+        private final MessageIdAdv messageId;
 
         public Impl(String topic, MessageId messageId) {
             this.topic = topic;
-            this.messageId = messageId;
+            this.messageId = (MessageIdAdv) messageId;
         }
 
         @Override
@@ -68,6 +70,41 @@ public String getOwnerTopic() {
             return topic;
         }
 
+        @Override
+        public long getLedgerId() {
+            return messageId.getLedgerId();
+        }
+
+        @Override
+        public long getEntryId() {
+            return messageId.getEntryId();
+        }
+
+        @Override
+        public int getPartitionIndex() {
+            return messageId.getPartitionIndex();
+        }
+
+        @Override
+        public int getBatchIndex() {
+            return messageId.getBatchIndex();
+        }
+
+        @Override
+        public int getBatchSize() {
+            return messageId.getBatchSize();
+        }
+
+        @Override
+        public BitSet getAckSet() {
+            return messageId.getAckSet();
+        }
+
+        @Override
+        public MessageIdAdv getFirstChunkMessageId() {
+            return messageId.getFirstChunkMessageId();
+        }
+
         @Override
         public int compareTo(MessageId o) {
             return messageId.compareTo(o);
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AcknowledgmentsGroupingTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AcknowledgmentsGroupingTracker.java
index d46af1a99e7f0..60d7135e5e4ae 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AcknowledgmentsGroupingTracker.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AcknowledgmentsGroupingTracker.java
@@ -31,7 +31,7 @@ public interface AcknowledgmentsGroupingTracker extends AutoCloseable {
 
     boolean isDuplicate(MessageId messageId);
 
-    CompletableFuture addAcknowledgment(MessageIdImpl msgId, AckType ackType, Map properties);
+    CompletableFuture addAcknowledgment(MessageId msgId, AckType ackType, Map properties);
 
     CompletableFuture addListAcknowledgment(List messageIds, AckType ackType,
                                                   Map properties);
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageAcker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageAcker.java
deleted file mode 100644
index 1c9b66fd2bad5..0000000000000
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageAcker.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.impl;
-
-import java.util.BitSet;
-
-public class BatchMessageAcker {
-
-    private BatchMessageAcker() {
-        this.bitSet = new BitSet();
-        this.batchSize = 0;
-    }
-
-    static BatchMessageAcker newAcker(int batchSize) {
-        BitSet bitSet = new BitSet(batchSize);
-        bitSet.set(0, batchSize);
-        return new BatchMessageAcker(bitSet, batchSize);
-    }
-
-    // Use the param bitSet as the BatchMessageAcker's bitSet, don't care about the batchSize.
-    static BatchMessageAcker newAcker(BitSet bitSet) {
-        return new BatchMessageAcker(bitSet, -1);
-    }
-
-    // bitset shared across messages in the same batch.
-    private final int batchSize;
-    private final BitSet bitSet;
-    private volatile boolean prevBatchCumulativelyAcked = false;
-
-    BatchMessageAcker(BitSet bitSet, int batchSize) {
-        this.bitSet = bitSet;
-        this.batchSize = batchSize;
-    }
-
-    BitSet getBitSet() {
-        return bitSet;
-    }
-
-    public synchronized int getBatchSize() {
-        return batchSize;
-    }
-
-    public synchronized boolean ackIndividual(int batchIndex) {
-        bitSet.clear(batchIndex);
-        return bitSet.isEmpty();
-    }
-
-    public synchronized int getBitSetSize() {
-        return bitSet.size();
-    }
-
-    public synchronized boolean ackCumulative(int batchIndex) {
-        // +1 since to argument is exclusive
-        bitSet.clear(0, batchIndex + 1);
-        return bitSet.isEmpty();
-    }
-
-    // debug purpose
-    public synchronized int getOutstandingAcks() {
-        return bitSet.cardinality();
-    }
-
-    public void setPrevBatchCumulativelyAcked(boolean acked) {
-        this.prevBatchCumulativelyAcked = acked;
-    }
-
-    public boolean isPrevBatchCumulativelyAcked() {
-        return prevBatchCumulativelyAcked;
-    }
-
-    @Override
-    public String toString() {
-        return "BatchMessageAcker{"
-                + "batchSize=" + batchSize
-                + ", bitSet=" + bitSet
-                + ", prevBatchCumulativelyAcked=" + prevBatchCumulativelyAcked
-                + '}';
-    }
-}
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageAckerDisabled.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageAckerDisabled.java
deleted file mode 100644
index b70c928b29650..0000000000000
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageAckerDisabled.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.impl;
-
-import java.util.BitSet;
-
-class BatchMessageAckerDisabled extends BatchMessageAcker {
-
-    static final BatchMessageAckerDisabled INSTANCE = new BatchMessageAckerDisabled();
-
-    private BatchMessageAckerDisabled() {
-        super(new BitSet(), 0);
-    }
-
-    @Override
-    public synchronized int getBatchSize() {
-        return 0;
-    }
-
-    @Override
-    public boolean ackIndividual(int batchIndex) {
-        return true;
-    }
-
-    @Override
-    public boolean ackCumulative(int batchIndex) {
-        return true;
-    }
-
-    @Override
-    public int getOutstandingAcks() {
-        return 0;
-    }
-}
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageIdImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageIdImpl.java
index ed28082ff6a30..e9cddeb65d7f2 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageIdImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageIdImpl.java
@@ -18,20 +18,16 @@
  */
 package org.apache.pulsar.client.impl;
 
-import javax.annotation.Nonnull;
-import org.apache.pulsar.client.api.MessageId;
-import org.apache.pulsar.client.api.TopicMessageId;
+import java.util.BitSet;
+import org.apache.pulsar.client.api.MessageIdAdv;
 
-/**
- */
 public class BatchMessageIdImpl extends MessageIdImpl {
 
     private static final long serialVersionUID = 1L;
-    static final int NO_BATCH = -1;
     private final int batchIndex;
     private final int batchSize;
 
-    private final transient BatchMessageAcker acker;
+    private final BitSet ackSet;
 
     // Private constructor used only for json deserialization
     @SuppressWarnings("unused")
@@ -40,59 +36,35 @@ private BatchMessageIdImpl() {
     }
 
     public BatchMessageIdImpl(long ledgerId, long entryId, int partitionIndex, int batchIndex) {
-        this(ledgerId, entryId, partitionIndex, batchIndex, 0, BatchMessageAckerDisabled.INSTANCE);
+        this(ledgerId, entryId, partitionIndex, batchIndex, 0, null);
     }
 
     public BatchMessageIdImpl(long ledgerId, long entryId, int partitionIndex, int batchIndex, int batchSize,
-                              BatchMessageAcker acker) {
+                              BitSet ackSet) {
         super(ledgerId, entryId, partitionIndex);
         this.batchIndex = batchIndex;
         this.batchSize = batchSize;
-        this.acker = acker;
+        this.ackSet = ackSet;
     }
 
-    public BatchMessageIdImpl(MessageIdImpl other) {
-        super(other.ledgerId, other.entryId, other.partitionIndex);
-        if (other instanceof BatchMessageIdImpl) {
-            BatchMessageIdImpl otherId = (BatchMessageIdImpl) other;
-            this.batchIndex = otherId.batchIndex;
-            this.batchSize = otherId.batchSize;
-            this.acker = otherId.acker;
-        } else {
-            this.batchIndex = NO_BATCH;
-            this.batchSize = 0;
-            this.acker = BatchMessageAckerDisabled.INSTANCE;
-        }
+    public BatchMessageIdImpl(MessageIdAdv other) {
+        this(other.getLedgerId(), other.getEntryId(), other.getPartitionIndex(),
+                other.getBatchIndex(), other.getBatchSize(), other.getAckSet());
     }
 
+    @Override
     public int getBatchIndex() {
         return batchIndex;
     }
 
-    @Override
-    public int compareTo(@Nonnull MessageId o) {
-        if (o instanceof MessageIdImpl) {
-            MessageIdImpl other = (MessageIdImpl) o;
-            int batchIndex = (o instanceof BatchMessageIdImpl) ? ((BatchMessageIdImpl) o).batchIndex : NO_BATCH;
-            return messageIdCompare(
-                this.ledgerId, this.entryId, this.partitionIndex, this.batchIndex,
-                other.ledgerId, other.entryId, other.partitionIndex, batchIndex
-            );
-        } else if (o instanceof TopicMessageId) {
-            return compareTo(MessageIdImpl.convertToMessageIdImpl(o));
-        } else {
-            throw new UnsupportedOperationException("Unknown MessageId type: " + o.getClass().getName());
-        }
-    }
-
     @Override
     public int hashCode() {
-        return messageIdHashCode(ledgerId, entryId, partitionIndex, batchIndex);
+        return MessageIdAdvUtils.hashCode(this);
     }
 
     @Override
     public boolean equals(Object o) {
-        return super.equals(o);
+        return MessageIdAdvUtils.equals(this, o);
     }
 
     @Override
@@ -106,39 +78,51 @@ public byte[] toByteArray() {
         return toByteArray(batchIndex, batchSize);
     }
 
+    @Deprecated
     public boolean ackIndividual() {
-        return acker.ackIndividual(batchIndex);
+        return MessageIdAdvUtils.acknowledge(this, true);
     }
 
+    @Deprecated
     public boolean ackCumulative() {
-        return acker.ackCumulative(batchIndex);
+        return MessageIdAdvUtils.acknowledge(this, false);
     }
 
+    @Deprecated
     public int getOutstandingAcksInSameBatch() {
-        return acker.getOutstandingAcks();
+        return 0;
     }
 
+    @Override
     public int getBatchSize() {
-        return acker.getBatchSize();
+        return batchSize;
     }
 
+    @Deprecated
     public int getOriginalBatchSize() {
         return this.batchSize;
     }
 
+    @Deprecated
     public MessageIdImpl prevBatchMessageId() {
-        return new MessageIdImpl(
-            ledgerId, entryId - 1, partitionIndex);
+        return (MessageIdImpl) MessageIdAdvUtils.prevMessageId(this);
     }
 
     // MessageIdImpl is widely used as the key of a hash map, in this case, we should convert the batch message id to
     // have the correct hash code.
+    @Deprecated
     public MessageIdImpl toMessageIdImpl() {
-        return new MessageIdImpl(ledgerId, entryId, partitionIndex);
+        return (MessageIdImpl) MessageIdAdvUtils.discardBatch(this);
     }
 
-    public BatchMessageAcker getAcker() {
-        return acker;
+    @Override
+    public BitSet getAckSet() {
+        return ackSet;
     }
 
+    static BitSet newAckSet(int batchSize) {
+        final BitSet ackSet = new BitSet(batchSize);
+        ackSet.set(0, batchSize);
+        return ackSet;
+    }
 }
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ChunkMessageIdImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ChunkMessageIdImpl.java
index 28d5047c8ef25..29ce160442a3b 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ChunkMessageIdImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ChunkMessageIdImpl.java
@@ -21,10 +21,10 @@
 import io.netty.buffer.ByteBuf;
 import io.netty.buffer.Unpooled;
 import java.util.Objects;
-import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.MessageIdAdv;
 import org.apache.pulsar.common.api.proto.MessageIdData;
 
-public class ChunkMessageIdImpl extends MessageIdImpl implements MessageId {
+public class ChunkMessageIdImpl extends MessageIdImpl {
     private final MessageIdImpl firstChunkMsgId;
 
     public ChunkMessageIdImpl(MessageIdImpl firstChunkMsgId, MessageIdImpl lastChunkMsgId) {
@@ -32,11 +32,12 @@ public ChunkMessageIdImpl(MessageIdImpl firstChunkMsgId, MessageIdImpl lastChunk
         this.firstChunkMsgId = firstChunkMsgId;
     }
 
-    public MessageIdImpl getFirstChunkMessageId() {
+    @Override
+    public MessageIdAdv getFirstChunkMessageId() {
         return firstChunkMsgId;
     }
 
-    public MessageIdImpl getLastChunkMessageId() {
+    public MessageIdAdv getLastChunkMessageId() {
         return this;
     }
 
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java
index 75d3b2edf6e28..973b3302f4199 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java
@@ -47,6 +47,7 @@
 import org.apache.pulsar.client.api.ConsumerEventListener;
 import org.apache.pulsar.client.api.Message;
 import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.MessageIdAdv;
 import org.apache.pulsar.client.api.MessageListener;
 import org.apache.pulsar.client.api.Messages;
 import org.apache.pulsar.client.api.PulsarClientException;
@@ -82,7 +83,7 @@ public abstract class ConsumerBase extends HandlerState implements Consumer> incomingMessages;
-    protected ConcurrentOpenHashMap unAckedChunkedMessageIdSequenceMap;
+    protected ConcurrentOpenHashMap unAckedChunkedMessageIdSequenceMap;
     protected final ConcurrentLinkedQueue>> pendingReceives;
     protected final int maxReceiverQueueSize;
     private volatile int currentReceiverQueueSize;
@@ -128,7 +129,7 @@ protected ConsumerBase(PulsarClientImpl client, String topic, ConsumerConfigurat
         // Always use growable queue since items can exceed the advertised size
         this.incomingMessages = new GrowableArrayBlockingQueue<>();
         this.unAckedChunkedMessageIdSequenceMap =
-                ConcurrentOpenHashMap.newBuilder().build();
+                ConcurrentOpenHashMap.newBuilder().build();
         this.executorProvider = executorProvider;
         this.externalPinnedExecutor = executorProvider.getExecutor();
         this.internalPinnedExecutor = client.getInternalExecutorService();
@@ -223,14 +224,6 @@ protected void trackUnAckedMsgIfNoListener(MessageId messageId, int redeliveryCo
         }
     }
 
-    protected MessageId normalizeMessageId(MessageId messageId) {
-        if (messageId instanceof BatchMessageIdImpl) {
-            // do not add each item in batch message into tracker
-            return ((BatchMessageIdImpl) messageId).toMessageIdImpl();
-        }
-        return messageId;
-    }
-
     protected void reduceCurrentReceiverQueueSize() {
         if (!conf.isAutoScaledReceiverQueueSizeEnabled()) {
             return;
@@ -1131,7 +1124,7 @@ protected void callMessageListener(Message msg) {
                                 ? ((TopicMessageImpl) msg).getMessage() : msg));
             MessageId id;
             if (this instanceof ConsumerImpl) {
-                id = normalizeMessageId(msg.getMessageId());
+                id = MessageIdAdvUtils.discardBatch(msg.getMessageId());
             } else {
                 id = msg.getMessageId();
             }
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
index beaa34bf20520..1feef6ca0a642 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
@@ -35,6 +35,7 @@
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.BitSet;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedList;
@@ -71,6 +72,7 @@
 import org.apache.pulsar.client.api.Message;
 import org.apache.pulsar.client.api.MessageCrypto;
 import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.MessageIdAdv;
 import org.apache.pulsar.client.api.Messages;
 import org.apache.pulsar.client.api.Producer;
 import org.apache.pulsar.client.api.PulsarClientException;
@@ -154,12 +156,12 @@ public class ConsumerImpl extends ConsumerBase implements ConnectionHandle
     @Getter(AccessLevel.PACKAGE)
     private final int priorityLevel;
     private final SubscriptionMode subscriptionMode;
-    private volatile BatchMessageIdImpl startMessageId;
+    private volatile MessageIdAdv startMessageId;
 
-    private volatile BatchMessageIdImpl seekMessageId;
+    private volatile MessageIdAdv seekMessageId;
     private final AtomicBoolean duringSeek;
 
-    private final BatchMessageIdImpl initialStartMessageId;
+    private final MessageIdAdv initialStartMessageId;
 
     private final long startMessageRollbackDurationInSec;
 
@@ -178,7 +180,7 @@ public class ConsumerImpl extends ConsumerBase implements ConnectionHandle
     private final TopicName topicName;
     private final String topicNameWithoutPartition;
 
-    private final Map>> possibleSendToDeadLetterTopicMessages;
+    private final Map>> possibleSendToDeadLetterTopicMessages;
 
     private final DeadLetterPolicy deadLetterPolicy;
 
@@ -258,12 +260,8 @@ protected ConsumerImpl(PulsarClientImpl client, String topic, ConsumerConfigurat
         this.consumerId = client.newConsumerId();
         this.subscriptionMode = conf.getSubscriptionMode();
         if (startMessageId != null) {
-            if (startMessageId instanceof ChunkMessageIdImpl) {
-                this.startMessageId = new BatchMessageIdImpl(
-                        ((ChunkMessageIdImpl) startMessageId).getFirstChunkMessageId());
-            } else {
-                this.startMessageId = new BatchMessageIdImpl((MessageIdImpl) startMessageId);
-            }
+            MessageIdAdv firstChunkMessageId = ((MessageIdAdv) startMessageId).getFirstChunkMessageId();
+            this.startMessageId = (firstChunkMessageId == null) ? (MessageIdAdv) startMessageId : firstChunkMessageId;
         }
         this.initialStartMessageId = this.startMessageId;
         this.startMessageRollbackDurationInSec = startMessageRollbackDurationInSec;
@@ -535,7 +533,6 @@ protected CompletableFuture> internalBatchReceiveAsync() {
     protected CompletableFuture doAcknowledge(MessageId messageId, AckType ackType,
                                                     Map properties,
                                                     TransactionImpl txn) {
-        messageId = MessageIdImpl.convertToMessageIdImpl(messageId);
         if (getState() != State.Ready && getState() != State.Connecting) {
             stats.incrementNumAcksFailed();
             PulsarClientException exception = new PulsarClientException("Consumer not ready. State: " + getState());
@@ -551,16 +548,12 @@ protected CompletableFuture doAcknowledge(MessageId messageId, AckType ack
             return doTransactionAcknowledgeForResponse(messageId, ackType, null, properties,
                     new TxnID(txn.getTxnIdMostBits(), txn.getTxnIdLeastBits()));
         }
-        return acknowledgmentsGroupingTracker.addAcknowledgment((MessageIdImpl) messageId, ackType, properties);
+        return acknowledgmentsGroupingTracker.addAcknowledgment(messageId, ackType, properties);
     }
 
     @Override
     protected CompletableFuture doAcknowledge(List messageIdList, AckType ackType,
                                                     Map properties, TransactionImpl txn) {
-
-        for (MessageId messageId : messageIdList) {
-            checkArgument(messageId instanceof MessageIdImpl);
-        }
         if (getState() != State.Ready && getState() != State.Connecting) {
             stats.incrementNumAcksFailed();
             PulsarClientException exception = new PulsarClientException("Consumer not ready. State: " + getState());
@@ -572,7 +565,7 @@ protected CompletableFuture doAcknowledge(List messageIdList, A
             return FutureUtil.failedFuture(exception);
         }
         if (txn != null) {
-            return doTransactionAcknowledgeForResponse(messageIdList, ackType, null,
+            return doTransactionAcknowledgeForResponse(messageIdList, ackType,
                     properties, new TxnID(txn.getTxnIdMostBits(), txn.getTxnIdLeastBits()));
         } else {
             return this.acknowledgmentsGroupingTracker.addListAcknowledgment(messageIdList, ackType, properties);
@@ -592,7 +585,6 @@ protected CompletableFuture doReconsumeLater(Message message, AckType a
                     .InvalidMessageException("Cannot handle message with null messageId"));
         }
 
-        messageId = MessageIdImpl.convertToMessageIdImpl(messageId);
         if (getState() != State.Ready && getState() != State.Connecting) {
             stats.incrementNumAcksFailed();
             PulsarClientException exception = new PulsarClientException("Consumer not ready. State: " + getState());
@@ -922,7 +914,7 @@ protected void consumerIsReconnectedToBroker(ClientCnx cnx, int currentQueueSize
      * Clear the internal receiver queue and returns the message id of what was the 1st message in the queue that was
      * not seen by the application.
      */
-    private BatchMessageIdImpl clearReceiverQueue() {
+    private MessageIdAdv clearReceiverQueue() {
         List> currentMessageQueue = new ArrayList<>(incomingMessages.size());
         incomingMessages.drainTo(currentMessageQueue);
         resetIncomingMessageSize();
@@ -934,17 +926,16 @@ private BatchMessageIdImpl clearReceiverQueue() {
         }
 
         if (!currentMessageQueue.isEmpty()) {
-            MessageIdImpl nextMessageInQueue = (MessageIdImpl) currentMessageQueue.get(0).getMessageId();
-            BatchMessageIdImpl previousMessage;
-            if (nextMessageInQueue instanceof BatchMessageIdImpl) {
+            MessageIdAdv nextMessageInQueue = (MessageIdAdv) currentMessageQueue.get(0).getMessageId();
+            MessageIdAdv previousMessage;
+            if (MessageIdAdvUtils.isBatch(nextMessageInQueue)) {
                 // Get on the previous message within the current batch
                 previousMessage = new BatchMessageIdImpl(nextMessageInQueue.getLedgerId(),
                         nextMessageInQueue.getEntryId(), nextMessageInQueue.getPartitionIndex(),
-                        ((BatchMessageIdImpl) nextMessageInQueue).getBatchIndex() - 1);
+                        nextMessageInQueue.getBatchIndex() - 1);
             } else {
                 // Get on previous message in previous entry
-                previousMessage = new BatchMessageIdImpl(nextMessageInQueue.getLedgerId(),
-                        nextMessageInQueue.getEntryId() - 1, nextMessageInQueue.getPartitionIndex(), -1);
+                previousMessage = MessageIdAdvUtils.prevMessageId(nextMessageInQueue);
             }
             // release messages if they are pooled messages
             currentMessageQueue.forEach(Message::release);
@@ -1126,7 +1117,7 @@ protected  MessageImpl newSingleMessage(final int index,
                                                   final Schema schema,
                                                   final boolean containMetadata,
                                                   final BitSetRecyclable ackBitSet,
-                                                  final BatchMessageAcker acker,
+                                                  final BitSet ackSetInMessageId,
                                                   final int redeliveryCount,
                                                   final long consumerEpoch) {
         if (log.isDebugEnabled()) {
@@ -1161,7 +1152,7 @@ protected  MessageImpl newSingleMessage(final int index,
             }
 
             BatchMessageIdImpl batchMessageIdImpl = new BatchMessageIdImpl(messageId.getLedgerId(),
-                    messageId.getEntryId(), getPartitionIndex(), index, numMessages, acker);
+                    messageId.getEntryId(), getPartitionIndex(), index, numMessages, ackSetInMessageId);
 
             final ByteBuf payloadBuffer = (singleMessagePayload != null) ? singleMessagePayload : payload;
             final MessageImpl message = MessageImpl.create(topicName.toString(), batchMessageIdImpl,
@@ -1525,7 +1516,7 @@ void receiveIndividualMessagesFromBatch(BrokerEntryMetadata brokerEntryMetadata,
             possibleToDeadLetter = new ArrayList<>();
         }
 
-        BatchMessageAcker acker = BatchMessageAcker.newAcker(batchSize);
+        BitSet ackSetInMessageId = BatchMessageIdImpl.newAckSet(batchSize);
         BitSetRecyclable ackBitSet = null;
         if (ackSet != null && ackSet.size() > 0) {
             ackBitSet = BitSetRecyclable.valueOf(SafeCollectionUtils.longListToArray(ackSet));
@@ -1537,7 +1528,7 @@ void receiveIndividualMessagesFromBatch(BrokerEntryMetadata brokerEntryMetadata,
             for (int i = 0; i < batchSize; ++i) {
                 final MessageImpl message = newSingleMessage(i, batchSize, brokerEntryMetadata, msgMetadata,
                         singleMessageMetadata, uncompressedPayload, batchMessage, schema, true,
-                        ackBitSet, acker, redeliveryCount, consumerEpoch);
+                        ackBitSet, ackSetInMessageId, redeliveryCount, consumerEpoch);
                 if (message == null) {
                     skippedMessages++;
                     continue;
@@ -1634,7 +1625,7 @@ protected void trackMessage(MessageId messageId) {
 
     protected void trackMessage(MessageId messageId, int redeliveryCount) {
         if (conf.getAckTimeoutMillis() > 0 && messageId instanceof MessageIdImpl) {
-            MessageId id = normalizeMessageId(messageId);
+            MessageId id = MessageIdAdvUtils.discardBatch(messageId);
             if (hasParentConsumer) {
                 //TODO: check parent consumer here
                 // we should no longer track this message, TopicsConsumer will take care from now onwards
@@ -1931,8 +1922,6 @@ public void redeliverUnacknowledgedMessages(Set messageIds) {
             return;
         }
 
-        checkArgument(messageIds.stream().findFirst().get() instanceof MessageIdImpl);
-
         if (conf.getSubscriptionType() != SubscriptionType.Shared
                 && conf.getSubscriptionType() != SubscriptionType.Key_Shared) {
             // We cannot redeliver single messages if subscription type is not Shared
@@ -1942,11 +1931,7 @@ public void redeliverUnacknowledgedMessages(Set messageIds) {
         ClientCnx cnx = cnx();
         if (isConnected() && cnx.getRemoteEndpointProtocolVersion() >= ProtocolVersion.v2.getValue()) {
             int messagesFromQueue = removeExpiredMessagesFromQueue(messageIds);
-            Iterable> batches = Iterables.partition(
-                messageIds.stream()
-                    .map(messageId -> (MessageIdImpl) messageId)
-                    .collect(Collectors.toSet()), MAX_REDELIVER_UNACKNOWLEDGED);
-            batches.forEach(ids -> {
+            Iterables.partition(messageIds, MAX_REDELIVER_UNACKNOWLEDGED).forEach(ids -> {
                 getRedeliveryMessageIdData(ids).thenAccept(messageIdData -> {
                     if (!messageIdData.isEmpty()) {
                         ByteBuf cmd = Commands.newRedeliverUnacknowledgedMessages(consumerId, messageIdData);
@@ -1985,11 +1970,12 @@ protected void completeOpBatchReceive(OpBatchReceive op) {
         notifyPendingBatchReceivedCallBack(op.future);
     }
 
-    private CompletableFuture> getRedeliveryMessageIdData(List messageIds) {
+    private CompletableFuture> getRedeliveryMessageIdData(List messageIds) {
         if (messageIds == null || messageIds.isEmpty()) {
             return CompletableFuture.completedFuture(Collections.emptyList());
         }
-        List> futures = messageIds.stream().map(messageId -> {
+        List> futures = messageIds.stream().map(originalMessageId -> {
+            final MessageIdAdv messageId = (MessageIdAdv) originalMessageId;
             CompletableFuture future = processPossibleToDLQ(messageId);
             return future.thenApply(sendToDLQ -> {
                 if (!sendToDLQ) {
@@ -2005,20 +1991,15 @@ private CompletableFuture> getRedeliveryMessageIdData(List processPossibleToDLQ(MessageIdImpl messageId) {
+    private CompletableFuture processPossibleToDLQ(MessageIdAdv messageId) {
         List> deadLetterMessages = null;
         if (possibleSendToDeadLetterTopicMessages != null) {
-            if (messageId instanceof BatchMessageIdImpl) {
-                messageId = new MessageIdImpl(messageId.getLedgerId(), messageId.getEntryId(),
-                        getPartitionIndex());
-            }
-            deadLetterMessages = possibleSendToDeadLetterTopicMessages.get(messageId);
+            deadLetterMessages = possibleSendToDeadLetterTopicMessages.get(MessageIdAdvUtils.discardBatch(messageId));
         }
         CompletableFuture result = new CompletableFuture<>();
         if (deadLetterMessages != null) {
             initDeadLetterProducerIfNeeded();
             List> finalDeadLetterMessages = deadLetterMessages;
-            MessageIdImpl finalMessageId = messageId;
             deadLetterProducer.thenAcceptAsync(producerDLQ -> {
                 for (MessageImpl message : finalDeadLetterMessages) {
                     String originMessageIdStr = message.getMessageId().toString();
@@ -2032,12 +2013,12 @@ private CompletableFuture processPossibleToDLQ(MessageIdImpl messageId)
                     }
                     typedMessageBuilderNew.sendAsync()
                             .thenAccept(messageIdInDLQ -> {
-                                possibleSendToDeadLetterTopicMessages.remove(finalMessageId);
-                                acknowledgeAsync(finalMessageId).whenComplete((v, ex) -> {
+                                possibleSendToDeadLetterTopicMessages.remove(messageId);
+                                acknowledgeAsync(messageId).whenComplete((v, ex) -> {
                                     if (ex != null) {
                                         log.warn("[{}] [{}] [{}] Failed to acknowledge the message {} of the original"
                                                         + " topic but send to the DLQ successfully.",
-                                                topicName, subscription, consumerName, finalMessageId, ex);
+                                                topicName, subscription, consumerName, messageId, ex);
                                         result.complete(false);
                                     } else {
                                         result.complete(true);
@@ -2047,11 +2028,11 @@ private CompletableFuture processPossibleToDLQ(MessageIdImpl messageId)
                                 if (ex instanceof PulsarClientException.ProducerQueueIsFullError) {
                                     log.warn("[{}] [{}] [{}] Failed to send DLQ message to {} for message id {}: {}",
                                             topicName, subscription, consumerName,
-                                            deadLetterPolicy.getDeadLetterTopic(), finalMessageId, ex.getMessage());
+                                            deadLetterPolicy.getDeadLetterTopic(), messageId, ex.getMessage());
                                 } else {
                                     log.warn("[{}] [{}] [{}] Failed to send DLQ message to {} for message id {}",
                                             topicName, subscription, consumerName,
-                                            deadLetterPolicy.getDeadLetterTopic(), finalMessageId, ex);
+                                            deadLetterPolicy.getDeadLetterTopic(), messageId, ex);
                                 }
                                 result.complete(false);
                                 return null;
@@ -2154,8 +2135,8 @@ private CompletableFuture seekAsyncInternal(long requestId, ByteBuf seek,
         final CompletableFuture seekFuture = new CompletableFuture<>();
         ClientCnx cnx = cnx();
 
-        BatchMessageIdImpl originSeekMessageId = seekMessageId;
-        seekMessageId = new BatchMessageIdImpl((MessageIdImpl) seekId);
+        MessageIdAdv originSeekMessageId = seekMessageId;
+        seekMessageId = (MessageIdAdv) seekId;
         duringSeek.set(true);
         log.info("[{}][{}] Seeking subscription to {}", topic, subscription, seekBy);
 
@@ -2193,29 +2174,28 @@ public CompletableFuture seekAsync(long timestamp) {
     }
 
     @Override
-    public CompletableFuture seekAsync(MessageId originalMessageId) {
-        final MessageIdImpl messageId = MessageIdImpl.convertToMessageIdImpl(originalMessageId);
+    public CompletableFuture seekAsync(MessageId messageId) {
         String seekBy = String.format("the message %s", messageId.toString());
         return seekAsyncCheckState(seekBy).orElseGet(() -> {
             long requestId = client.newRequestId();
-            ByteBuf seek = null;
-            if (messageId instanceof BatchMessageIdImpl) {
-                BatchMessageIdImpl msgId = (BatchMessageIdImpl) messageId;
-                // Initialize ack set
-                BitSetRecyclable ackSet = BitSetRecyclable.create();
-                ackSet.set(0, msgId.getBatchSize());
-                ackSet.clear(0, Math.max(msgId.getBatchIndex(), 0));
-                long[] ackSetArr = ackSet.toLongArray();
-                ackSet.recycle();
-
-                seek = Commands.newSeek(consumerId, requestId, msgId.getLedgerId(), msgId.getEntryId(), ackSetArr);
-            } else if (messageId instanceof ChunkMessageIdImpl) {
-                ChunkMessageIdImpl msgId = (ChunkMessageIdImpl) messageId;
-                seek = Commands.newSeek(consumerId, requestId, msgId.getFirstChunkMessageId().getLedgerId(),
-                        msgId.getFirstChunkMessageId().getEntryId(), new long[0]);
+            final MessageIdAdv msgId = (MessageIdAdv) messageId;
+            final MessageIdAdv firstChunkMsgId = msgId.getFirstChunkMessageId();
+            final ByteBuf seek;
+            if (msgId.getFirstChunkMessageId() != null) {
+                seek = Commands.newSeek(consumerId, requestId, firstChunkMsgId.getLedgerId(),
+                        firstChunkMsgId.getEntryId(), new long[0]);
             } else {
-                seek = Commands.newSeek(
-                        consumerId, requestId, messageId.getLedgerId(), messageId.getEntryId(), new long[0]);
+                final long[] ackSetArr;
+                if (MessageIdAdvUtils.isBatch(msgId)) {
+                    final BitSetRecyclable ackSet = BitSetRecyclable.create();
+                    ackSet.set(0, msgId.getBatchSize());
+                    ackSet.clear(0, Math.max(msgId.getBatchIndex(), 0));
+                    ackSetArr = ackSet.toLongArray();
+                    ackSet.recycle();
+                } else {
+                    ackSetArr = new long[0];
+                }
+                seek = Commands.newSeek(consumerId, requestId, msgId.getLedgerId(), msgId.getEntryId(), ackSetArr);
             }
             return seekAsyncInternal(requestId, seek, messageId, seekBy);
         });
@@ -2247,9 +2227,8 @@ public CompletableFuture hasMessageAvailableAsync() {
                 }
 
                 future.thenAccept(response -> {
-                    MessageIdImpl lastMessageId = MessageIdImpl.convertToMessageIdImpl(response.lastMessageId);
-                    MessageIdImpl markDeletePosition = MessageIdImpl
-                            .convertToMessageIdImpl(response.markDeletePosition);
+                    MessageIdAdv lastMessageId = (MessageIdAdv) response.lastMessageId;
+                    MessageIdAdv markDeletePosition = (MessageIdAdv) response.markDeletePosition;
 
                     if (markDeletePosition != null && !(markDeletePosition.getEntryId() < 0
                             && markDeletePosition.getLedgerId() > lastMessageId.getLedgerId())) {
@@ -2434,16 +2413,6 @@ private void internalGetLastMessageIdAsync(final Backoff backoff,
         }
     }
 
-    private MessageIdImpl getMessageIdImpl(Message msg) {
-        MessageIdImpl messageId = (MessageIdImpl) msg.getMessageId();
-        if (messageId instanceof BatchMessageIdImpl) {
-            // messageIds contain MessageIdImpl, not BatchMessageIdImpl
-            messageId = new MessageIdImpl(messageId.getLedgerId(), messageId.getEntryId(), getPartitionIndex());
-        }
-        return messageId;
-    }
-
-
     private boolean isMessageUndecryptable(MessageMetadata msgMetadata) {
         return (msgMetadata.getEncryptionKeysCount() > 0 && conf.getCryptoKeyReader() == null
                 && conf.getCryptoFailureAction() == ConsumerCryptoFailureAction.CONSUME);
@@ -2486,7 +2455,7 @@ private int removeExpiredMessagesFromQueue(Set messageIds) {
         int messagesFromQueue = 0;
         Message peek = incomingMessages.peek();
         if (peek != null) {
-            MessageIdImpl messageId = getMessageIdImpl(peek);
+            MessageIdAdv messageId = MessageIdAdvUtils.discardBatch(peek.getMessageId());
             if (!messageIds.contains(messageId)) {
                 // first message is not expired, then no message is expired in queue.
                 return 0;
@@ -2497,7 +2466,7 @@ private int removeExpiredMessagesFromQueue(Set messageIds) {
             while (message != null) {
                 decreaseIncomingMessageSize(message);
                 messagesFromQueue++;
-                MessageIdImpl id = getMessageIdImpl(message);
+                MessageIdAdv id = MessageIdAdvUtils.discardBatch(message.getMessageId());
                 if (!messageIds.contains(id)) {
                     messageIds.add(id);
                     break;
@@ -2691,33 +2660,26 @@ private void removeChunkMessage(String msgUUID, ChunkedMessageCtx chunkedMsgCtx,
     private CompletableFuture doTransactionAcknowledgeForResponse(MessageId messageId, AckType ackType,
                                                                         ValidationError validationError,
                                                                         Map properties, TxnID txnID) {
-        BitSetRecyclable bitSetRecyclable = null;
-        long ledgerId;
-        long entryId;
-        ByteBuf cmd;
         long requestId = client.newRequestId();
-        if (messageId instanceof BatchMessageIdImpl) {
-            BatchMessageIdImpl batchMessageId = (BatchMessageIdImpl) messageId;
-            bitSetRecyclable = BitSetRecyclable.create();
-            ledgerId = batchMessageId.getLedgerId();
-            entryId = batchMessageId.getEntryId();
+        final MessageIdAdv messageIdAdv = (MessageIdAdv) messageId;
+        final long ledgerId = messageIdAdv.getLedgerId();
+        final long entryId = messageIdAdv.getEntryId();
+        final ByteBuf cmd;
+        if (MessageIdAdvUtils.isBatch(messageIdAdv)) {
+            BitSetRecyclable bitSetRecyclable = BitSetRecyclable.create();
+            bitSetRecyclable.set(0, messageIdAdv.getBatchSize());
             if (ackType == AckType.Cumulative) {
-                batchMessageId.ackCumulative();
-                bitSetRecyclable.set(0, batchMessageId.getBatchSize());
-                bitSetRecyclable.clear(0, batchMessageId.getBatchIndex() + 1);
+                MessageIdAdvUtils.acknowledge(messageIdAdv, false);
+                bitSetRecyclable.clear(0, messageIdAdv.getBatchIndex() + 1);
             } else {
-                bitSetRecyclable.set(0, batchMessageId.getBatchSize());
-                bitSetRecyclable.clear(batchMessageId.getBatchIndex());
+                bitSetRecyclable.clear(messageIdAdv.getBatchIndex());
             }
             cmd = Commands.newAck(consumerId, ledgerId, entryId, bitSetRecyclable, ackType, validationError, properties,
-                    txnID.getLeastSigBits(), txnID.getMostSigBits(), requestId, batchMessageId.getBatchSize());
+                    txnID.getLeastSigBits(), txnID.getMostSigBits(), requestId, messageIdAdv.getBatchSize());
             bitSetRecyclable.recycle();
         } else {
-            MessageIdImpl singleMessage = (MessageIdImpl) messageId;
-            ledgerId = singleMessage.getLedgerId();
-            entryId = singleMessage.getEntryId();
-            cmd = Commands.newAck(consumerId, ledgerId, entryId, bitSetRecyclable, ackType,
-                    validationError, properties, txnID.getLeastSigBits(), txnID.getMostSigBits(), requestId);
+            cmd = Commands.newAck(consumerId, ledgerId, entryId, null, ackType, validationError, properties,
+                    txnID.getLeastSigBits(), txnID.getMostSigBits(), requestId);
         }
 
         if (ackType == AckType.Cumulative) {
@@ -2736,58 +2698,42 @@ private CompletableFuture doTransactionAcknowledgeForResponse(MessageId me
     }
 
     private CompletableFuture doTransactionAcknowledgeForResponse(List messageIds, AckType ackType,
-                                                                        ValidationError validationError,
                                                                         Map properties, TxnID txnID) {
-        BitSetRecyclable bitSetRecyclable = null;
-        long ledgerId;
-        long entryId;
-        ByteBuf cmd;
         long requestId = client.newRequestId();
         List messageIdDataList = new LinkedList<>();
         for (MessageId messageId : messageIds) {
-            if (messageId instanceof BatchMessageIdImpl) {
-                BatchMessageIdImpl batchMessageId = (BatchMessageIdImpl) messageId;
-                bitSetRecyclable = BitSetRecyclable.create();
+            final MessageIdAdv messageIdAdv = (MessageIdAdv) messageId;
+            final MessageIdData messageIdData = new MessageIdData();
+            messageIdData.setLedgerId(messageIdAdv.getLedgerId());
+            messageIdData.setEntryId(messageIdAdv.getEntryId());
+            if (MessageIdAdvUtils.isBatch(messageIdAdv)) {
+                final BitSetRecyclable bitSetRecyclable = BitSetRecyclable.create();
+                bitSetRecyclable.set(0, messageIdAdv.getBatchSize());
                 if (ackType == AckType.Cumulative) {
-                    batchMessageId.ackCumulative();
-                    bitSetRecyclable.set(0, batchMessageId.getBatchSize());
-                    bitSetRecyclable.clear(0, batchMessageId.getBatchIndex() + 1);
+                    MessageIdAdvUtils.acknowledge(messageIdAdv, false);
+                    bitSetRecyclable.clear(0, messageIdAdv.getBatchIndex() + 1);
                 } else {
-                    bitSetRecyclable.set(0, batchMessageId.getBatchSize());
-                    bitSetRecyclable.clear(batchMessageId.getBatchIndex());
+                    bitSetRecyclable.clear(messageIdAdv.getBatchIndex());
                 }
-                MessageIdData messageIdData = new MessageIdData();
-                messageIdData.setLedgerId(batchMessageId.getLedgerId());
-                messageIdData.setEntryId(batchMessageId.getEntryId());
-                messageIdData.setBatchSize(batchMessageId.getBatchSize());
-                long[] as = bitSetRecyclable.toLongArray();
-                for (int i = 0; i < as.length; i++) {
-                    messageIdData.addAckSet(as[i]);
+                for (long x : bitSetRecyclable.toLongArray()) {
+                    messageIdData.addAckSet(x);
                 }
                 bitSetRecyclable.recycle();
-                messageIdDataList.add(messageIdData);
-            } else {
-                MessageIdImpl singleMessage = (MessageIdImpl) messageId;
-                ledgerId = singleMessage.getLedgerId();
-                entryId = singleMessage.getEntryId();
-                MessageIdData messageIdData = new MessageIdData();
-                messageIdData.setLedgerId(ledgerId);
-                messageIdData.setEntryId(entryId);
-                messageIdDataList.add(messageIdData);
             }
 
+            messageIdDataList.add(messageIdData);
             if (ackType == AckType.Cumulative) {
                 unAckedMessageTracker.removeMessagesTill(messageId);
             } else {
                 unAckedMessageTracker.remove(messageId);
             }
         }
-        cmd = Commands.newAck(consumerId, messageIdDataList, ackType, validationError, properties,
+        final ByteBuf cmd = Commands.newAck(consumerId, messageIdDataList, ackType, null, properties,
                 txnID.getLeastSigBits(), txnID.getMostSigBits(), requestId);
         return cnx().newAckForReceipt(cmd, requestId);
     }
 
-    public Map>> getPossibleSendToDeadLetterTopicMessages() {
+    public Map>> getPossibleSendToDeadLetterTopicMessages() {
         return possibleSendToDeadLetterTopicMessages;
     }
 
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdAdvUtils.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdAdvUtils.java
new file mode 100644
index 0000000000000..c8b18524ec052
--- /dev/null
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdAdvUtils.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.client.impl;
+
+import java.util.BitSet;
+import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.MessageIdAdv;
+
+public class MessageIdAdvUtils {
+
+    static int hashCode(MessageIdAdv msgId) {
+        return (int) (31 * (msgId.getLedgerId() + 31 * msgId.getEntryId())
+                + (31 * (long) msgId.getPartitionIndex()) + msgId.getBatchIndex());
+    }
+
+    static boolean equals(MessageIdAdv lhs, Object o) {
+        if (!(o instanceof MessageIdAdv)) {
+            return false;
+        }
+        final MessageIdAdv rhs = (MessageIdAdv) o;
+        return lhs.getLedgerId() == rhs.getLedgerId()
+                && lhs.getEntryId() == rhs.getEntryId()
+                && lhs.getPartitionIndex() == rhs.getPartitionIndex()
+                && lhs.getBatchIndex() == rhs.getBatchIndex();
+    }
+
+    static boolean acknowledge(MessageIdAdv msgId, boolean individual) {
+        if (!isBatch(msgId)) {
+            return true;
+        }
+        final BitSet ackSet = msgId.getAckSet();
+        if (ackSet == null) {
+            // The internal MessageId implementation should never reach here. If users have implemented their own
+            // MessageId and getAckSet() is not override, return false to avoid acknowledge current entry.
+            return false;
+        }
+        int batchIndex = msgId.getBatchIndex();
+        if (individual) {
+            ackSet.clear(batchIndex);
+        } else {
+            ackSet.clear(0, batchIndex + 1);
+        }
+        return ackSet.isEmpty();
+    }
+
+    static boolean isBatch(MessageIdAdv msgId) {
+        return msgId.getBatchIndex() >= 0 && msgId.getBatchSize() > 0;
+    }
+
+    static MessageIdAdv discardBatch(MessageId messageId) {
+        MessageIdAdv msgId = (MessageIdAdv) messageId;
+        return new MessageIdImpl(msgId.getLedgerId(), msgId.getEntryId(), msgId.getPartitionIndex());
+    }
+
+    static MessageIdAdv prevMessageId(MessageIdAdv msgId) {
+        return new MessageIdImpl(msgId.getLedgerId(), msgId.getEntryId() - 1, msgId.getPartitionIndex());
+    }
+}
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdImpl.java
index 1a0f491a6a7bb..83ee762578390 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdImpl.java
@@ -18,21 +18,17 @@
  */
 package org.apache.pulsar.client.impl;
 
-import static org.apache.pulsar.client.impl.BatchMessageIdImpl.NO_BATCH;
-import com.google.common.collect.ComparisonChain;
 import io.netty.buffer.ByteBuf;
 import io.netty.buffer.Unpooled;
 import io.netty.util.concurrent.FastThreadLocal;
 import java.io.IOException;
 import java.util.Objects;
-import javax.annotation.Nonnull;
 import org.apache.pulsar.client.api.MessageId;
-import org.apache.pulsar.client.api.TopicMessageId;
+import org.apache.pulsar.client.api.MessageIdAdv;
 import org.apache.pulsar.common.api.proto.MessageIdData;
-import org.apache.pulsar.common.classification.InterfaceStability;
 import org.apache.pulsar.common.naming.TopicName;
 
-public class MessageIdImpl implements MessageId {
+public class MessageIdImpl implements MessageIdAdv {
     protected final long ledgerId;
     protected final long entryId;
     protected final int partitionIndex;
@@ -49,28 +45,29 @@ public MessageIdImpl(long ledgerId, long entryId, int partitionIndex) {
         this.partitionIndex = partitionIndex;
     }
 
+    @Override
     public long getLedgerId() {
         return ledgerId;
     }
 
+    @Override
     public long getEntryId() {
         return entryId;
     }
 
+    @Override
     public int getPartitionIndex() {
         return partitionIndex;
     }
 
     @Override
     public int hashCode() {
-        return messageIdHashCode(ledgerId, entryId, partitionIndex, NO_BATCH);
+        return MessageIdAdvUtils.hashCode(this);
     }
 
     @Override
     public boolean equals(Object o) {
-        return (o instanceof MessageId)
-                && !(o instanceof MultiMessageIdImpl)
-                && (compareTo((MessageId) o) == 0);
+        return MessageIdAdvUtils.equals(this, o);
     }
 
     @Override
@@ -100,7 +97,7 @@ public static MessageId fromByteArray(byte[] data) throws IOException {
         if (idData.hasBatchIndex()) {
             if (idData.hasBatchSize()) {
                 messageId = new BatchMessageIdImpl(idData.getLedgerId(), idData.getEntryId(), idData.getPartition(),
-                    idData.getBatchIndex(), idData.getBatchSize(), BatchMessageAcker.newAcker(idData.getBatchSize()));
+                    idData.getBatchIndex(), idData.getBatchSize(), BatchMessageIdImpl.newAckSet(idData.getBatchSize()));
             } else {
                 messageId = new BatchMessageIdImpl(idData.getLedgerId(), idData.getEntryId(), idData.getPartition(),
                     idData.getBatchIndex());
@@ -118,22 +115,6 @@ public static MessageId fromByteArray(byte[] data) throws IOException {
         return messageId;
     }
 
-    @InterfaceStability.Unstable
-    public static MessageIdImpl convertToMessageIdImpl(MessageId messageId) {
-        if (messageId instanceof TopicMessageId) {
-            if (messageId instanceof TopicMessageIdImpl) {
-                return (MessageIdImpl) ((TopicMessageIdImpl) messageId).getInnerMessageId();
-            } else {
-                try {
-                    return (MessageIdImpl) MessageId.fromByteArray(messageId.toByteArray());
-                } catch (IOException e) {
-                    throw new RuntimeException(e);
-                }
-            }
-        }
-        return (MessageIdImpl) messageId;
-    }
-
     public static MessageId fromByteArrayWithTopic(byte[] data, String topicName) throws IOException {
         return fromByteArrayWithTopic(data, TopicName.get(topicName));
     }
@@ -152,10 +133,10 @@ public static MessageId fromByteArrayWithTopic(byte[] data, TopicName topicName)
             if (idData.hasBatchSize()) {
                 messageId = new BatchMessageIdImpl(idData.getLedgerId(), idData.getEntryId(), idData.getPartition(),
                         idData.getBatchIndex(), idData.getBatchSize(),
-                        BatchMessageAcker.newAcker(idData.getBatchSize()));
+                        BatchMessageIdImpl.newAckSet(idData.getBatchSize()));
             } else {
                 messageId = new BatchMessageIdImpl(idData.getLedgerId(), idData.getEntryId(), idData.getPartition(),
-                        idData.getBatchIndex(), 0, BatchMessageAckerDisabled.INSTANCE);
+                        idData.getBatchIndex(), 0, null);
             }
         } else {
             messageId = new MessageIdImpl(idData.getLedgerId(), idData.getEntryId(), idData.getPartition());
@@ -207,36 +188,4 @@ public byte[] toByteArray() {
         // there is no message batch so we pass -1
         return toByteArray(-1, 0);
     }
-
-    @Override
-    public int compareTo(@Nonnull MessageId o) {
-        if (o instanceof MessageIdImpl) {
-            MessageIdImpl other = (MessageIdImpl) o;
-            int batchIndex = (o instanceof BatchMessageIdImpl) ? ((BatchMessageIdImpl) o).getBatchIndex() : NO_BATCH;
-            return messageIdCompare(
-                this.ledgerId, this.entryId, this.partitionIndex, NO_BATCH,
-                other.ledgerId, other.entryId, other.partitionIndex, batchIndex
-            );
-        } else if (o instanceof TopicMessageId) {
-            return compareTo(convertToMessageIdImpl(o));
-        } else {
-            throw new UnsupportedOperationException("Unknown MessageId type: " + o.getClass().getName());
-        }
-    }
-
-    static int messageIdHashCode(long ledgerId, long entryId, int partitionIndex, int batchIndex) {
-        return (int) (31 * (ledgerId + 31 * entryId) + (31 * (long) partitionIndex) + batchIndex);
-    }
-
-    static int messageIdCompare(
-        long ledgerId1, long entryId1, int partitionIndex1, int batchIndex1,
-        long ledgerId2, long entryId2, int partitionIndex2, int batchIndex2
-    ) {
-        return ComparisonChain.start()
-            .compare(ledgerId1, ledgerId2)
-            .compare(entryId1, entryId2)
-            .compare(partitionIndex1, partitionIndex2)
-            .compare(batchIndex1, batchIndex2)
-            .result();
-    }
 }
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java
index 0b6fb608ee62b..d369d639a73a0 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java
@@ -40,6 +40,7 @@
 import lombok.Getter;
 import org.apache.pulsar.client.api.Message;
 import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.MessageIdAdv;
 import org.apache.pulsar.client.api.Schema;
 import org.apache.pulsar.client.api.SchemaSerializationException;
 import org.apache.pulsar.client.impl.schema.AbstractSchema;
@@ -714,9 +715,10 @@ public boolean hasIndex() {
     @Override
     public Optional getIndex() {
         if (brokerEntryMetadata != null && brokerEntryMetadata.hasIndex()) {
-            if (msgMetadata.hasNumMessagesInBatch() && messageId instanceof BatchMessageIdImpl) {
-                int batchSize = ((BatchMessageIdImpl) messageId).getBatchSize();
-                int batchIndex = ((BatchMessageIdImpl) messageId).getBatchIndex();
+            MessageIdAdv messageIdAdv = (MessageIdAdv) messageId;
+            if (msgMetadata.hasNumMessagesInBatch() && MessageIdAdvUtils.isBatch(messageIdAdv)) {
+                int batchSize = messageIdAdv.getBatchSize();
+                int batchIndex = messageIdAdv.getBatchIndex();
                 return Optional.of(brokerEntryMetadata.getIndex() - batchSize + batchIndex + 1);
             }
             return Optional.of(brokerEntryMetadata.getIndex());
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessagePayloadContextImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessagePayloadContextImpl.java
index dcae86bd01a3b..f4c9aa2707477 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessagePayloadContextImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessagePayloadContextImpl.java
@@ -21,6 +21,7 @@
 import static org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH;
 import io.netty.buffer.ByteBuf;
 import io.netty.util.Recycler;
+import java.util.BitSet;
 import java.util.List;
 import lombok.NonNull;
 import org.apache.pulsar.client.api.Message;
@@ -50,7 +51,7 @@ protected MessagePayloadContextImpl newObject(Handle
     private MessageIdImpl messageId;
     private ConsumerImpl consumer;
     private int redeliveryCount;
-    private BatchMessageAcker acker;
+    private BitSet ackSetInMessageId;
     private BitSetRecyclable ackBitSet;
     private long consumerEpoch;
 
@@ -73,7 +74,7 @@ public static MessagePayloadContextImpl get(final BrokerEntryMetadata brokerEntr
         context.messageId = messageId;
         context.consumer = consumer;
         context.redeliveryCount = redeliveryCount;
-        context.acker = BatchMessageAcker.newAcker(context.getNumMessages());
+        context.ackSetInMessageId = BatchMessageIdImpl.newAckSet(context.getNumMessages());
         context.ackBitSet = (ackSet != null && ackSet.size() > 0)
                 ? BitSetRecyclable.valueOf(SafeCollectionUtils.longListToArray(ackSet))
                 : null;
@@ -88,7 +89,7 @@ public void recycle() {
         consumer = null;
         redeliveryCount = 0;
         consumerEpoch = DEFAULT_CONSUMER_EPOCH;
-        acker = null;
+        ackSetInMessageId = null;
         if (ackBitSet != null) {
             ackBitSet.recycle();
             ackBitSet = null;
@@ -134,7 +135,7 @@ public  Message getMessageAt(int index,
                     schema,
                     containMetadata,
                     ackBitSet,
-                    acker,
+                    ackSetInMessageId,
                     redeliveryCount,
                     consumerEpoch);
         } finally {
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java
index f993304b0780a..5fe0e4a82b840 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java
@@ -55,6 +55,7 @@
 import org.apache.pulsar.client.api.ConsumerStats;
 import org.apache.pulsar.client.api.Message;
 import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.MessageIdAdv;
 import org.apache.pulsar.client.api.Messages;
 import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.client.api.PulsarClientException.NotSupportedException;
@@ -100,7 +101,7 @@ public class MultiTopicsConsumerImpl extends ConsumerBase {
     private final MultiTopicConsumerStatsRecorderImpl stats;
     private final ConsumerConfigurationData internalConfig;
 
-    private volatile BatchMessageIdImpl startMessageId = null;
+    private volatile MessageIdAdv startMessageId;
     private final long startMessageRollbackDurationInSec;
     MultiTopicsConsumerImpl(PulsarClientImpl client, ConsumerConfigurationData conf,
             ExecutorProvider executorProvider, CompletableFuture> subscribeFuture, Schema schema,
@@ -139,9 +140,7 @@ public class MultiTopicsConsumerImpl extends ConsumerBase {
         this.consumers = new ConcurrentHashMap<>();
         this.pausedConsumers = new ConcurrentLinkedQueue<>();
         this.allTopicPartitionsNumber = new AtomicInteger(0);
-        this.startMessageId = startMessageId != null
-                ? new BatchMessageIdImpl(MessageIdImpl.convertToMessageIdImpl(startMessageId))
-                : null;
+        this.startMessageId = (MessageIdAdv) startMessageId;
         this.startMessageRollbackDurationInSec = startMessageRollbackDurationInSec;
         this.paused = conf.isStartPaused();
 
@@ -454,18 +453,15 @@ protected CompletableFuture doAcknowledge(MessageId messageId, AckType ack
             return FutureUtil.failedFuture(new PulsarClientException("Consumer already closed"));
         }
 
-        TopicMessageId topicMessageId = (TopicMessageId) messageId;
-        ConsumerImpl consumer = consumers.get(topicMessageId.getOwnerTopic());
+        ConsumerImpl consumer = consumers.get(((TopicMessageId) messageId).getOwnerTopic());
         if (consumer == null) {
             return FutureUtil.failedFuture(new PulsarClientException.NotConnectedException());
         }
-        MessageId innerMessageId = MessageIdImpl.convertToMessageIdImpl(topicMessageId);
         if (ackType == AckType.Cumulative) {
-            return consumer.acknowledgeCumulativeAsync(innerMessageId);
+            return consumer.acknowledgeCumulativeAsync(messageId);
         } else {
-            return consumer.doAcknowledgeWithTxn(innerMessageId, ackType, properties, txnImpl)
-                .thenRun(() ->
-                    unAckedMessageTracker.remove(topicMessageId));
+            return consumer.doAcknowledgeWithTxn(messageId, ackType, properties, txnImpl)
+                .thenRun(() -> unAckedMessageTracker.remove(messageId));
         }
     }
 
@@ -490,10 +486,9 @@ protected CompletableFuture doAcknowledge(List messageIdList,
             }
             Map> topicToMessageIdMap = new HashMap<>();
             for (MessageId messageId : messageIdList) {
-                TopicMessageId topicMessageId = (TopicMessageId) messageId;
-                topicToMessageIdMap.putIfAbsent(topicMessageId.getOwnerTopic(), new ArrayList<>());
-                topicToMessageIdMap.get(topicMessageId.getOwnerTopic())
-                        .add(MessageIdImpl.convertToMessageIdImpl(topicMessageId));
+                String ownerTopic = ((TopicMessageId) messageId).getOwnerTopic();
+                topicToMessageIdMap.putIfAbsent(ownerTopic, new ArrayList<>());
+                topicToMessageIdMap.get(ownerTopic).add(messageId);
             }
             final Map, List> consumerToMessageIds = new IdentityHashMap<>();
             for (Map.Entry> entry : topicToMessageIdMap.entrySet()) {
@@ -549,10 +544,8 @@ protected CompletableFuture doReconsumeLater(Message message, AckType a
     @Override
     public void negativeAcknowledge(MessageId messageId) {
         checkArgument(messageId instanceof TopicMessageId);
-        TopicMessageId topicMessageId = (TopicMessageId) messageId;
-
-        ConsumerImpl consumer = consumers.get(topicMessageId.getOwnerTopic());
-        consumer.negativeAcknowledge(MessageIdImpl.convertToMessageIdImpl(topicMessageId));
+        ConsumerImpl consumer = consumers.get(((TopicMessageId) messageId).getOwnerTopic());
+        consumer.negativeAcknowledge(messageId);
     }
 
     @Override
@@ -705,12 +698,11 @@ public void redeliverUnacknowledgedMessages(Set messageIds) {
             return;
         }
         removeExpiredMessagesFromQueue(messageIds);
-        messageIds.stream().map(messageId -> (TopicMessageId) messageId)
-            .collect(Collectors.groupingBy(TopicMessageId::getOwnerTopic, Collectors.toSet()))
-            .forEach((topicName, messageIds1) ->
-                consumers.get(topicName)
-                    .redeliverUnacknowledgedMessages(messageIds1.stream()
-                        .map(MessageIdImpl::convertToMessageIdImpl).collect(Collectors.toSet())));
+        messageIds.stream()
+                .collect(Collectors.groupingBy(
+                        msgId -> ((TopicMessageIdImpl) msgId).getOwnerTopic(), Collectors.toSet()))
+                .forEach((topicName, messageIds1) ->
+                        consumers.get(topicName).redeliverUnacknowledgedMessages(messageIds1));
         resumeReceivingFromPausedConsumersIfNeeded();
     }
 
@@ -1508,7 +1500,7 @@ public CompletableFuture getLastMessageIdAsync() {
 
     public static boolean isIllegalMultiTopicsMessageId(MessageId messageId) {
         //only support earliest/latest
-        return !MessageId.earliest.equals(messageId) && !MessageId.latest.equals(messageId);
+        return !messageId.equals(MessageId.earliest) && !messageId.equals(MessageId.latest);
     }
 
     public void tryAcknowledgeMessage(Message msg) {
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NegativeAcksTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NegativeAcksTracker.java
index 70d57db3bb691..37f58a0218091 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NegativeAcksTracker.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NegativeAcksTracker.java
@@ -95,14 +95,6 @@ public synchronized void add(Message message) {
     }
 
     private synchronized void add(MessageId messageId, int redeliveryCount) {
-        messageId = MessageIdImpl.convertToMessageIdImpl(messageId);
-
-        if (messageId instanceof BatchMessageIdImpl) {
-            BatchMessageIdImpl batchMessageId = (BatchMessageIdImpl) messageId;
-            messageId = new MessageIdImpl(batchMessageId.getLedgerId(), batchMessageId.getEntryId(),
-                    batchMessageId.getPartitionIndex());
-        }
-
         if (nackedMessages == null) {
             nackedMessages = new HashMap<>();
         }
@@ -113,7 +105,7 @@ private synchronized void add(MessageId messageId, int redeliveryCount) {
         } else {
             backoffNs = nackDelayNanos;
         }
-        nackedMessages.put(messageId, System.nanoTime() + backoffNs);
+        nackedMessages.put(MessageIdAdvUtils.discardBatch(messageId), System.nanoTime() + backoffNs);
 
         if (this.timeout == null) {
             // Schedule a task and group all the redeliveries for same period. Leave a small buffer to allow for
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NonPersistentAcknowledgmentGroupingTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NonPersistentAcknowledgmentGroupingTracker.java
index 32f8fb922304b..e8951cd3d1692 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NonPersistentAcknowledgmentGroupingTracker.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/NonPersistentAcknowledgmentGroupingTracker.java
@@ -43,7 +43,7 @@ public boolean isDuplicate(MessageId messageId) {
         return false;
     }
 
-    public CompletableFuture addAcknowledgment(MessageIdImpl msgId, AckType ackType, Map addAcknowledgment(MessageId msgId, AckType ackType, Map properties) {
         // no-op
         return CompletableFuture.completedFuture(null);
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java
index fef0bcb8906f1..9086ccc4ef0e0 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java
@@ -23,6 +23,7 @@
 import io.netty.channel.EventLoopGroup;
 import io.netty.util.concurrent.FastThreadLocal;
 import java.util.ArrayList;
+import java.util.BitSet;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -43,6 +44,7 @@
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.tuple.Triple;
 import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.MessageIdAdv;
 import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData;
 import org.apache.pulsar.client.util.TimedCompletableFuture;
@@ -79,8 +81,8 @@ public class PersistentAcknowledgmentsGroupingTracker implements Acknowledgments
      * This is a set of all the individual acks that the application has issued and that were not already sent to
      * broker.
      */
-    private final ConcurrentSkipListSet pendingIndividualAcks;
-    private final ConcurrentHashMap pendingIndividualBatchIndexAcks;
+    private final ConcurrentSkipListSet pendingIndividualAcks;
+    private final ConcurrentHashMap pendingIndividualBatchIndexAcks;
 
     private final ScheduledFuture scheduledTask;
     private final boolean batchIndexAckEnabled;
@@ -113,18 +115,16 @@ public PersistentAcknowledgmentsGroupingTracker(ConsumerImpl consumer, Consum
      */
     @Override
     public boolean isDuplicate(MessageId messageId) {
-        if (!(messageId instanceof MessageIdImpl)) {
+        if (!(messageId instanceof MessageIdAdv)) {
             throw new IllegalArgumentException("isDuplicated cannot accept "
                     + messageId.getClass().getName() + ": " + messageId);
         }
-        if (lastCumulativeAck.compareTo(messageId) >= 0) {
+        final MessageIdAdv messageIdAdv = (MessageIdAdv) messageId;
+        if (lastCumulativeAck.compareTo(messageIdAdv) >= 0) {
             // Already included in a cumulative ack
             return true;
         } else {
-            final MessageIdImpl messageIdImpl = (messageId instanceof BatchMessageIdImpl)
-                    ? ((BatchMessageIdImpl) messageId).toMessageIdImpl()
-                    : (MessageIdImpl) messageId;
-            return pendingIndividualAcks.contains(messageIdImpl);
+            return pendingIndividualAcks.contains(MessageIdAdvUtils.discardBatch(messageIdAdv));
         }
     }
 
@@ -135,10 +135,10 @@ public CompletableFuture addListAcknowledgment(List messageIds,
             if (consumer.isAckReceiptEnabled()) {
                 Set> completableFutureSet = new HashSet<>();
                 messageIds.forEach(messageId ->
-                        completableFutureSet.add(addAcknowledgment((MessageIdImpl) messageId, ackType, properties)));
+                        completableFutureSet.add(addAcknowledgment(messageId, ackType, properties)));
                 return FutureUtil.waitForAll(new ArrayList<>(completableFutureSet));
             } else {
-                messageIds.forEach(messageId -> addAcknowledgment((MessageIdImpl) messageId, ackType, properties));
+                messageIds.forEach(messageId -> addAcknowledgment(messageId, ackType, properties));
                 return CompletableFuture.completedFuture(null);
             }
         } else {
@@ -162,46 +162,43 @@ public CompletableFuture addListAcknowledgment(List messageIds,
 
     private void addListAcknowledgment(List messageIds) {
         for (MessageId messageId : messageIds) {
-            if (messageId instanceof BatchMessageIdImpl) {
-                BatchMessageIdImpl batchMessageId = (BatchMessageIdImpl) messageId;
-                addIndividualAcknowledgment(batchMessageId.toMessageIdImpl(),
-                        batchMessageId,
+            MessageIdAdv messageIdAdv = (MessageIdAdv) messageId;
+            if (MessageIdAdvUtils.isBatch(messageIdAdv)) {
+                addIndividualAcknowledgment(MessageIdAdvUtils.discardBatch(messageIdAdv),
+                        messageIdAdv,
                         this::doIndividualAckAsync,
                         this::doIndividualBatchAckAsync);
-            } else if (messageId instanceof MessageIdImpl) {
-                addIndividualAcknowledgment((MessageIdImpl) messageId,
+            } else {
+                addIndividualAcknowledgment(messageIdAdv,
                         null,
                         this::doIndividualAckAsync,
                         this::doIndividualBatchAckAsync);
-            } else {
-                throw new IllegalStateException("Unsupported message id type in addListAcknowledgement: "
-                        + messageId.getClass().getCanonicalName());
             }
         }
     }
 
     @Override
-    public CompletableFuture addAcknowledgment(MessageIdImpl msgId, AckType ackType,
+    public CompletableFuture addAcknowledgment(MessageId msgId, AckType ackType,
                                                      Map properties) {
-        if (msgId instanceof BatchMessageIdImpl) {
-            BatchMessageIdImpl batchMessageId = (BatchMessageIdImpl) msgId;
-            return addAcknowledgment(batchMessageId.toMessageIdImpl(), ackType, properties, batchMessageId);
+        MessageIdAdv msgIdAdv = (MessageIdAdv) msgId;
+        if (MessageIdAdvUtils.isBatch(msgIdAdv)) {
+            return addAcknowledgment(MessageIdAdvUtils.discardBatch(msgId), ackType, properties, msgIdAdv);
         } else {
-            return addAcknowledgment(msgId, ackType, properties, null);
+            return addAcknowledgment(msgIdAdv, ackType, properties, null);
         }
     }
 
     private CompletableFuture addIndividualAcknowledgment(
-            MessageIdImpl msgId,
-            @Nullable BatchMessageIdImpl batchMessageId,
-            Function> individualAckFunction,
-            Function> batchAckFunction) {
+            MessageIdAdv msgId,
+            @Nullable MessageIdAdv batchMessageId,
+            Function> individualAckFunction,
+            Function> batchAckFunction) {
         if (batchMessageId != null) {
             consumer.onAcknowledge(batchMessageId, null);
         } else {
             consumer.onAcknowledge(msgId, null);
         }
-        if (batchMessageId == null || batchMessageId.ackIndividual()) {
+        if (batchMessageId == null || MessageIdAdvUtils.acknowledge(batchMessageId, true)) {
             consumer.getStats().incrementNumAcksSent((batchMessageId != null) ? batchMessageId.getBatchSize() : 1);
             consumer.getUnAckedMessageTracker().remove(msgId);
             if (consumer.getPossibleSendToDeadLetterTopicMessages() != null) {
@@ -215,10 +212,10 @@ private CompletableFuture addIndividualAcknowledgment(
         }
     }
 
-    private CompletableFuture addAcknowledgment(MessageIdImpl msgId,
+    private CompletableFuture addAcknowledgment(MessageIdAdv msgId,
                                                       AckType ackType,
                                                       Map properties,
-                                                      @Nullable BatchMessageIdImpl batchMessageId) {
+                                                      @Nullable MessageIdAdv batchMessageId) {
         switch (ackType) {
             case Individual:
                 return addIndividualAcknowledgment(msgId,
@@ -231,15 +228,12 @@ private CompletableFuture addAcknowledgment(MessageIdImpl msgId,
                 } else {
                     consumer.onAcknowledgeCumulative(msgId, null);
                 }
-                if (batchMessageId == null || batchMessageId.ackCumulative()) {
+                if (batchMessageId == null || MessageIdAdvUtils.acknowledge(batchMessageId, false)) {
                     return doCumulativeAck(msgId, properties, null);
                 } else if (batchIndexAckEnabled) {
                     return doCumulativeBatchIndexAck(batchMessageId, properties);
                 } else {
-                    if (!batchMessageId.getAcker().isPrevBatchCumulativelyAcked()) {
-                        doCumulativeAck(batchMessageId.prevBatchMessageId(), properties, null);
-                        batchMessageId.getAcker().setPrevBatchCumulativelyAcked(true);
-                    }
+                    doCumulativeAck(MessageIdAdvUtils.prevMessageId(batchMessageId), properties, null);
                     return CompletableFuture.completedFuture(null);
                 }
             default:
@@ -247,7 +241,7 @@ private CompletableFuture addAcknowledgment(MessageIdImpl msgId,
         }
     }
 
-    private CompletableFuture doIndividualAck(MessageIdImpl messageId, Map properties) {
+    private CompletableFuture doIndividualAck(MessageIdAdv messageId, Map properties) {
         if (acknowledgementGroupTimeMicros == 0 || (properties != null && !properties.isEmpty())) {
             // We cannot group acks if the delay is 0 or when there are properties attached to it. Fortunately that's an
             // uncommon condition since it's only used for the compaction subscription.
@@ -267,13 +261,13 @@ private CompletableFuture doIndividualAck(MessageIdImpl messageId, Map doIndividualAckAsync(MessageIdImpl messageId) {
+    private CompletableFuture doIndividualAckAsync(MessageIdAdv messageId) {
         pendingIndividualAcks.add(messageId);
         pendingIndividualBatchIndexAcks.remove(messageId);
         return CompletableFuture.completedFuture(null);
     }
 
-    private CompletableFuture doIndividualBatchAck(BatchMessageIdImpl batchMessageId,
+    private CompletableFuture doIndividualBatchAck(MessageIdAdv batchMessageId,
                                                          Map properties) {
         if (acknowledgementGroupTimeMicros == 0 || (properties != null && !properties.isEmpty())) {
             return doImmediateBatchIndexAck(batchMessageId, batchMessageId.getBatchIndex(),
@@ -283,7 +277,7 @@ private CompletableFuture doIndividualBatchAck(BatchMessageIdImpl batchMes
         }
     }
 
-    private CompletableFuture doIndividualBatchAck(BatchMessageIdImpl batchMessageId) {
+    private CompletableFuture doIndividualBatchAck(MessageIdAdv batchMessageId) {
         Optional readLock = acquireReadLock();
         try {
             doIndividualBatchAckAsync(batchMessageId);
@@ -296,7 +290,7 @@ private CompletableFuture doIndividualBatchAck(BatchMessageIdImpl batchMes
         }
     }
 
-    private CompletableFuture doCumulativeAck(MessageIdImpl messageId, Map properties,
+    private CompletableFuture doCumulativeAck(MessageIdAdv messageId, Map properties,
                                                     BitSetRecyclable bitSet) {
         consumer.getStats().incrementNumAcksSent(consumer.getUnAckedMessageTracker().removeMessagesTill(messageId));
         if (acknowledgementGroupTimeMicros == 0 || (properties != null && !properties.isEmpty())) {
@@ -314,29 +308,29 @@ private CompletableFuture doCumulativeAck(MessageIdImpl messageId, Map doIndividualBatchAckAsync(BatchMessageIdImpl batchMessageId) {
+    private CompletableFuture doIndividualBatchAckAsync(MessageIdAdv msgId) {
         ConcurrentBitSetRecyclable bitSet = pendingIndividualBatchIndexAcks.computeIfAbsent(
-                batchMessageId.toMessageIdImpl(), __ -> {
-                    ConcurrentBitSetRecyclable value;
-                    if (batchMessageId.getAcker() != null
-                            && !(batchMessageId.getAcker() instanceof BatchMessageAckerDisabled)) {
-                        value = ConcurrentBitSetRecyclable.create(batchMessageId.getAcker().getBitSet());
+                MessageIdAdvUtils.discardBatch(msgId), __ -> {
+                    final BitSet ackSet = msgId.getAckSet();
+                    final ConcurrentBitSetRecyclable value;
+                    if (ackSet != null && !ackSet.isEmpty()) {
+                        value = ConcurrentBitSetRecyclable.create(ackSet);
                     } else {
                         value = ConcurrentBitSetRecyclable.create();
-                        value.set(0, batchMessageId.getOriginalBatchSize());
+                        value.set(0, msgId.getBatchSize());
                     }
                     return value;
                 });
-        bitSet.clear(batchMessageId.getBatchIndex());
+        bitSet.clear(msgId.getBatchIndex());
         return CompletableFuture.completedFuture(null);
     }
 
-    private void doCumulativeAckAsync(MessageIdImpl msgId, BitSetRecyclable bitSet) {
+    private void doCumulativeAckAsync(MessageIdAdv msgId, BitSetRecyclable bitSet) {
         // Handle concurrent updates from different threads
         lastCumulativeAck.update(msgId, bitSet);
     }
 
-    private CompletableFuture doCumulativeBatchIndexAck(BatchMessageIdImpl batchMessageId,
+    private CompletableFuture doCumulativeBatchIndexAck(MessageIdAdv batchMessageId,
                                                               Map properties) {
         if (acknowledgementGroupTimeMicros == 0 || (properties != null && !properties.isEmpty())) {
             return doImmediateBatchIndexAck(batchMessageId, batchMessageId.getBatchIndex(),
@@ -349,7 +343,7 @@ private CompletableFuture doCumulativeBatchIndexAck(BatchMessageIdImpl bat
         }
     }
 
-    private CompletableFuture doImmediateAck(MessageIdImpl msgId, AckType ackType, Map properties,
+    private CompletableFuture doImmediateAck(MessageIdAdv msgId, AckType ackType, Map properties,
                                                    BitSetRecyclable bitSet) {
         ClientCnx cnx = consumer.getClientCnx();
 
@@ -360,7 +354,7 @@ private CompletableFuture doImmediateAck(MessageIdImpl msgId, AckType ackT
         return newImmediateAckAndFlush(consumer.consumerId, msgId, bitSet, ackType, properties, cnx);
     }
 
-    private CompletableFuture doImmediateBatchIndexAck(BatchMessageIdImpl msgId, int batchIndex, int batchSize,
+    private CompletableFuture doImmediateBatchIndexAck(MessageIdAdv msgId, int batchIndex, int batchSize,
                                                              AckType ackType, Map properties) {
         ClientCnx cnx = consumer.getClientCnx();
 
@@ -369,8 +363,8 @@ private CompletableFuture doImmediateBatchIndexAck(BatchMessageIdImpl msgI
                     .ConnectException("Consumer connect fail! consumer state:" + consumer.getState()));
         }
         BitSetRecyclable bitSet;
-        if (msgId.getAcker() != null && !(msgId.getAcker() instanceof BatchMessageAckerDisabled)) {
-            bitSet = BitSetRecyclable.valueOf(msgId.getAcker().getBitSet().toLongArray());
+        if (msgId.getAckSet() != null) {
+            bitSet = BitSetRecyclable.valueOf(msgId.getAckSet().toLongArray());
         } else {
             bitSet = BitSetRecyclable.create();
             bitSet.set(0, batchSize);
@@ -382,7 +376,7 @@ private CompletableFuture doImmediateBatchIndexAck(BatchMessageIdImpl msgI
         }
 
         CompletableFuture completableFuture = newMessageAckCommandAndWrite(cnx, consumer.consumerId,
-                msgId.ledgerId, msgId.entryId, bitSet, ackType, properties, true, null, null);
+                msgId.getLedgerId(), msgId.getEntryId(), bitSet, ackType, properties, true, null, null);
         bitSet.recycle();
         return completableFuture;
     }
@@ -414,7 +408,7 @@ private void flushAsync(ClientCnx cnx) {
         boolean shouldFlush = false;
         if (lastCumulativeAckToFlush != null) {
             shouldFlush = true;
-            final MessageIdImpl messageId = lastCumulativeAckToFlush.getMessageId();
+            final MessageIdAdv messageId = lastCumulativeAckToFlush.getMessageId();
             newMessageAckCommandAndWrite(cnx, consumer.consumerId, messageId.getLedgerId(), messageId.getEntryId(),
                     lastCumulativeAckToFlush.getBitSetRecyclable(), AckType.Cumulative,
                     Collections.emptyMap(), false,
@@ -429,7 +423,7 @@ private void flushAsync(ClientCnx cnx) {
             if (Commands.peerSupportsMultiMessageAcknowledgment(cnx.getRemoteEndpointProtocolVersion())) {
                 // We can send 1 single protobuf command with all individual acks
                 while (true) {
-                    MessageIdImpl msgId = pendingIndividualAcks.pollFirst();
+                    MessageIdAdv msgId = pendingIndividualAcks.pollFirst();
                     if (msgId == null) {
                         break;
                     }
@@ -452,7 +446,7 @@ private void flushAsync(ClientCnx cnx) {
             } else {
                 // When talking to older brokers, send the acknowledgements individually
                 while (true) {
-                    MessageIdImpl msgId = pendingIndividualAcks.pollFirst();
+                    MessageIdAdv msgId = pendingIndividualAcks.pollFirst();
                     if (msgId == null) {
                         break;
                     }
@@ -465,12 +459,13 @@ private void flushAsync(ClientCnx cnx) {
         }
 
         if (!pendingIndividualBatchIndexAcks.isEmpty()) {
-            Iterator> iterator =
+            Iterator> iterator =
                     pendingIndividualBatchIndexAcks.entrySet().iterator();
 
             while (iterator.hasNext()) {
-                Map.Entry entry = iterator.next();
-                entriesToAck.add(Triple.of(entry.getKey().ledgerId, entry.getKey().entryId, entry.getValue()));
+                Map.Entry entry = iterator.next();
+                entriesToAck.add(Triple.of(
+                        entry.getKey().getLedgerId(), entry.getKey().getEntryId(), entry.getValue()));
                 iterator.remove();
             }
         }
@@ -509,7 +504,7 @@ public void close() {
         }
     }
 
-    private CompletableFuture newImmediateAckAndFlush(long consumerId, MessageIdImpl msgId,
+    private CompletableFuture newImmediateAckAndFlush(long consumerId, MessageIdAdv msgId,
                                                             BitSetRecyclable bitSet, AckType ackType,
                                                             Map map, ClientCnx cnx) {
         MessageIdImpl[] chunkMsgIds = this.consumer.unAckedChunkedMessageIdSequenceMap.remove(msgId);
@@ -535,7 +530,7 @@ private CompletableFuture newImmediateAckAndFlush(long consumerId, Message
                 completableFuture = CompletableFuture.completedFuture(null);
             }
         } else {
-            completableFuture = newMessageAckCommandAndWrite(cnx, consumerId, msgId.ledgerId, msgId.getEntryId(),
+            completableFuture = newMessageAckCommandAndWrite(cnx, consumerId, msgId.getLedgerId(), msgId.getEntryId(),
                     bitSet, ackType, map, true, null, null);
         }
         return completableFuture;
@@ -621,13 +616,13 @@ protected LastCumulativeAck initialValue() {
                     return new LastCumulativeAck();
                 }
             };
-    public static final MessageIdImpl DEFAULT_MESSAGE_ID = (MessageIdImpl) MessageIdImpl.earliest;
+    public static final MessageIdAdv DEFAULT_MESSAGE_ID = (MessageIdAdv) MessageId.earliest;
 
-    private volatile MessageIdImpl messageId = DEFAULT_MESSAGE_ID;
+    private volatile MessageIdAdv messageId = DEFAULT_MESSAGE_ID;
     private BitSetRecyclable bitSetRecyclable = null;
     private boolean flushRequired = false;
 
-    public synchronized void update(final MessageIdImpl messageId, final BitSetRecyclable bitSetRecyclable) {
+    public synchronized void update(final MessageIdAdv messageId, final BitSetRecyclable bitSetRecyclable) {
         if (compareTo(messageId) < 0) {
             if (this.bitSetRecyclable != null && this.bitSetRecyclable != bitSetRecyclable) {
                 this.bitSetRecyclable.recycle();
@@ -662,25 +657,22 @@ public synchronized void reset() {
         flushRequired = false;
     }
 
-    public synchronized int compareTo(MessageId messageId) {
-        if (this.messageId instanceof BatchMessageIdImpl && (!(messageId instanceof BatchMessageIdImpl))) {
-            final BatchMessageIdImpl lhs = (BatchMessageIdImpl) this.messageId;
-            final MessageIdImpl rhs = (MessageIdImpl) messageId;
-            return MessageIdImpl.messageIdCompare(
-                    lhs.getLedgerId(), lhs.getEntryId(), lhs.getPartitionIndex(), lhs.getBatchIndex(),
-                    rhs.getLedgerId(), rhs.getEntryId(), rhs.getPartitionIndex(), Integer.MAX_VALUE);
-        } else if (messageId instanceof BatchMessageIdImpl && (!(this.messageId instanceof BatchMessageIdImpl))){
-            final MessageIdImpl lhs = this.messageId;
-            final BatchMessageIdImpl rhs = (BatchMessageIdImpl) messageId;
-            return MessageIdImpl.messageIdCompare(
-                    lhs.getLedgerId(), lhs.getEntryId(), lhs.getPartitionIndex(), Integer.MAX_VALUE,
-                    rhs.getLedgerId(), rhs.getEntryId(), rhs.getPartitionIndex(), rhs.getBatchIndex());
-        } else {
-            return this.messageId.compareTo(messageId);
+    public synchronized int compareTo(MessageIdAdv messageId) {
+        int result = Long.compare(this.messageId.getLedgerId(), messageId.getLedgerId());
+        if (result != 0) {
+            return result;
+        }
+        result = Long.compare(this.messageId.getEntryId(), messageId.getEntryId());
+        if (result != 0) {
+            return result;
         }
+        return Integer.compare(
+                (this.messageId.getBatchIndex() >= 0) ? this.messageId.getBatchIndex() : Integer.MAX_VALUE,
+                (messageId.getBatchIndex() >= 0) ? messageId.getBatchIndex() : Integer.MAX_VALUE
+        );
     }
 
-    private synchronized void set(final MessageIdImpl messageId, final BitSetRecyclable bitSetRecyclable) {
+    private synchronized void set(final MessageIdAdv messageId, final BitSetRecyclable bitSetRecyclable) {
         this.messageId = messageId;
         this.bitSetRecyclable = bitSetRecyclable;
     }
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ResetCursorData.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ResetCursorData.java
index 06f79024c4f24..1c4230470dbc2 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ResetCursorData.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ResetCursorData.java
@@ -22,6 +22,8 @@
 import lombok.Data;
 import lombok.NoArgsConstructor;
 import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.MessageIdAdv;
+import org.apache.pulsar.client.api.TopicMessageId;
 
 @Data
 @NoArgsConstructor
@@ -67,18 +69,12 @@ private ResetCursorData(String position) {
     }
 
     public ResetCursorData(MessageId messageId) {
-        if (messageId instanceof BatchMessageIdImpl) {
-            BatchMessageIdImpl batchMessageId = (BatchMessageIdImpl) messageId;
-            this.ledgerId = batchMessageId.getLedgerId();
-            this.entryId = batchMessageId.getEntryId();
-            this.batchIndex = batchMessageId.getBatchIndex();
-            this.partitionIndex = batchMessageId.partitionIndex;
-        } else if (messageId instanceof MessageIdImpl) {
-            MessageIdImpl messageIdImpl = (MessageIdImpl) messageId;
-            this.ledgerId = messageIdImpl.getLedgerId();
-            this.entryId = messageIdImpl.getEntryId();
-            this.partitionIndex = messageIdImpl.partitionIndex;
-        }  else if (messageId instanceof TopicMessageIdImpl) {
+        MessageIdAdv messageIdAdv = (MessageIdAdv) messageId;
+        this.ledgerId = messageIdAdv.getLedgerId();
+        this.entryId = messageIdAdv.getEntryId();
+        this.batchIndex = messageIdAdv.getBatchIndex();
+        this.partitionIndex = messageIdAdv.getPartitionIndex();
+        if (messageId instanceof TopicMessageId) {
             throw new IllegalArgumentException("Not supported operation on partitioned-topic");
         }
     }
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java
index 941f18cf65a2c..189dc1c608379 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java
@@ -21,16 +21,12 @@
 import org.apache.pulsar.client.api.MessageId;
 import org.apache.pulsar.client.api.TopicMessageId;
 
-public class TopicMessageIdImpl implements TopicMessageId {
+public class TopicMessageIdImpl extends TopicMessageId.Impl {
 
-    /** This topicPartitionName is get from ConsumerImpl, it contains partition part. */
-    private final String topicPartitionName;
     private final String topicName;
-    private final MessageId messageId;
 
     public TopicMessageIdImpl(String topicPartitionName, String topicName, MessageId messageId) {
-        this.messageId = messageId;
-        this.topicPartitionName = topicPartitionName;
+        super(topicPartitionName, messageId);
         this.topicName = topicName;
     }
 
@@ -49,40 +45,21 @@ public String getTopicName() {
      */
     @Deprecated
     public String getTopicPartitionName() {
-        return this.topicPartitionName;
+        return getOwnerTopic();
     }
 
+    @Deprecated
     public MessageId getInnerMessageId() {
-        return messageId;
-    }
-
-    @Override
-    public String toString() {
-        return messageId.toString();
-    }
-
-    @Override
-    public byte[] toByteArray() {
-        return messageId.toByteArray();
-    }
-
-    @Override
-    public int hashCode() {
-        return messageId.hashCode();
+        return new MessageIdImpl(getLedgerId(), getEntryId(), getPartitionIndex());
     }
 
     @Override
     public boolean equals(Object obj) {
-        return messageId.equals(obj);
+        return super.equals(obj);
     }
 
     @Override
-    public int compareTo(MessageId o) {
-        return messageId.compareTo(o);
-    }
-
-    @Override
-    public String getOwnerTopic() {
-        return topicPartitionName;
+    public int hashCode() {
+        return super.hashCode();
     }
 }
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageImpl.java
index c3fcb0a16a383..d24ecbd6aa917 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageImpl.java
@@ -70,7 +70,7 @@ public MessageId getMessageId() {
 
     @Deprecated
     public MessageId getInnerMessageId() {
-        return MessageIdImpl.convertToMessageIdImpl(messageId);
+        return messageId.getInnerMessageId();
     }
 
     @Override
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java
index 42eb197d632d3..ae874b4da6d6b 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java
@@ -174,7 +174,8 @@ private void triggerZeroQueueSizeListener(final Message message) {
                 }
                 waitingOnListenerForZeroQueueSize = true;
                 trackMessage(message);
-                unAckedMessageTracker.add(normalizeMessageId(message.getMessageId()), message.getRedeliveryCount());
+                unAckedMessageTracker.add(
+                        MessageIdAdvUtils.discardBatch(message.getMessageId()), message.getRedeliveryCount());
                 listener.received(ZeroQueueConsumerImpl.this, beforeConsume(message));
             } catch (Throwable t) {
                 log.error("[{}][{}] Message listener error in processing unqueued message: {}", topic, subscription,
diff --git a/pulsar-client/src/main/resources/findbugsExclude.xml b/pulsar-client/src/main/resources/findbugsExclude.xml
index e5f8babe841b8..92ec9e934ee1e 100644
--- a/pulsar-client/src/main/resources/findbugsExclude.xml
+++ b/pulsar-client/src/main/resources/findbugsExclude.xml
@@ -1007,4 +1007,9 @@
         
         
     
+    
+        
+        
+        
+    
 
diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java
index ddca6951e49e1..0418a54c772cc 100644
--- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java
+++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java
@@ -38,6 +38,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.MessageIdAdv;
 import org.apache.pulsar.client.impl.conf.ClientConfigurationData;
 import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData;
 import org.apache.pulsar.client.util.TimedCompletableFuture;
@@ -61,7 +62,7 @@ public void setup() throws NoSuchFieldException, IllegalAccessException {
         eventLoopGroup = new NioEventLoopGroup(1);
         consumer = mock(ConsumerImpl.class);
         consumer.unAckedChunkedMessageIdSequenceMap =
-                ConcurrentOpenHashMap.newBuilder().build();
+                ConcurrentOpenHashMap.newBuilder().build();
         cnx = spy(new ClientCnxTest(new ClientConfigurationData(), eventLoopGroup));
         PulsarClientImpl client = mock(PulsarClientImpl.class);
         doReturn(client).when(consumer).getClient();
@@ -391,21 +392,21 @@ public void testBatchAckTrackerMultiAck(boolean isNeedReceipt) throws Exception
     public void testDoIndividualBatchAckAsync() throws Exception{
         ConsumerConfigurationData conf = new ConsumerConfigurationData<>();
         AcknowledgmentsGroupingTracker tracker = new PersistentAcknowledgmentsGroupingTracker(consumer, conf, eventLoopGroup);
-        MessageId messageId1 = new BatchMessageIdImpl(5, 1, 0, 3, 10, BatchMessageAckerDisabled.INSTANCE);
+        MessageId messageId1 = new BatchMessageIdImpl(5, 1, 0, 3, 10, null);
         BitSet bitSet = new BitSet(20);
         for(int i = 0; i < 20; i ++) {
             bitSet.set(i, true);
         }
-        MessageId messageId2 = new BatchMessageIdImpl(3, 2, 0, 5, 20, BatchMessageAcker.newAcker(bitSet));
+        MessageId messageId2 = new BatchMessageIdImpl(3, 2, 0, 5, 20, bitSet);
         Method doIndividualBatchAckAsync = PersistentAcknowledgmentsGroupingTracker.class
-                .getDeclaredMethod("doIndividualBatchAckAsync", BatchMessageIdImpl.class);
+                .getDeclaredMethod("doIndividualBatchAckAsync", MessageIdAdv.class);
         doIndividualBatchAckAsync.setAccessible(true);
         doIndividualBatchAckAsync.invoke(tracker, messageId1);
         doIndividualBatchAckAsync.invoke(tracker, messageId2);
         Field pendingIndividualBatchIndexAcks = PersistentAcknowledgmentsGroupingTracker.class.getDeclaredField("pendingIndividualBatchIndexAcks");
         pendingIndividualBatchIndexAcks.setAccessible(true);
-        ConcurrentHashMap batchIndexAcks =
-                (ConcurrentHashMap) pendingIndividualBatchIndexAcks.get(tracker);
+        ConcurrentHashMap batchIndexAcks =
+                (ConcurrentHashMap) pendingIndividualBatchIndexAcks.get(tracker);
         MessageIdImpl position1 = new MessageIdImpl(5, 1, 0);
         MessageIdImpl position2 = new MessageIdImpl(3, 2, 0);
         assertTrue(batchIndexAcks.containsKey(position1));
diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageAckerDisabledTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageAckerDisabledTest.java
deleted file mode 100644
index 1b3795d878cd8..0000000000000
--- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageAckerDisabledTest.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.impl;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertTrue;
-
-import org.testng.annotations.Test;
-
-public class BatchMessageAckerDisabledTest {
-
-    @Test
-    public void testAckIndividual() {
-        for (int i = 0; i < 10; i++) {
-            assertTrue(BatchMessageAckerDisabled.INSTANCE.ackIndividual(i));
-        }
-    }
-
-    @Test
-    public void testAckCumulative() {
-        for (int i = 0; i < 10; i++) {
-            assertTrue(BatchMessageAckerDisabled.INSTANCE.ackCumulative(i));
-        }
-    }
-
-    @Test
-    public void testGetOutstandingAcks() {
-        assertEquals(0, BatchMessageAckerDisabled.INSTANCE.getOutstandingAcks());
-    }
-
-}
diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageAckerTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageAckerTest.java
deleted file mode 100644
index d31fd18cba971..0000000000000
--- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageAckerTest.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.impl;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertTrue;
-
-import org.testng.Assert;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import java.util.BitSet;
-
-public class BatchMessageAckerTest {
-
-    private static final int BATCH_SIZE = 10;
-
-    private BatchMessageAcker acker;
-
-    @BeforeMethod
-    public void setup() {
-        acker = BatchMessageAcker.newAcker(10);
-    }
-
-    @Test
-    public void testAckers() {
-        assertEquals(BATCH_SIZE, acker.getOutstandingAcks());
-        assertEquals(BATCH_SIZE, acker.getBatchSize());
-
-        assertFalse(acker.ackIndividual(4));
-        for (int i = 0; i < BATCH_SIZE; i++) {
-            if (4 == i) {
-                assertFalse(acker.getBitSet().get(i));
-            } else {
-                assertTrue(acker.getBitSet().get(i));
-            }
-        }
-
-        assertFalse(acker.ackCumulative(6));
-        for (int i = 0; i < BATCH_SIZE; i++) {
-            if (i <= 6) {
-                assertFalse(acker.getBitSet().get(i));
-            } else {
-                assertTrue(acker.getBitSet().get(i));
-            }
-        }
-
-        for (int i = BATCH_SIZE - 1; i >= 8; i--) {
-            assertFalse(acker.ackIndividual(i));
-            assertFalse(acker.getBitSet().get(i));
-        }
-
-        assertTrue(acker.ackIndividual(7));
-        assertEquals(0, acker.getOutstandingAcks());
-    }
-
-    @Test
-    public void testBitSetAcker() {
-        BitSet bitSet = BitSet.valueOf(acker.getBitSet().toLongArray());
-        BatchMessageAcker bitSetAcker = BatchMessageAcker.newAcker(bitSet);
-
-        Assert.assertEquals(acker.getBitSet(), bitSetAcker.getBitSet());
-        Assert.assertEquals(acker.getOutstandingAcks(), bitSetAcker.getOutstandingAcks());
-    }
-
-}
diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageIdImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageIdImplTest.java
index 6bf9cd943483f..10d805cdc4db3 100644
--- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageIdImplTest.java
+++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageIdImplTest.java
@@ -20,13 +20,8 @@
 
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertNotEquals;
-import static org.testng.Assert.assertTrue;
-import static org.testng.Assert.fail;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectWriter;
 import java.io.IOException;
 import java.util.Collections;
-import org.apache.pulsar.common.util.ObjectMapperFactory;
 import org.testng.annotations.Test;
 
 public class BatchMessageIdImplTest {
@@ -123,36 +118,10 @@ public void hashCodeUnbatchedTest() {
         assertEquals(batchMsgId2.hashCode(), msgId2.hashCode());
     }
 
-    @Test
-    public void deserializationTest() {
-        // initialize BitSet with null
-        BatchMessageAcker ackerDisabled = new BatchMessageAcker(null, 0);
-        BatchMessageIdImpl batchMsgId = new BatchMessageIdImpl(0, 0, 0, 0, 0, ackerDisabled);
-
-        ObjectWriter writer = ObjectMapperFactory.create().writerWithDefaultPrettyPrinter();
-
-        try {
-            writer.writeValueAsString(batchMsgId);
-            fail("Shouldn't be deserialized");
-        } catch (JsonProcessingException e) {
-            // expected
-            assertTrue(e.getCause() instanceof NullPointerException);
-        }
-
-        // use the default BatchMessageAckerDisabled
-        BatchMessageIdImpl batchMsgIdToDeserialize = new BatchMessageIdImpl(0, 0, 0, 0);
-
-        try {
-            writer.writeValueAsString(batchMsgIdToDeserialize);
-        } catch (JsonProcessingException e) {
-            fail("Should be successful");
-        }
-    }
-
     @Test
     public void serializeAndDeserializeTest() throws IOException {
         BatchMessageIdImpl batchMessageId = new BatchMessageIdImpl(1, 1, 0,
-            1, 10, BatchMessageAcker.newAcker(10));
+            1, 10, BatchMessageIdImpl.newAckSet(10));
         byte[] serialized = batchMessageId.toByteArray();
         BatchMessageIdImpl deserialized = (BatchMessageIdImpl) MessageIdImpl.fromByteArray(serialized);
         assertEquals(deserialized.getBatchSize(), batchMessageId.getBatchSize());
diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MessageIdSerializationTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MessageIdSerializationTest.java
index 7f029635241de..4173d6439b931 100644
--- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MessageIdSerializationTest.java
+++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MessageIdSerializationTest.java
@@ -43,8 +43,7 @@ public void testProtobufSerialization2() throws Exception {
 
     @Test
     public void testBatchSizeNotSet() throws Exception {
-        MessageId id = new BatchMessageIdImpl(1L, 2L, 3, 4, -1,
-                BatchMessageAckerDisabled.INSTANCE);
+        MessageId id = new BatchMessageIdImpl(1L, 2L, 3, 4, -1, null);
         byte[] serialized = id.toByteArray();
         assertEquals(MessageId.fromByteArray(serialized), id);
         assertEquals(MessageId.fromByteArrayWithTopic(serialized, "my-topic"), id);
diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java
index 28cce0fe62209..7df173da0f195 100644
--- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java
+++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java
@@ -48,6 +48,7 @@
 import org.apache.commons.lang3.StringUtils;
 import org.apache.pulsar.client.api.CompressionType;
 import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.MessageIdAdv;
 import org.apache.pulsar.client.api.SubscriptionInitialPosition;
 import org.apache.pulsar.client.impl.MessageIdImpl;
 import org.apache.pulsar.client.impl.auth.AuthenticationDataBasic;
@@ -325,7 +326,7 @@ public static String getFullyQualifiedInstanceId(String tenant, String namespace
     }
 
     public static final long getSequenceId(MessageId messageId) {
-        MessageIdImpl msgId = MessageIdImpl.convertToMessageIdImpl(messageId);
+        MessageIdAdv msgId = (MessageIdAdv) messageId;
         long ledgerId = msgId.getLedgerId();
         long entryId = msgId.getEntryId();
 
diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java
index 10efc91ccdaad..ff0bfd391e80e 100644
--- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java
+++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSink.java
@@ -26,6 +26,7 @@
 import com.google.common.collect.Maps;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -389,6 +390,23 @@ static class BatchMessageSequenceRef {
         int batchIdx;
     }
 
+    private static Method getMethodOfMessageId(MessageId messageId, String name) throws NoSuchMethodException {
+        Class clazz = messageId.getClass();
+        NoSuchMethodException firstException = null;
+        while (clazz != null) {
+            try {
+                return clazz.getDeclaredMethod(name);
+            } catch (NoSuchMethodException e) {
+                if (firstException == null) {
+                    firstException = e;
+                }
+                clazz = clazz.getSuperclass();
+            }
+        }
+        assert firstException != null;
+        throw firstException;
+    }
+
     @VisibleForTesting
     static BatchMessageSequenceRef getMessageSequenceRefForBatchMessage(MessageId messageId) {
         long ledgerId;
@@ -396,23 +414,17 @@ static BatchMessageSequenceRef getMessageSequenceRefForBatchMessage(MessageId me
         int batchIdx;
         try {
             try {
-                messageId = (MessageId) messageId.getClass().getDeclaredMethod("getInnerMessageId").invoke(messageId);
-            } catch (NoSuchMethodException noSuchMethodException) {
-                // not a TopicMessageIdImpl
-            }
-
-            try {
-                batchIdx = (int) messageId.getClass().getDeclaredMethod("getBatchIndex").invoke(messageId);
+                batchIdx = (int) getMethodOfMessageId(messageId, "getBatchIndex").invoke(messageId);
+                if (batchIdx < 0) {
+                    return null;
+                }
             } catch (NoSuchMethodException noSuchMethodException) {
                 // not a BatchMessageIdImpl, returning null to use the standard sequenceId
                 return null;
             }
 
-            // if getBatchIndex exists it means messageId is a 'BatchMessageIdImpl' instance.
-            final Class messageIdImplClass = messageId.getClass().getSuperclass();
-
-            ledgerId = (long) messageIdImplClass.getDeclaredMethod("getLedgerId").invoke(messageId);
-            entryId = (long) messageIdImplClass.getDeclaredMethod("getEntryId").invoke(messageId);
+            ledgerId = (long) getMethodOfMessageId(messageId, "getLedgerId").invoke(messageId);
+            entryId = (long) getMethodOfMessageId(messageId, "getEntryId").invoke(messageId);
         } catch (IllegalAccessException | NoSuchMethodException | InvocationTargetException ex) {
             log.error("Unexpected error while retrieving sequenceId, messageId class: {}, error: {}",
                     messageId.getClass().getName(), ex.getMessage(), ex);

From 6f9c933745cdec28a5a9d69040bf2f0e223bc023 Mon Sep 17 00:00:00 2001
From: Vineeth 
Date: Sun, 12 Feb 2023 14:05:04 -0800
Subject: [PATCH 174/174] [feat] [broker] PIP-188 support blue-green cluster
 migration [part-2]

---
 .../pulsar/broker/service/Replicator.java     |   2 +
 .../pulsar/broker/service/ServerCnx.java      |  24 +-
 .../apache/pulsar/broker/service/Topic.java   |   2 +
 .../NonPersistentReplicator.java              |   2 +-
 .../nonpersistent/NonPersistentTopic.java     |   5 +
 .../persistent/PersistentReplicator.java      |   4 +-
 .../service/persistent/PersistentTopic.java   |  21 +-
 .../auth/MockedPulsarServiceBaseTest.java     |   8 +-
 .../broker/service/ClusterMigrationTest.java  | 229 +++++++++++++++---
 9 files changed, 253 insertions(+), 44 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java
index b8d13256d225a..482fa2cbd2300 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java
@@ -49,4 +49,6 @@ default Optional getRateLimiter() {
     }
 
     boolean isConnected();
+
+    long getNumberOfEntriesInBacklog();
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
index 4fc79a124acd8..b995dd6628944 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
@@ -1578,13 +1578,23 @@ private void buildProducerAndAddTopic(Topic topic, long producerId, String produ
             if (ex.getCause() instanceof BrokerServiceException.TopicMigratedException) {
                 Optional clusterURL = getMigratedClusterUrl(service.getPulsar());
                 if (clusterURL.isPresent()) {
-                    log.info("[{}] redirect migrated producer to topic {}: producerId={}, {}", remoteAddress, topicName,
-                            producerId, ex.getCause().getMessage());
-                    commandSender.sendTopicMigrated(ResourceType.Producer, producerId,
-                            clusterURL.get().getBrokerServiceUrl(), clusterURL.get().getBrokerServiceUrlTls());
-                    closeProducer(producer);
-                    return null;
-
+                    if (topic.isReplicationBacklogExist()) {
+                        log.info("Topic {} is migrated but replication backlog exist: "
+                                        + "producerId = {}, producerName = {}, {}", topicName,
+                                producerId, producerName, ex.getCause().getMessage());
+                    } else {
+                        log.info("[{}] redirect migrated producer to topic {}: "
+                                        + "producerId={}, producerName = {}, {}", remoteAddress,
+                                topicName, producerId, producerName, ex.getCause().getMessage());
+                        boolean msgSent = commandSender.sendTopicMigrated(ResourceType.Producer, producerId,
+                                clusterURL.get().getBrokerServiceUrl(), clusterURL.get().getBrokerServiceUrlTls());
+                        if (!msgSent) {
+                            log.info("client doesn't support topic migration handling {}-{}-{}", topic,
+                                    remoteAddress, producerId);
+                        }
+                        closeProducer(producer);
+                        return null;
+                    }
                 } else {
                     log.warn("[{}] failed producer because migration url not configured topic {}: producerId={}, {}",
                             remoteAddress, topicName, producerId, ex.getCause().getMessage());
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java
index e6a29368dbb85..7657d77e1299f 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java
@@ -234,6 +234,8 @@ CompletableFuture createSubscription(String subscriptionName, Init
 
     boolean isBrokerPublishRateExceeded();
 
+    boolean isReplicationBacklogExist();
+
     void disableCnxAutoRead();
 
     void enableCnxAutoRead();
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java
index 7ad231926189b..514db4219db98 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java
@@ -248,7 +248,7 @@ protected Position getReplicatorReadPosition() {
     }
 
     @Override
-    protected long getNumberOfEntriesInBacklog() {
+    public long getNumberOfEntriesInBacklog() {
         // No-op
         return 0;
     }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java
index a0a8462a22753..317b8df6b9a82 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java
@@ -236,6 +236,11 @@ public void checkMessageDeduplicationInfo() {
         // No-op
     }
 
+    @Override
+    public boolean isReplicationBacklogExist() {
+        return false;
+    }
+
     @Override
     public void removeProducer(Producer producer) {
         checkArgument(producer.getTopic() == this);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java
index f38bcc71582a1..a556237f4342c 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java
@@ -165,8 +165,8 @@ protected Position getReplicatorReadPosition() {
     }
 
     @Override
-    protected long getNumberOfEntriesInBacklog() {
-        return cursor.getNumberOfEntriesInBacklog(false);
+    public long getNumberOfEntriesInBacklog() {
+        return cursor.getNumberOfEntriesInBacklog(true);
     }
 
     @Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
index 18a662c4b7a38..0374fc98212f2 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
@@ -575,6 +575,7 @@ public void addComplete(Position pos, ByteBuf entryData, Object ctx) {
 
     @Override
     public synchronized void addFailed(ManagedLedgerException exception, Object ctx) {
+        PublishContext callback = (PublishContext) ctx;
         if (exception instanceof ManagedLedgerFencedException) {
             // If the managed ledger has been fenced, we cannot continue using it. We need to close and reopen
             close();
@@ -587,7 +588,11 @@ public synchronized void addFailed(ManagedLedgerException exception, Object ctx)
                 List> futures = new ArrayList<>();
                 // send migration url metadata to producers before disconnecting them
                 if (isMigrated()) {
-                    producers.forEach((__, producer) -> producer.topicMigrated(getMigratedClusterUrl()));
+                    if (isReplicationBacklogExist()) {
+                        log.info("Topic {} is migrated but replication backlog exists. Closing producers.", topic);
+                    } else {
+                        producers.forEach((__, producer) -> producer.topicMigrated(getMigratedClusterUrl()));
+                    }
                 }
                 producers.forEach((__, producer) -> futures.add(producer.disconnect()));
                 disconnectProducersFuture = FutureUtil.waitForAll(futures);
@@ -599,8 +604,6 @@ public synchronized void addFailed(ManagedLedgerException exception, Object ctx)
                 return null;
             });
 
-            PublishContext callback = (PublishContext) ctx;
-
             if (exception instanceof ManagedLedgerAlreadyClosedException) {
                 if (log.isDebugEnabled()) {
                     log.debug("[{}] Failed to persist msg in store: {}", topic, exception.getMessage());
@@ -2510,6 +2513,18 @@ public CompletableFuture checkClusterMigration() {
         }
     }
 
+    public boolean isReplicationBacklogExist() {
+        ConcurrentOpenHashMap replicators = getReplicators();
+        if (replicators != null) {
+            for (Replicator replicator : replicators.values()) {
+                if (replicator.getNumberOfEntriesInBacklog() != 0) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
     @Override
     public void checkGC() {
         if (!isDeleteWhileInactive()) {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java
index cd31f9150e619..3fe8c22b1c4de 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java
@@ -112,7 +112,7 @@ public abstract class MockedPulsarServiceBaseTest extends TestRetrySupport {
     protected URI lookupUrl;
 
     protected boolean isTcpLookup = false;
-    protected static final String configClusterName = "test";
+    protected String configClusterName = "test";
 
     protected boolean enableBrokerInterceptor = false;
 
@@ -120,6 +120,12 @@ public MockedPulsarServiceBaseTest() {
         resetConfig();
     }
 
+    protected void setupWithClusterName(String clusterName) throws Exception {
+        this.conf.setClusterName(clusterName);
+        this.configClusterName = clusterName;
+        this.internalSetup();
+    }
+
     protected PulsarService getPulsar() {
         return pulsar;
     }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java
index f376ea4541737..df4f66c43d2b4 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java
@@ -18,10 +18,12 @@
  */
 package org.apache.pulsar.broker.service;
 
+import static java.lang.Thread.sleep;
 import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.retryStrategically;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
 import static org.testng.Assert.assertTrue;
 
 import java.lang.reflect.Method;
@@ -59,7 +61,8 @@ public class ClusterMigrationTest {
     protected String methodName;
 
     String namespace = "pulsar/migrationNs";
-    TestBroker broker1, broker2;
+    TestBroker broker1, broker2, broker3, broker4;
+
     URL url1;
     URL urlTls1;
     PulsarService pulsar1;
@@ -71,6 +74,16 @@ public class ClusterMigrationTest {
     PulsarService pulsar2;
     PulsarAdmin admin2;
 
+    URL url3;
+    URL urlTls3;
+    PulsarService pulsar3;
+    PulsarAdmin admin3;
+
+    URL url4;
+    URL urlTls4;
+    PulsarService pulsar4;
+    PulsarAdmin admin4;
+
     @DataProvider(name = "TopicsubscriptionTypes")
     public Object[][] subscriptionTypes() {
         return new Object[][] {
@@ -91,9 +104,10 @@ public void setup() throws Exception {
 
         log.info("--- Starting ReplicatorTestBase::setup ---");
 
-        broker1 = new TestBroker();
-        broker2 = new TestBroker();
-        String clusterName = broker1.getClusterName();
+        broker1 = new TestBroker("r1");
+        broker2 = new TestBroker("r2");
+        broker3 = new TestBroker("r3");
+        broker4 = new TestBroker("r4");
 
         pulsar1 = broker1.getPulsarService();
         url1 = new URL(pulsar1.getWebServiceAddress());
@@ -105,32 +119,81 @@ public void setup() throws Exception {
         urlTls2 = new URL(pulsar2.getWebServiceAddressTls());
         admin2 = PulsarAdmin.builder().serviceHttpUrl(url2.toString()).build();
 
-        // Start region 3
+        pulsar3 = broker3.getPulsarService();
+        url3 = new URL(pulsar3.getWebServiceAddress());
+        urlTls3 = new URL(pulsar3.getWebServiceAddressTls());
+        admin3 = PulsarAdmin.builder().serviceHttpUrl(url3.toString()).build();
 
-        // Provision the global namespace
-        admin1.clusters().createCluster(clusterName,
+        pulsar4 = broker4.getPulsarService();
+        url4 = new URL(pulsar4.getWebServiceAddress());
+        urlTls4 = new URL(pulsar4.getWebServiceAddressTls());
+        admin4 = PulsarAdmin.builder().serviceHttpUrl(url4.toString()).build();
+
+
+        admin1.clusters().createCluster("r1",
                 ClusterData.builder().serviceUrl(url1.toString()).serviceUrlTls(urlTls1.toString())
                         .brokerServiceUrl(pulsar1.getBrokerServiceUrl())
                         .brokerServiceUrlTls(pulsar1.getBrokerServiceUrlTls()).build());
-        admin2.clusters().createCluster(clusterName,
+        admin3.clusters().createCluster("r1",
+                ClusterData.builder().serviceUrl(url1.toString()).serviceUrlTls(urlTls1.toString())
+                        .brokerServiceUrl(pulsar1.getBrokerServiceUrl())
+                        .brokerServiceUrlTls(pulsar1.getBrokerServiceUrlTls()).build());
+        admin2.clusters().createCluster("r2",
+                ClusterData.builder().serviceUrl(url2.toString()).serviceUrlTls(urlTls2.toString())
+                        .brokerServiceUrl(pulsar2.getBrokerServiceUrl())
+                        .brokerServiceUrlTls(pulsar2.getBrokerServiceUrlTls()).build());
+        admin4.clusters().createCluster("r2",
                 ClusterData.builder().serviceUrl(url2.toString()).serviceUrlTls(urlTls2.toString())
                         .brokerServiceUrl(pulsar2.getBrokerServiceUrl())
                         .brokerServiceUrlTls(pulsar2.getBrokerServiceUrlTls()).build());
 
+        admin1.clusters().createCluster("r3",
+                ClusterData.builder().serviceUrl(url3.toString()).serviceUrlTls(urlTls3.toString())
+                        .brokerServiceUrl(pulsar3.getBrokerServiceUrl())
+                        .brokerServiceUrlTls(pulsar3.getBrokerServiceUrlTls()).build());
+
+        admin3.clusters().createCluster("r3",
+                ClusterData.builder().serviceUrl(url3.toString()).serviceUrlTls(urlTls3.toString())
+                        .brokerServiceUrl(pulsar3.getBrokerServiceUrl())
+                        .brokerServiceUrlTls(pulsar3.getBrokerServiceUrlTls()).build());
+
+        admin2.clusters().createCluster("r4",
+                ClusterData.builder().serviceUrl(url4.toString()).serviceUrlTls(urlTls4.toString())
+                        .brokerServiceUrl(pulsar4.getBrokerServiceUrl())
+                        .brokerServiceUrlTls(pulsar4.getBrokerServiceUrlTls()).build());
+        admin4.clusters().createCluster("r4",
+                ClusterData.builder().serviceUrl(url4.toString()).serviceUrlTls(urlTls4.toString())
+                        .brokerServiceUrl(pulsar4.getBrokerServiceUrl())
+                        .brokerServiceUrlTls(pulsar4.getBrokerServiceUrlTls()).build());
+
+        // Setting r3 as replication cluster for r1
         admin1.tenants().createTenant("pulsar",
-                new TenantInfoImpl(Sets.newHashSet("appid1", "appid2", "appid3"), Sets.newHashSet(clusterName)));
-        admin1.namespaces().createNamespace(namespace, Sets.newHashSet(clusterName));
-
+                new TenantInfoImpl(Sets.newHashSet("appid1", "appid2", "appid3"), Sets.newHashSet("r1", "r3")));
+        admin3.tenants().createTenant("pulsar",
+                new TenantInfoImpl(Sets.newHashSet("appid1", "appid2", "appid3"), Sets.newHashSet("r1", "r3")));
+        admin1.namespaces().createNamespace(namespace, Sets.newHashSet("r1", "r3"));
+        admin3.namespaces().createNamespace(namespace);
+        admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1", "r3"));
+
+        // Setting r4 as replication cluster for r2
         admin2.tenants().createTenant("pulsar",
-                new TenantInfoImpl(Sets.newHashSet("appid1", "appid2", "appid3"), Sets.newHashSet(clusterName)));
-        admin2.namespaces().createNamespace(namespace, Sets.newHashSet(clusterName));
-
-        assertEquals(admin1.clusters().getCluster(clusterName).getServiceUrl(), url1.toString());
-        assertEquals(admin2.clusters().getCluster(clusterName).getServiceUrl(), url2.toString());
-        assertEquals(admin1.clusters().getCluster(clusterName).getBrokerServiceUrl(), pulsar1.getBrokerServiceUrl());
-        assertEquals(admin2.clusters().getCluster(clusterName).getBrokerServiceUrl(), pulsar2.getBrokerServiceUrl());
-
-        Thread.sleep(100);
+                new TenantInfoImpl(Sets.newHashSet("appid1", "appid2", "appid3"), Sets.newHashSet("r2", "r4")));
+        admin4.tenants().createTenant("pulsar",
+                new TenantInfoImpl(Sets.newHashSet("appid1", "appid2", "appid3"), Sets.newHashSet("r2", "r4")));
+        admin2.namespaces().createNamespace(namespace, Sets.newHashSet("r2", "r4"));
+        admin4.namespaces().createNamespace(namespace);
+        admin2.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r2", "r4"));
+
+        assertEquals(admin1.clusters().getCluster("r1").getServiceUrl(), url1.toString());
+        assertEquals(admin2.clusters().getCluster("r2").getServiceUrl(), url2.toString());
+        assertEquals(admin3.clusters().getCluster("r3").getServiceUrl(), url3.toString());
+        assertEquals(admin4.clusters().getCluster("r4").getServiceUrl(), url4.toString());
+        assertEquals(admin1.clusters().getCluster("r1").getBrokerServiceUrl(), pulsar1.getBrokerServiceUrl());
+        assertEquals(admin2.clusters().getCluster("r2").getBrokerServiceUrl(), pulsar2.getBrokerServiceUrl());
+        assertEquals(admin3.clusters().getCluster("r3").getBrokerServiceUrl(), pulsar3.getBrokerServiceUrl());
+        assertEquals(admin4.clusters().getCluster("r4").getBrokerServiceUrl(), pulsar4.getBrokerServiceUrl());
+
+        sleep(100);
         log.info("--- ReplicatorTestBase::setup completed ---");
 
     }
@@ -140,6 +203,8 @@ protected void cleanup() throws Exception {
         log.info("--- Shutting down ---");
         broker1.cleanup();
         broker2.cleanup();
+        broker3.cleanup();
+        broker4.cleanup();
     }
 
     @BeforeMethod(alwaysRun = true)
@@ -154,20 +219,19 @@ public void beforeMethod(Method m) throws Exception {
      * (3) Migrate topic to cluster-2
      * (4) Validate producer-1 is connected to cluster-2
      * (5) create consumer1, drain backlog and migrate and reconnect to cluster-2
-     * (6) Create new consumer2 with different subscription on cluster-1, 
+     * (6) Create new consumer2 with different subscription on cluster-1,
      *     which immediately migrate and reconnect to cluster-2
      * (7) Create producer-2 directly to cluster-2
-     * (8) Create producer-3 on cluster-1 which should be redirected to cluster-2 
+     * (8) Create producer-3 on cluster-1 which should be redirected to cluster-2
      * (8) Publish messages using producer1, producer2, and producer3
      * (9) Consume all messages by both consumer1 and consumer2
-     * (10) Create Producer/consumer on non-migrated cluster and verify their connection with cluster-1 
-     * (11) Restart Broker-1 and connect producer/consumer on cluster-1 
+     * (10) Create Producer/consumer on non-migrated cluster and verify their connection with cluster-1
+     * (11) Restart Broker-1 and connect producer/consumer on cluster-1
      * @throws Exception
      */
     @Test(dataProvider = "TopicsubscriptionTypes")
     public void testClusterMigration(boolean persistent, SubscriptionType subType) throws Exception {
         log.info("--- Starting ReplicatorTest::testClusterMigration ---");
-        persistent = false;
         final String topicName = BrokerTestUtil
                 .newUniqueName((persistent ? "persistent" : "non-persistent") + "://" + namespace + "/migrationTopic");
 
@@ -202,7 +266,7 @@ public void testClusterMigration(boolean persistent, SubscriptionType subType) t
         assertFalse(topic2.getProducers().isEmpty());
 
         ClusterUrl migratedUrl = new ClusterUrl(pulsar2.getBrokerServiceUrl(), pulsar2.getBrokerServiceUrlTls());
-        admin1.clusters().updateClusterMigration(broker2.getClusterName(), true, migratedUrl);
+        admin1.clusters().updateClusterMigration("r1", true, migratedUrl);
 
         retryStrategically((test) -> {
             try {
@@ -214,12 +278,16 @@ public void testClusterMigration(boolean persistent, SubscriptionType subType) t
             return false;
         }, 10, 500);
 
+
         topic1.checkClusterMigration().get();
 
+        log.info("before sending message");
+        sleep(1000);
         producer1.sendAsync("test1".getBytes());
 
         // producer is disconnected from cluster-1
         retryStrategically((test) -> topic1.getProducers().isEmpty(), 10, 500);
+        log.info("before asserting");
         assertTrue(topic1.getProducers().isEmpty());
 
         // create 3rd producer on cluster-1 which should be redirected to cluster-2
@@ -297,15 +365,116 @@ public void testClusterMigration(boolean persistent, SubscriptionType subType) t
         log.info("Successfully consumed messages by migrated consumers");
     }
 
+    @Test(dataProvider = "TopicsubscriptionTypes")
+    public void testClusterMigrationWithReplicationBacklog(boolean persistent, SubscriptionType subType) throws Exception {
+        log.info("--- Starting ReplicatorTest::testClusterMigrationWithReplicationBacklog ---");
+        persistent = true;
+        final String topicName = BrokerTestUtil
+                .newUniqueName((persistent ? "persistent" : "non-persistent") + "://" + namespace + "/migrationTopic");
+
+        @Cleanup
+        PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()).statsInterval(0, TimeUnit.SECONDS)
+                .build();
+        @Cleanup
+        PulsarClient client3 = PulsarClient.builder().serviceUrl(url3.toString()).statsInterval(0, TimeUnit.SECONDS)
+                .build();
+        // cluster-1 producer/consumer
+        Producer producer1 = client1.newProducer().topic(topicName).enableBatching(false)
+                .producerName("cluster1-1").messageRoutingMode(MessageRoutingMode.SinglePartition).create();
+        Consumer consumer1 = client1.newConsumer().topic(topicName).subscriptionType(subType)
+                .subscriptionName("s1").subscribe();
+
+        // cluster-3 consumer
+        Consumer consumer3 = client3.newConsumer().topic(topicName).subscriptionType(subType)
+                .subscriptionName("s1").subscribe();
+        AbstractTopic topic1 = (AbstractTopic) pulsar1.getBrokerService().getTopic(topicName, false).getNow(null).get();
+        retryStrategically((test) -> !topic1.getProducers().isEmpty(), 5, 500);
+        retryStrategically((test) -> !topic1.getSubscriptions().isEmpty(), 5, 500);
+        assertFalse(topic1.getProducers().isEmpty());
+        assertFalse(topic1.getSubscriptions().isEmpty());
+
+        // build backlog
+        consumer1.close();
+        retryStrategically((test) -> topic1.getReplicators().size() == 1, 10, 3000);
+        assertEquals(topic1.getReplicators().size(), 1);
+
+       // stop service in the replication cluster to build replication backlog
+        broker3.cleanup();
+        retryStrategically((test) -> broker3.getPulsarService() == null, 10, 1000);
+        assertNull(pulsar3.getBrokerService());
+
+        //publish messages into topic in "r1" cluster
+        int n = 5;
+        for (int i = 0; i < n; i++) {
+            producer1.send("test1".getBytes());
+        }
+        retryStrategically((test) -> topic1.isReplicationBacklogExist(), 10, 1000);
+        assertTrue(topic1.isReplicationBacklogExist());
+
+        @Cleanup
+        PulsarClient client2 = PulsarClient.builder().serviceUrl(url2.toString()).statsInterval(0, TimeUnit.SECONDS)
+                .build();
+        // cluster-2 producer/consumer
+        Producer producer2 = client2.newProducer().topic(topicName).enableBatching(false)
+                .producerName("cluster2-1").messageRoutingMode(MessageRoutingMode.SinglePartition).create();
+        AbstractTopic topic2 = (AbstractTopic) pulsar2.getBrokerService().getTopic(topicName, false).getNow(null).get();
+        log.info("name of topic 2 - {}", topic2.getName());
+        assertFalse(topic2.getProducers().isEmpty());
+
+        retryStrategically((test) -> topic2.getReplicators().size() == 1, 10, 2000);
+        log.info("replicators should be ready");
+        ClusterUrl migratedUrl = new ClusterUrl(pulsar2.getBrokerServiceUrl(), pulsar2.getBrokerServiceUrlTls());
+        admin1.clusters().updateClusterMigration("r1", true, migratedUrl);
+        log.info("update cluster migration called");
+        retryStrategically((test) -> {
+            try {
+                topic1.checkClusterMigration().get();
+                return true;
+            } catch (Exception e) {
+                // ok
+            }
+            return false;
+        }, 10, 500);
+
+        topic1.checkClusterMigration().get();
+
+        producer1.sendAsync("test1".getBytes());
+
+        // producer is disconnected from cluster-1
+        retryStrategically((test) -> topic1.getProducers().isEmpty(), 10, 500);
+        assertTrue(topic1.getProducers().isEmpty());
+
+        // verify that the disconnected producer is not redirected
+        // to replication cluster since there is replication backlog.
+        assertEquals(topic2.getProducers().size(), 1);
+
+        // Restart the service in cluster "r3".
+        broker3.restart();
+        retryStrategically((test) -> broker3.getPulsarService() != null, 10, 1000);
+        assertNotNull(broker3.getPulsarService());
+        pulsar3 = broker3.getPulsarService();
+
+        // verify that the replication backlog drains once service in cluster "r3" is restarted.
+        retryStrategically((test) -> !topic1.isReplicationBacklogExist(), 10, 1000);
+        assertFalse(topic1.isReplicationBacklogExist());
+
+        // verify that the producer1 is now is now connected to migrated cluster "r2" since backlog is cleared.
+        retryStrategically((test) -> topic2.getProducers().size()==2, 10, 500);
+        assertEquals(topic2.getProducers().size(), 2);
+    }
+
     static class TestBroker extends MockedPulsarServiceBaseTest {
 
-        public TestBroker() throws Exception {
+        private String clusterName;
+
+        public TestBroker(String clusterName) throws Exception {
+            this.clusterName = clusterName;
             setup();
         }
 
         @Override
         protected void setup() throws Exception {
-            super.internalSetup();
+            super.setupWithClusterName(clusterName);
         }
 
         public PulsarService getPulsarService() {
@@ -318,9 +487,9 @@ public String getClusterName() {
 
         @Override
         protected void cleanup() throws Exception {
-            internalCleanup();
+            stopBroker();
         }
-        
+
         public void restart() throws Exception {
             restartBroker();
         }